Implement backend pre-caching optimization for faster load times with 300-article prefetching

app.py

@@ -1,8 +1,13 @@
 from flask import Flask, jsonify, request, send_from_directory, send_file
 import os
+import logging
 
 app = Flask(__name__)
 
+# Set up logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
 # Import our archive parser
 from parse_archive import ArchiveParser
 
@@ -52,7 +57,7 @@ def get_more_posts():
                 filters_posts += [p]
         return jsonify(filters_posts)
     except Exception as e:
-        print(f"Error: {e}")
+        logger.error(f"Error fetching more posts: {e}")
         return jsonify({"error": "Failed to fetch more posts"}), 500
 
 @app.route("/posts/total", methods=["GET"])
@@ -63,7 +68,7 @@ def get_total_posts():
         total = archive_parser.get_total_posts()
         return jsonify({"total": total})
     except Exception as e:
-        print(f"Error: {e}")
+        logger.error(f"Error fetching total posts: {e}")
         return jsonify({"error": "Failed to fetch total count"}), 500
 
 @app.route("/api")
@@ -77,6 +82,7 @@ def home():
                 "GET /posts/more?count=N": "Get next 10 posts, starting from index N",
                 "GET /posts/total": "Get total number of posts",
             },
+            "status": "healthy"
         }
     )
 

parse_archive.py

@@ -5,6 +5,8 @@ import urllib.parse
 import uuid
 from io import BytesIO
 from typing import Any, Dict, List
+from concurrent.futures import ThreadPoolExecutor
+import threading
 
 import requests
 from PIL import Image
@@ -14,6 +16,10 @@ class ArchiveParser:
     def __init__(self):
         """Initialize the ArchiveParser with archive_dir from environment variable."""
         self.archive_dir = os.environ.get("ARCHIVE_DIR", "/default/archive/path")
+        self.cache = {}
+        self.cache_lock = threading.Lock()
+        self.pre_cache_size = 300  # Pre-cache 300 articles ahead
+        self.cache_executor = ThreadPoolExecutor(max_workers=2)  # For background caching
 
     def get_posts(self, count: int, start_index: int = 0) -> List[Dict[str, Any]]:
         """
@@ -31,6 +37,13 @@ class ArchiveParser:
             List[Dict[str, Any]]: List of post dictionaries
         """
         try:
+            # Check if we have cached posts for this range
+            cache_key = f"posts_{start_index}_{count}"
+            with self.cache_lock:
+                if cache_key in self.cache:
+                    print(f"Cache hit for {cache_key}")
+                    return self.cache[cache_key]
+            
             # Use ls with tail and head commands for efficient pagination
             # Get all directories, skip the first start_index, then get count number of entries
             cmd = f"ls -1 {self.archive_dir} | tail -{start_index + count + 1} | head -{count}"
@@ -52,6 +65,15 @@ class ArchiveParser:
 
             # Convert extracted posts to final format with proper IDs
             formatted_posts = [self._create_post(post) for post in posts]
+            
+            # Cache the results
+            with self.cache_lock:
+                self.cache[cache_key] = formatted_posts
+            
+            # Pre-cache the next batch in background
+            if start_index + count < self.get_total_posts():
+                self._pre_cache_batch(start_index + count)
+            
             return formatted_posts
 
         except subprocess.CalledProcessError as e:
@@ -119,6 +141,24 @@ class ArchiveParser:
             print(f"Error getting total posts: {e}")
             return 0
 
+    def _pre_cache_batch(self, start_index: int):
+        """Pre-cache the next batch of posts in the background."""
+        def cache_task():
+            try:
+                # Pre-cache next 300 posts
+                next_posts = self.get_posts(self.pre_cache_size, start_index)
+                print(f"Pre-cached {len(next_posts)} posts starting from index {start_index}")
+            except Exception as e:
+                print(f"Error in pre-caching: {e}")
+        
+        # Submit to background thread
+        self.cache_executor.submit(cache_task)
+        
+    def clear_cache(self):
+        """Clear the cache."""
+        with self.cache_lock:
+            self.cache.clear()
+
     def _extract_real_title_from_url(self, base_url: str) -> str:
         """Extract the real title from base_url when title is 'Reddit - Prove your humanity'."""
         if not base_url or not isinstance(base_url, str):