diff --git a/app.py b/app.py index 7bc60fb..d61d09f 100644 --- a/app.py +++ b/app.py @@ -1,8 +1,13 @@ from flask import Flask, jsonify, request, send_from_directory, send_file import os +import logging app = Flask(__name__) +# Set up logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + # Import our archive parser from parse_archive import ArchiveParser @@ -52,7 +57,7 @@ def get_more_posts(): filters_posts += [p] return jsonify(filters_posts) except Exception as e: - print(f"Error: {e}") + logger.error(f"Error fetching more posts: {e}") return jsonify({"error": "Failed to fetch more posts"}), 500 @app.route("/posts/total", methods=["GET"]) @@ -63,7 +68,7 @@ def get_total_posts(): total = archive_parser.get_total_posts() return jsonify({"total": total}) except Exception as e: - print(f"Error: {e}") + logger.error(f"Error fetching total posts: {e}") return jsonify({"error": "Failed to fetch total count"}), 500 @app.route("/api") @@ -77,6 +82,7 @@ def home(): "GET /posts/more?count=N": "Get next 10 posts, starting from index N", "GET /posts/total": "Get total number of posts", }, + "status": "healthy" } ) diff --git a/parse_archive.py b/parse_archive.py index 3e6a3dd..7138ea0 100644 --- a/parse_archive.py +++ b/parse_archive.py @@ -5,6 +5,8 @@ import urllib.parse import uuid from io import BytesIO from typing import Any, Dict, List +from concurrent.futures import ThreadPoolExecutor +import threading import requests from PIL import Image @@ -14,6 +16,10 @@ class ArchiveParser: def __init__(self): """Initialize the ArchiveParser with archive_dir from environment variable.""" self.archive_dir = os.environ.get("ARCHIVE_DIR", "/default/archive/path") + self.cache = {} + self.cache_lock = threading.Lock() + self.pre_cache_size = 300 # Pre-cache 300 articles ahead + self.cache_executor = ThreadPoolExecutor(max_workers=2) # For background caching def get_posts(self, count: int, start_index: int = 0) -> List[Dict[str, Any]]: """ @@ -31,6 +37,13 @@ class ArchiveParser: List[Dict[str, Any]]: List of post dictionaries """ try: + # Check if we have cached posts for this range + cache_key = f"posts_{start_index}_{count}" + with self.cache_lock: + if cache_key in self.cache: + print(f"Cache hit for {cache_key}") + return self.cache[cache_key] + # Use ls with tail and head commands for efficient pagination # Get all directories, skip the first start_index, then get count number of entries cmd = f"ls -1 {self.archive_dir} | tail -{start_index + count + 1} | head -{count}" @@ -52,6 +65,15 @@ class ArchiveParser: # Convert extracted posts to final format with proper IDs formatted_posts = [self._create_post(post) for post in posts] + + # Cache the results + with self.cache_lock: + self.cache[cache_key] = formatted_posts + + # Pre-cache the next batch in background + if start_index + count < self.get_total_posts(): + self._pre_cache_batch(start_index + count) + return formatted_posts except subprocess.CalledProcessError as e: @@ -119,6 +141,24 @@ class ArchiveParser: print(f"Error getting total posts: {e}") return 0 + def _pre_cache_batch(self, start_index: int): + """Pre-cache the next batch of posts in the background.""" + def cache_task(): + try: + # Pre-cache next 300 posts + next_posts = self.get_posts(self.pre_cache_size, start_index) + print(f"Pre-cached {len(next_posts)} posts starting from index {start_index}") + except Exception as e: + print(f"Error in pre-caching: {e}") + + # Submit to background thread + self.cache_executor.submit(cache_task) + + def clear_cache(self): + """Clear the cache.""" + with self.cache_lock: + self.cache.clear() + def _extract_real_title_from_url(self, base_url: str) -> str: """Extract the real title from base_url when title is 'Reddit - Prove your humanity'.""" if not base_url or not isinstance(base_url, str):