Preview images

src/server/parse_archive.py

@@ -3,12 +3,19 @@ import subprocess
 import json
 from typing import List, Dict, Any
 import uuid
+import urllib.parse
+from PIL import Image, ImageDraw, ImageFont
+import requests
+from bs4 import BeautifulSoup
+from io import BytesIO
 
 
 class ArchiveParser:
     def __init__(self):
         """Initialize the ArchiveParser with archive_dir from environment variable."""
         self.archive_dir = os.environ.get("ARCHIVE_DIR", "/default/archive/path")
+        self.cache_dir = os.path.join(os.path.dirname(__file__), "..", "..", "cache")
+        os.makedirs(self.cache_dir, exist_ok=True)
 
     def get_posts(self, count: int, start_index: int = 0) -> List[Dict[str, Any]]:
         """
@@ -28,7 +35,7 @@ class ArchiveParser:
         try:
             # Use ls with tail and head commands for efficient pagination
             # Get all directories, skip the first start_index, then get count number of entries
-            cmd = f"ls -1 {self.archive_dir} | tail -{start_index + 1} | head -{count}"
+            cmd = f"ls -1 {self.archive_dir} | tail -{start_index + count + 1} | head -{count}"
             print("Running cmd: " + cmd)
             result = subprocess.run(
                 cmd, shell=True, capture_output=True, text=True, check=True, timeout=30
@@ -129,9 +136,182 @@ class ArchiveParser:
                 "/Reddit_Logo.webp"  # Default to the logo in website directory
             )
 
+            # Generate HTML preview for the post URL
+            post["image"] = self._generate_preview_image(post["url"], post["id"])
+
             return post
 
         except Exception as e:
             print(f"Error converting index to post: {e}")
             # Return a minimal post if there's an error
             raise e
+
+    def _generate_preview_image(self, url: str, post_id: str) -> str:
+        """
+        Generate a preview image for the HTML page at the given URL.
+
+        This implementation extracts the largest image from the HTML page content,
+        falling back to a placeholder if no suitable images are found.
+
+        Args:
+            url (str): The URL to generate preview for
+            post_id (str): The ID of the post
+
+        Returns:
+            str: Path to the generated preview image
+        """
+        try:
+            # Validate the URL
+            parsed_url = urllib.parse.urlparse(url)
+            if not parsed_url.scheme or not parsed_url.netloc:
+                return "/Reddit_Logo.webp"
+
+            # Generate filename based on post ID
+            filename = f"{post_id}.webp"
+            image_path = os.path.join(self.cache_dir, filename)
+
+            # If image already exists, return its path
+            if os.path.exists(image_path):
+                return f"/cache/{filename}"
+
+            # Try to extract and use the largest image from the webpage content
+            image_extracted = self._extract_largest_image_from_webpage(url, image_path)
+
+            if not image_extracted:
+                # Fall back to creating a placeholder image
+                self._create_placeholder_image(url, image_path)
+
+            return f"/cache/{filename}"
+
+        except Exception as e:
+            print(f"Error generating preview for URL {url}: {e}")
+            # Return default fallback image in case of any error
+            return "/Reddit_Logo.webp"
+
+    def _create_placeholder_image(self, url: str, save_path: str):
+        """
+        Create a simple placeholder image when real preview isn't possible.
+
+        Args:
+            url (str): The URL that was attempted
+            save_path (str): Path to save the image
+        """
+        try:
+            # Create a simple placeholder image
+            img = Image.new("RGB", (800, 600), color=(73, 109, 137))
+            draw = ImageDraw.Draw(img)
+
+            # Try to get default font or use None (which will fall back to default)
+            try:
+                font = ImageFont.load_default()
+            except Exception:
+                font = None
+
+            # Draw some text on the image
+            draw.text(
+                (10, 10), "HTML Preview Placeholder", fill=(255, 255, 0), font=font
+            )
+            draw.text(
+                (10, 40),
+                f"URL: {url[:50]}{'...' if len(url) > 50 else ''}",
+                fill=(255, 255, 255),
+                font=font,
+            )
+
+            # Add instructions
+            draw.text(
+                (10, 80),
+                "Largest image from webpage would appear here",
+                fill=(255, 255, 255),
+                font=font,
+            )
+            draw.text(
+                (10, 100),
+                "This is a placeholder image",
+                fill=(255, 255, 255),
+                font=font,
+            )
+
+            # Save as webp image
+            img.save(save_path, "WEBP", quality=80)
+        except Exception as e:
+            print(f"Error creating placeholder image: {e}")
+            # If everything fails, we still have the default fallback
+            return
+
+    def _extract_largest_image_from_webpage(self, url: str, save_path: str) -> bool:
+        """
+        Extract the largest image from a webpage and save it.
+
+        Args:
+            url (str): URL of the webpage to extract images from
+            save_path (str): Path where to save the extracted image
+
+        Returns:
+            bool: True if successful, False otherwise
+        """
+        try:
+            # Fetch the webpage content
+            headers = {
+                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
+            }
+            response = requests.get(url, timeout=10, headers=headers)
+            response.raise_for_status()
+
+            # Parse HTML with BeautifulSoup
+            soup = BeautifulSoup(response.content, "html.parser")
+
+            # Find all img tags
+            img_tags = soup.find_all("img")
+
+            if not img_tags:
+                return False
+
+            # Get the largest image by size (height * width)
+            largest_img = None
+            max_area = 0
+            largest_img_src = None
+
+            for img in img_tags:
+                src = img.get("src") or img.get("data-src")
+                if not src:
+                    continue
+
+                # Resolve relative URLs
+                if not src.startswith(("http://", "https://")):
+                    from urllib.parse import urljoin
+
+                    src = urljoin(url, src)
+
+                try:
+                    # Get image dimensions by downloading it
+                    img_response = requests.get(src, timeout=10)
+                    img_response.raise_for_status()
+
+                    # Open image with PIL to get dimensions
+                    img_data = Image.open(BytesIO(img_response.content))
+                    area = img_data.width * img_data.height
+
+                    if area > max_area:
+                        max_area = area
+                        largest_img_src = src
+
+                except Exception as e:
+                    print(f"Could not process image from {src}: {e}")
+                    continue
+
+            # If we found a suitable image, download and save it
+            if largest_img_src:
+                img_response = requests.get(largest_img_src, timeout=10)
+                img_response.raise_for_status()
+
+                # Save as webp with quality 80
+                img = Image.open(BytesIO(img_response.content))
+                img.save(save_path, "WEBP", quality=80)
+                return True
+
+        except Exception as e:
+            print(f"Error extracting image from webpage: {e}")
+            return False
+
+        return False

src/server/requirements.txt

@@ -1,2 +1,5 @@
-Flask==2.3.3
-Flask-CORS==4.0.0
+Flask
+Flask-CORS
+Pillow
+beautifulsoup4
+requests