From 7de5f31a7c33c19e7f3815500b1e31cd3190ce1e Mon Sep 17 00:00:00 2001 From: Jarian Cottingham Date: Tue, 14 Oct 2025 19:09:59 -0500 Subject: [PATCH] Preview images --- src/server/parse_archive.py | 182 +++++++++++++++++++++++++++++++++++- src/server/requirements.txt | 7 +- 2 files changed, 186 insertions(+), 3 deletions(-) diff --git a/src/server/parse_archive.py b/src/server/parse_archive.py index 96efa8d..666813e 100644 --- a/src/server/parse_archive.py +++ b/src/server/parse_archive.py @@ -3,12 +3,19 @@ import subprocess import json from typing import List, Dict, Any import uuid +import urllib.parse +from PIL import Image, ImageDraw, ImageFont +import requests +from bs4 import BeautifulSoup +from io import BytesIO class ArchiveParser: def __init__(self): """Initialize the ArchiveParser with archive_dir from environment variable.""" self.archive_dir = os.environ.get("ARCHIVE_DIR", "/default/archive/path") + self.cache_dir = os.path.join(os.path.dirname(__file__), "..", "..", "cache") + os.makedirs(self.cache_dir, exist_ok=True) def get_posts(self, count: int, start_index: int = 0) -> List[Dict[str, Any]]: """ @@ -28,7 +35,7 @@ class ArchiveParser: try: # Use ls with tail and head commands for efficient pagination # Get all directories, skip the first start_index, then get count number of entries - cmd = f"ls -1 {self.archive_dir} | tail -{start_index + 1} | head -{count}" + cmd = f"ls -1 {self.archive_dir} | tail -{start_index + count + 1} | head -{count}" print("Running cmd: " + cmd) result = subprocess.run( cmd, shell=True, capture_output=True, text=True, check=True, timeout=30 @@ -129,9 +136,182 @@ class ArchiveParser: "/Reddit_Logo.webp" # Default to the logo in website directory ) + # Generate HTML preview for the post URL + post["image"] = self._generate_preview_image(post["url"], post["id"]) + return post except Exception as e: print(f"Error converting index to post: {e}") # Return a minimal post if there's an error raise e + + def _generate_preview_image(self, url: str, post_id: str) -> str: + """ + Generate a preview image for the HTML page at the given URL. + + This implementation extracts the largest image from the HTML page content, + falling back to a placeholder if no suitable images are found. + + Args: + url (str): The URL to generate preview for + post_id (str): The ID of the post + + Returns: + str: Path to the generated preview image + """ + try: + # Validate the URL + parsed_url = urllib.parse.urlparse(url) + if not parsed_url.scheme or not parsed_url.netloc: + return "/Reddit_Logo.webp" + + # Generate filename based on post ID + filename = f"{post_id}.webp" + image_path = os.path.join(self.cache_dir, filename) + + # If image already exists, return its path + if os.path.exists(image_path): + return f"/cache/{filename}" + + # Try to extract and use the largest image from the webpage content + image_extracted = self._extract_largest_image_from_webpage(url, image_path) + + if not image_extracted: + # Fall back to creating a placeholder image + self._create_placeholder_image(url, image_path) + + return f"/cache/{filename}" + + except Exception as e: + print(f"Error generating preview for URL {url}: {e}") + # Return default fallback image in case of any error + return "/Reddit_Logo.webp" + + def _create_placeholder_image(self, url: str, save_path: str): + """ + Create a simple placeholder image when real preview isn't possible. + + Args: + url (str): The URL that was attempted + save_path (str): Path to save the image + """ + try: + # Create a simple placeholder image + img = Image.new("RGB", (800, 600), color=(73, 109, 137)) + draw = ImageDraw.Draw(img) + + # Try to get default font or use None (which will fall back to default) + try: + font = ImageFont.load_default() + except Exception: + font = None + + # Draw some text on the image + draw.text( + (10, 10), "HTML Preview Placeholder", fill=(255, 255, 0), font=font + ) + draw.text( + (10, 40), + f"URL: {url[:50]}{'...' if len(url) > 50 else ''}", + fill=(255, 255, 255), + font=font, + ) + + # Add instructions + draw.text( + (10, 80), + "Largest image from webpage would appear here", + fill=(255, 255, 255), + font=font, + ) + draw.text( + (10, 100), + "This is a placeholder image", + fill=(255, 255, 255), + font=font, + ) + + # Save as webp image + img.save(save_path, "WEBP", quality=80) + except Exception as e: + print(f"Error creating placeholder image: {e}") + # If everything fails, we still have the default fallback + return + + def _extract_largest_image_from_webpage(self, url: str, save_path: str) -> bool: + """ + Extract the largest image from a webpage and save it. + + Args: + url (str): URL of the webpage to extract images from + save_path (str): Path where to save the extracted image + + Returns: + bool: True if successful, False otherwise + """ + try: + # Fetch the webpage content + headers = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36" + } + response = requests.get(url, timeout=10, headers=headers) + response.raise_for_status() + + # Parse HTML with BeautifulSoup + soup = BeautifulSoup(response.content, "html.parser") + + # Find all img tags + img_tags = soup.find_all("img") + + if not img_tags: + return False + + # Get the largest image by size (height * width) + largest_img = None + max_area = 0 + largest_img_src = None + + for img in img_tags: + src = img.get("src") or img.get("data-src") + if not src: + continue + + # Resolve relative URLs + if not src.startswith(("http://", "https://")): + from urllib.parse import urljoin + + src = urljoin(url, src) + + try: + # Get image dimensions by downloading it + img_response = requests.get(src, timeout=10) + img_response.raise_for_status() + + # Open image with PIL to get dimensions + img_data = Image.open(BytesIO(img_response.content)) + area = img_data.width * img_data.height + + if area > max_area: + max_area = area + largest_img_src = src + + except Exception as e: + print(f"Could not process image from {src}: {e}") + continue + + # If we found a suitable image, download and save it + if largest_img_src: + img_response = requests.get(largest_img_src, timeout=10) + img_response.raise_for_status() + + # Save as webp with quality 80 + img = Image.open(BytesIO(img_response.content)) + img.save(save_path, "WEBP", quality=80) + return True + + except Exception as e: + print(f"Error extracting image from webpage: {e}") + return False + + return False diff --git a/src/server/requirements.txt b/src/server/requirements.txt index 7210db7..d01f62c 100644 --- a/src/server/requirements.txt +++ b/src/server/requirements.txt @@ -1,2 +1,5 @@ -Flask==2.3.3 -Flask-CORS==4.0.0 +Flask +Flask-CORS +Pillow +beautifulsoup4 +requests