Preview images with mercury

.gitignore

@@ -6,3 +6,6 @@ lib/
 
 __pycache__
 pyvenv.cfg
+
+/share
+/etc

src/server/parse_archive.py

@@ -6,7 +6,11 @@ import uuid
 import urllib.parse
 from PIL import Image, ImageDraw, ImageFont
 import requests
-from bs4 import BeautifulSoup
+
+try:
+    import mercury
+except ImportError:
+    mercury = None
 from io import BytesIO
 
 
@@ -241,7 +245,7 @@ class ArchiveParser:
 
     def _extract_largest_image_from_webpage(self, url: str, save_path: str) -> bool:
         """
-        Extract the largest image from a webpage and save it.
+        Extract the largest image from a webpage using Mercury parser.
 
         Args:
             url (str): URL of the webpage to extract images from
@@ -251,58 +255,18 @@ class ArchiveParser:
             bool: True if successful, False otherwise
         """
         try:
-            # Fetch the webpage content
+            if not mercury:
-            headers = {
-                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36"
-            }
-            response = requests.get(url, timeout=10, headers=headers)
-            response.raise_for_status()
-
-            # Parse HTML with BeautifulSoup
-            soup = BeautifulSoup(response.content, "html.parser")
-
-            # Find all img tags
-            img_tags = soup.find_all("img")
-
-            if not img_tags:
                 return False
 
-            # Get the largest image by size (height * width)
+            # Use mercury to parse the webpage
-            largest_img = None
+            parsed = mercury.parse(url)
-            max_area = 0
-            largest_img_src = None
 
-            for img in img_tags:
+            # Check if mercury found a lead image
-                src = img.get("src") or img.get("data-src")
+            if hasattr(parsed, "lead_image_url") and parsed.lead_image_url:
-                if not src:
+                image_url = parsed.lead_image_url
-                    continue
 
-                # Resolve relative URLs
+                # Download and save the image
-                if not src.startswith(("http://", "https://")):
+                img_response = requests.get(image_url, timeout=10)
-                    from urllib.parse import urljoin
-
-                    src = urljoin(url, src)
-
-                try:
-                    # Get image dimensions by downloading it
-                    img_response = requests.get(src, timeout=10)
-                    img_response.raise_for_status()
-
-                    # Open image with PIL to get dimensions
-                    img_data = Image.open(BytesIO(img_response.content))
-                    area = img_data.width * img_data.height
-
-                    if area > max_area:
-                        max_area = area
-                        largest_img_src = src
-
-                except Exception as e:
-                    print(f"Could not process image from {src}: {e}")
-                    continue
-
-            # If we found a suitable image, download and save it
-            if largest_img_src:
-                img_response = requests.get(largest_img_src, timeout=10)
                 img_response.raise_for_status()
 
                 # Save as webp with quality 80
@@ -311,7 +275,7 @@ class ArchiveParser:
                 return True
 
         except Exception as e:
-            print(f"Error extracting image from webpage: {e}")
+            print(f"Error extracting image using Mercury: {e}")
             return False
 
         return False

src/server/requirements.txt

@@ -3,3 +3,5 @@ Flask-CORS
 Pillow
 beautifulsoup4
 requests
+mercury
+mercury==1.0.0