diff --git a/.gitignore b/.gitignore index 3287a5a..4c7dde4 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,6 @@ lib/ __pycache__ pyvenv.cfg + +/share +/etc diff --git a/src/server/parse_archive.py b/src/server/parse_archive.py index 666813e..df85a87 100644 --- a/src/server/parse_archive.py +++ b/src/server/parse_archive.py @@ -6,7 +6,11 @@ import uuid import urllib.parse from PIL import Image, ImageDraw, ImageFont import requests -from bs4 import BeautifulSoup + +try: + import mercury +except ImportError: + mercury = None from io import BytesIO @@ -241,7 +245,7 @@ class ArchiveParser: def _extract_largest_image_from_webpage(self, url: str, save_path: str) -> bool: """ - Extract the largest image from a webpage and save it. + Extract the largest image from a webpage using Mercury parser. Args: url (str): URL of the webpage to extract images from @@ -251,58 +255,18 @@ class ArchiveParser: bool: True if successful, False otherwise """ try: - # Fetch the webpage content - headers = { - "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36" - } - response = requests.get(url, timeout=10, headers=headers) - response.raise_for_status() - - # Parse HTML with BeautifulSoup - soup = BeautifulSoup(response.content, "html.parser") - - # Find all img tags - img_tags = soup.find_all("img") - - if not img_tags: + if not mercury: return False - # Get the largest image by size (height * width) - largest_img = None - max_area = 0 - largest_img_src = None + # Use mercury to parse the webpage + parsed = mercury.parse(url) - for img in img_tags: - src = img.get("src") or img.get("data-src") - if not src: - continue + # Check if mercury found a lead image + if hasattr(parsed, "lead_image_url") and parsed.lead_image_url: + image_url = parsed.lead_image_url - # Resolve relative URLs - if not src.startswith(("http://", "https://")): - from urllib.parse import urljoin - - src = urljoin(url, src) - - try: - # Get image dimensions by downloading it - img_response = requests.get(src, timeout=10) - img_response.raise_for_status() - - # Open image with PIL to get dimensions - img_data = Image.open(BytesIO(img_response.content)) - area = img_data.width * img_data.height - - if area > max_area: - max_area = area - largest_img_src = src - - except Exception as e: - print(f"Could not process image from {src}: {e}") - continue - - # If we found a suitable image, download and save it - if largest_img_src: - img_response = requests.get(largest_img_src, timeout=10) + # Download and save the image + img_response = requests.get(image_url, timeout=10) img_response.raise_for_status() # Save as webp with quality 80 @@ -311,7 +275,7 @@ class ArchiveParser: return True except Exception as e: - print(f"Error extracting image from webpage: {e}") + print(f"Error extracting image using Mercury: {e}") return False return False diff --git a/src/server/requirements.txt b/src/server/requirements.txt index d01f62c..87aed72 100644 --- a/src/server/requirements.txt +++ b/src/server/requirements.txt @@ -3,3 +3,5 @@ Flask-CORS Pillow beautifulsoup4 requests +mercury +mercury==1.0.0