import asyncio import logging import re import urllib.parse from typing import Any, Dict, List from lib.playwright_manager import PlaywrightManager from lib.rate_limiter import RateLimiter logger = logging.getLogger(__name__) GOOGLE_URL = "https://www.google.com/search" class GoogleSearch: """ Search Google using Playwright headless browser with stealth mode. Respects rate limits and cleans up page resources after each search. """ def __init__(self, min_interval: float = 5.0): self.rate_limiter = RateLimiter(min_interval_seconds=min_interval) async def search( self, query: str, num_results: int = 10, language: str = "en" ) -> List[Dict[str, Any]]: await self.rate_limiter.acquire() page = None try: page = await PlaywrightManager.get_page(stealth=True) url_params = ( f"{GOOGLE_URL}?q={urllib.parse.quote(query)}" f"&num={min(num_results, 20)}&hl={language}" ) logger.info(f"Navigating to Google search: {query}") await page.goto(url_params, wait_until="domcontentloaded", timeout=30000) await asyncio.sleep(3) html = await page.content() results = self._parse_results(html, num_results) logger.info(f"Google search returned {len(results)} results for: {query}") return results except Exception as e: logger.error(f"Google search failed for '{query}': {e}") raise RuntimeError(f"Google search failed: {e}") from e finally: if page: await PlaywrightManager.close_page(page) def _parse_results( self, html: str, max_results: int ) -> List[Dict[str, Any]]: results = [] seen_urls = set() # Strategy 1: Parse /url?q= links (Google's redirect URLs) url_pattern = re.findall( r']*href="(/url\?q=([^&"]+)&[^"]*)"[^>]*>(.*?)', html, re.DOTALL, ) for _, raw_url, anchor_html in url_pattern: if len(results) >= max_results: break url = self._clean_url(raw_url) if not url or url in seen_urls or "google.com" in url: continue seen_urls.add(url) title = re.sub(r'<[^>]+>', "", anchor_html).strip() if not title or len(title) < 3: continue results.append({"title": title, "url": url, "snippet": ""}) # Strategy 2: If we got results, try to pair them with nearby snippets if results: results = self._add_snippets(html, results) else: # Strategy 3: Parse data-href attributes (modern Google) data_hrefs = re.findall(r'data-href="(.*?)"', html) for raw_url in data_hrefs: if len(results) >= max_results: break url = self._clean_url(raw_url) if not url or url in seen_urls or "google.com" in url: continue seen_urls.add(url) results.append({"title": url, "url": url, "snippet": ""}) # Strategy 4: Parse from