diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..5829868 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,11 @@ +.git +__pycache__ +*.py[cod] +.pytest_cache +.env +tests +test_client.py +integration_test.py +docs +*.md +docker-compose.yml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ade4eb9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +__pycache__/ +*.py[cod] +*.egg-info/ +.pytest_cache/ +.env +.venv/ +venv/ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..eba8c1d --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Jarian Cottingham + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..e018f2e --- /dev/null +++ b/README.md @@ -0,0 +1,134 @@ +# MCP Search Servers + +Model Context Protocol (MCP) servers that expose **Google** and **DuckDuckGo** search +as tools for LLM clients (Claude Desktop, Cursor, VS Code, ...), backed by a +self-hosted [SearXNG](https://github.com/searxng/searxng) metasearch instance. + +SearXNG aggregates results from Google, DuckDuckGo, Brave, Wikipedia, and more with +its own request handling, so a single SearXNG container serves both MCP servers — +no browser fingerprinting, no API keys, fully self-hosted. + +## Services + +| Service | Port | MCP tool | Description | +|-------------------|------|--------------------|--------------------------------------| +| `google-mcp` | 3001 | `google_search` | Google results via SearXNG | +| `duckduckgo-mcp` | 3002 | `duckduckgo_search`| DuckDuckGo results via SearXNG | +| `searxng` | — | — | Internal metasearch engine (not published) | + +Each server exposes: + +- `/sse` + `/messages/` — MCP SSE transport +- `/search?q=...&num=N` — plain HTTP JSON endpoint +- `/health` — liveness probe + +## Quick Start + +```bash +cp .env.example .env # fill in SEARXNG_SECRET +docker compose up -d --build +``` + +Generate a SearXNG secret: + +```bash +python3 -c "import secrets; print(secrets.token_hex(32))" +``` + +Verify: + +```bash +curl http://localhost:3001/health +curl "http://localhost:3001/search?q=python&num=3" +``` + +Then point your LLM client at `http://localhost:3001/sse` (Google) and/or +`http://localhost:3002/sse` (DuckDuckGo). Client configuration examples for +Claude Desktop, Cursor/Windsurf, and VS Code are in [USAGE.md](USAGE.md). + +## Tool Contract + +```json +{ + "name": "google_search", + "arguments": { "query": "Python programming language", "num_results": 5 } +} +``` + +Returns numbered results with title, URL, and snippet (max 20 results, +queries capped at 500 characters). + +## Configuration + +| Variable | Default | Description | +|----------------------|----------------------|------------------------------------------| +| `SEARXNG_SECRET` | — (required) | SearXNG session/CSRF secret | +| `RATE_LIMIT_SECONDS` | 5 / 3 | Minimum seconds between search requests | +| `MCP_PORT` | 3001 / 3002 | Port the MCP server listens on | +| `SEARXNG_URL` | `http://searxng:8080`| URL of the SearXNG instance | + +## Security Model + +- **Internal network** — SearXNG is on a `internal: true` Docker network; only the + two MCP containers can reach it. It is not published to the host. +- **Per-IP rate limiting** — HTTP endpoints allow 30 requests/minute per IP + (`/health` exempt); excess requests get `429`. +- **Request pacing** — each server also enforces a minimum interval between + upstream SearXNG queries to avoid throttling. +- **No auth on `/search` and `/sse`** — the servers assume they are bound to a + trusted network. Expose them publicly only behind an auth proxy. +- **Input validation** — query length capped, result counts clamped to 1..20, + malformed parameters return `400` instead of crashing. + +## Optional: Playwright Scraping Fallback + +`lib/google_search.py` and `lib/duckduckgo_search.py` contain a legacy +Playwright-based scraping path (single shared browser, stealth user agent, +HTML parsing with multiple fallback strategies). It is **not** used by the +production servers — Google aggressively blocks headless browsers and SearXNG +is the working path. If you want to run the scraper directly: + +```bash +pip install playwright +playwright install chromium +``` + +## Development + +```bash +python3 -m venv .venv && source .venv/bin/activate +pip install -e ".[dev]" + +ruff check . +pytest tests/ -v + +# End-to-end test (requires running containers) +python3 integration_test.py +``` + +## Project Structure + +``` +├── docker-compose.yml # All 3 services +├── searxng-settings.yml # SearXNG engine config +├── google-mcp/ +│ ├── Dockerfile +│ └── server.py # Google MCP server (SearXNG-backed) +├── duckduckgo-mcp/ +│ ├── Dockerfile +│ └── server.py # DuckDuckGo MCP server (SearXNG-backed) +├── lib/ +│ ├── playwright_manager.py # Single-instance browser manager (optional path) +│ ├── rate_limiter.py # Async minimum-interval rate limiter +│ ├── google_search.py # Playwright Google scraper (optional path) +│ └── duckduckgo_search.py # Playwright DuckDuckGo scraper (optional path) +├── tests/ +│ ├── test_rate_limiter.py +│ └── test_search_parsing.py +├── integration_test.py # End-to-end MCP SSE test +└── test_client.py # Manual stdio/SSE smoke client +``` + +## License + +[MIT](LICENSE) diff --git a/USAGE.md b/USAGE.md index e02b826..061a076 100644 --- a/USAGE.md +++ b/USAGE.md @@ -5,7 +5,7 @@ MCP (Model Context Protocol) servers that expose Google and DuckDuckGo search as ## Quick Start ```bash -cd google-mcp +# From the repository root docker compose up -d --build ``` @@ -169,9 +169,11 @@ python3 integration_test.py ## Troubleshooting -**No results returned** — SearXNG may be blocked on your network. Check which engines are working: +**No results returned** — SearXNG may be blocked on your network. Check which engines are working +(SearXNG is on an internal Docker network, so query it from inside its container): ```bash -curl "http://localhost:8080/search?q=test&format=json" | python3 -m json.tool +docker compose exec searxng python -c \ + "import urllib.request; print(urllib.request.urlopen('http://localhost:8080/search?q=test&format=json').read()[:500])" ``` **Services not starting** — Check for port conflicts: diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000..5569969 --- /dev/null +++ b/conftest.py @@ -0,0 +1,2 @@ +# Ensures the repo root is on sys.path so `lib` and the server packages +# are importable in tests without installation. diff --git a/docker-compose.yml b/docker-compose.yml index f371a0f..362fdc8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,7 +16,7 @@ services: networks: - mcp-internal healthcheck: - test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/health')"] + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/healthz')"] interval: 30s timeout: 10s retries: 3 diff --git a/duckduckgo-mcp/server.py b/duckduckgo-mcp/server.py index d6a8be7..3628873 100644 --- a/duckduckgo-mcp/server.py +++ b/duckduckgo-mcp/server.py @@ -1,5 +1,4 @@ import asyncio -import json import logging import os import signal @@ -9,15 +8,15 @@ from contextlib import asynccontextmanager from datetime import datetime, timezone import httpx - from mcp.server import Server from mcp.server.sse import SseServerTransport +from mcp.types import TextContent, Tool from starlette.applications import Starlette +from starlette.middleware import Middleware from starlette.middleware.base import BaseHTTPMiddleware from starlette.requests import Request from starlette.responses import JSONResponse, Response from starlette.routing import Mount, Route -from mcp.types import Tool, TextContent logging.basicConfig( level=logging.INFO, @@ -31,6 +30,19 @@ PORT = int(os.environ.get("MCP_PORT", "3002")) SEARXNG_URL = os.environ.get("SEARXNG_URL", "http://searxng:8080") RATE_LIMIT_SECONDS = float(os.environ.get("RATE_LIMIT_SECONDS", "3")) +DEFAULT_RESULTS = 10 +MAX_RESULTS = 20 +MAX_QUERY_LEN = 500 + + +def parse_num_results(value) -> int: + """Parse a result-count value, clamped to 1..MAX_RESULTS.""" + try: + num = int(value) + except (TypeError, ValueError): + return DEFAULT_RESULTS + return max(1, min(num, MAX_RESULTS)) + # --- HTTPX singleton (#8) --- @@ -105,10 +117,17 @@ class RateLimitMiddleware(BaseHTTPMiddleware): client_ip = request.client.host if request.client else "unknown" now = datetime.now(timezone.utc).timestamp() + cutoff = now - self.window_seconds async with self._lock: + if len(self._requests) > 1000: + for ip in [ + ip + for ip, ts in self._requests.items() + if not ts or ts[-1] <= cutoff + ]: + del self._requests[ip] timestamps = self._requests[client_ip] - cutoff = now - self.window_seconds self._requests[client_ip] = [t for t in timestamps if t > cutoff] if len(self._requests[client_ip]) >= self.max_requests: return JSONResponse( @@ -155,15 +174,20 @@ async def do_search(query: str, num_results: int, engine: str): # --- HTTP endpoint --- async def search_http(request: Request): - query = request.query_params.get("q", "") + query = request.query_params.get("q", "").strip() if not query: return JSONResponse({"error": "'q' parameter required"}, status_code=400) + if len(query) > MAX_QUERY_LEN: + return JSONResponse( + {"error": f"'q' must be {MAX_QUERY_LEN} characters or fewer"}, + status_code=400, + ) - num = min(int(request.query_params.get("num", "10")), 20) + num = parse_num_results(request.query_params.get("num")) try: results = await do_search(query, num, "duckduckgo") return JSONResponse({"query": query, "results": results}) - except Exception as e: + except Exception: logger.exception("Search failed") return JSONResponse({"error": "Search service unavailable"}, status_code=503) @@ -204,14 +228,21 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]: raise ValueError(f"Unknown tool: {name}") query = arguments.get("query", "") - if not query: + if not isinstance(query, str) or not query: return [TextContent(type="text", text="Error: 'query' is required.")] + if len(query) > MAX_QUERY_LEN: + return [ + TextContent( + type="text", + text=f"Error: query must be {MAX_QUERY_LEN} characters or fewer.", + ) + ] - num_results = min(int(arguments.get("num_results", 10)), 20) + num_results = parse_num_results(arguments.get("num_results")) try: results = await do_search(query, num_results, "duckduckgo") - except Exception as e: + except Exception: logger.exception("Search failed") return [TextContent(type="text", text="Search failed: service unavailable")] @@ -248,13 +279,14 @@ async def handle_sse(request): starlette_app = Starlette( debug=False, + lifespan=lifespan, routes=[ Route("/health", endpoint=health_check), Route("/search", endpoint=search_http), Route("/sse", endpoint=handle_sse), Mount("/messages/", app=sse.handle_post_message), ], - middleware=[RateLimitMiddleware], + middleware=[Middleware(RateLimitMiddleware)], ) diff --git a/google-mcp/server.py b/google-mcp/server.py index d031876..e39e33e 100644 --- a/google-mcp/server.py +++ b/google-mcp/server.py @@ -1,5 +1,4 @@ import asyncio -import json import logging import os import signal @@ -9,15 +8,15 @@ from contextlib import asynccontextmanager from datetime import datetime, timezone import httpx - from mcp.server import Server from mcp.server.sse import SseServerTransport +from mcp.types import TextContent, Tool from starlette.applications import Starlette +from starlette.middleware import Middleware from starlette.middleware.base import BaseHTTPMiddleware from starlette.requests import Request from starlette.responses import JSONResponse, Response from starlette.routing import Mount, Route -from mcp.types import Tool, TextContent logging.basicConfig( level=logging.INFO, @@ -31,6 +30,19 @@ PORT = int(os.environ.get("MCP_PORT", "3001")) SEARXNG_URL = os.environ.get("SEARXNG_URL", "http://searxng:8080") RATE_LIMIT_SECONDS = float(os.environ.get("RATE_LIMIT_SECONDS", "5")) +DEFAULT_RESULTS = 10 +MAX_RESULTS = 20 +MAX_QUERY_LEN = 500 + + +def parse_num_results(value) -> int: + """Parse a result-count value, clamped to 1..MAX_RESULTS.""" + try: + num = int(value) + except (TypeError, ValueError): + return DEFAULT_RESULTS + return max(1, min(num, MAX_RESULTS)) + # --- HTTPX singleton (#8) --- @@ -105,10 +117,17 @@ class RateLimitMiddleware(BaseHTTPMiddleware): client_ip = request.client.host if request.client else "unknown" now = datetime.now(timezone.utc).timestamp() + cutoff = now - self.window_seconds async with self._lock: + if len(self._requests) > 1000: + for ip in [ + ip + for ip, ts in self._requests.items() + if not ts or ts[-1] <= cutoff + ]: + del self._requests[ip] timestamps = self._requests[client_ip] - cutoff = now - self.window_seconds self._requests[client_ip] = [t for t in timestamps if t > cutoff] if len(self._requests[client_ip]) >= self.max_requests: return JSONResponse( @@ -155,15 +174,20 @@ async def do_search(query: str, num_results: int, engine: str): # --- HTTP endpoint --- async def search_http(request: Request): - query = request.query_params.get("q", "") + query = request.query_params.get("q", "").strip() if not query: return JSONResponse({"error": "'q' parameter required"}, status_code=400) + if len(query) > MAX_QUERY_LEN: + return JSONResponse( + {"error": f"'q' must be {MAX_QUERY_LEN} characters or fewer"}, + status_code=400, + ) - num = min(int(request.query_params.get("num", "10")), 20) + num = parse_num_results(request.query_params.get("num")) try: results = await do_search(query, num, "google") return JSONResponse({"query": query, "results": results}) - except Exception as e: + except Exception: logger.exception("Search failed") return JSONResponse({"error": "Search service unavailable"}, status_code=503) @@ -204,14 +228,21 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]: raise ValueError(f"Unknown tool: {name}") query = arguments.get("query", "") - if not query: + if not isinstance(query, str) or not query: return [TextContent(type="text", text="Error: 'query' is required.")] + if len(query) > MAX_QUERY_LEN: + return [ + TextContent( + type="text", + text=f"Error: query must be {MAX_QUERY_LEN} characters or fewer.", + ) + ] - num_results = min(int(arguments.get("num_results", 10)), 20) + num_results = parse_num_results(arguments.get("num_results")) try: results = await do_search(query, num_results, "google") - except Exception as e: + except Exception: logger.exception("Search failed") return [TextContent(type="text", text="Search failed: service unavailable")] @@ -248,13 +279,14 @@ async def handle_sse(request): starlette_app = Starlette( debug=False, + lifespan=lifespan, routes=[ Route("/health", endpoint=health_check), Route("/search", endpoint=search_http), Route("/sse", endpoint=handle_sse), Mount("/messages/", app=sse.handle_post_message), ], - middleware=[RateLimitMiddleware], + middleware=[Middleware(RateLimitMiddleware)], ) diff --git a/integration_test.py b/integration_test.py index e40557b..f5ca78b 100644 --- a/integration_test.py +++ b/integration_test.py @@ -3,6 +3,7 @@ import asyncio import json import sys + import aiohttp @@ -76,7 +77,7 @@ async def test_server(name, host, port, tool_name, query): "jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {"protocolVersion": "2024-11-05", "capabilities": {}, "clientInfo": {"name": "test", "version": "1.0"}}, - }) as resp: + }): pass init = await collector.wait_for_id(1, timeout=10) @@ -88,25 +89,31 @@ async def test_server(name, host, port, tool_name, query): # Initialized notification async with session.post(messages_endpoint, json={ "jsonrpc": "2.0", "method": "notifications/initialized", - }) as resp: + }): pass # List tools async with session.post(messages_endpoint, json={ "jsonrpc": "2.0", "id": 2, "method": "tools/list", "params": {}, - }) as resp: + }): pass tools = await collector.wait_for_id(2, timeout=10) if tools: - print(f" Tools: {[t['name'] for t in tools.get('result', {}).get('tools', [])]}") + tool_names = [ + t["name"] for t in tools.get("result", {}).get("tools", []) + ] + print(f" Tools: {tool_names}") # Call search print(f" Searching: '{query}'") async with session.post(messages_endpoint, json={ "jsonrpc": "2.0", "id": 3, "method": "tools/call", - "params": {"name": tool_name, "arguments": {"query": query, "num_results": 5}}, - }) as resp: + "params": { + "name": tool_name, + "arguments": {"query": query, "num_results": 5}, + }, + }): pass result = await collector.wait_for_id(3, timeout=30) @@ -145,11 +152,14 @@ async def main(): print("Waiting for services to start...") await asyncio.sleep(5) - results["google"] = await test_server("Google", host, 3001, "google_search", "Python programming language") + query = "Python programming language" + results["google"] = await test_server("Google", host, 3001, "google_search", query) await asyncio.sleep(2) - results["duckduckgo"] = await test_server("DuckDuckGo", host, 3002, "duckduckgo_search", "Python programming language") + results["duckduckgo"] = await test_server( + "DuckDuckGo", host, 3002, "duckduckgo_search", query + ) print(f"\n{'='*60}") print("SUMMARY") @@ -161,4 +171,4 @@ async def main(): if __name__ == "__main__": - asyncio.run(main()) \ No newline at end of file + asyncio.run(main()) diff --git a/lib/__init__.py b/lib/__init__.py index f962e0f..d4d3986 100644 --- a/lib/__init__.py +++ b/lib/__init__.py @@ -1,11 +1,25 @@ -from lib.playwright_manager import PlaywrightManager -from lib.rate_limiter import RateLimiter -from lib.google_search import GoogleSearch -from lib.duckduckgo_search import DuckDuckGoSearch +"""Shared search utilities. -__all__ = [ - "PlaywrightManager", - "RateLimiter", - "GoogleSearch", - "DuckDuckGoSearch", -] \ No newline at end of file +Playwright-based scrapers are imported lazily so the package (and its +rate limiter) works without the optional ``playwright`` dependency. +""" + +from lib.rate_limiter import RateLimiter + +__all__ = ["PlaywrightManager", "RateLimiter", "GoogleSearch", "DuckDuckGoSearch"] + + +def __getattr__(name): + if name == "PlaywrightManager": + from lib.playwright_manager import PlaywrightManager + + return PlaywrightManager + if name == "GoogleSearch": + from lib.google_search import GoogleSearch + + return GoogleSearch + if name == "DuckDuckGoSearch": + from lib.duckduckgo_search import DuckDuckGoSearch + + return DuckDuckGoSearch + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/lib/__pycache__/__init__.cpython-313.pyc b/lib/__pycache__/__init__.cpython-313.pyc deleted file mode 100644 index 0611d3a..0000000 Binary files a/lib/__pycache__/__init__.cpython-313.pyc and /dev/null differ diff --git a/lib/__pycache__/duckduckgo_search.cpython-313.pyc b/lib/__pycache__/duckduckgo_search.cpython-313.pyc deleted file mode 100644 index 0838972..0000000 Binary files a/lib/__pycache__/duckduckgo_search.cpython-313.pyc and /dev/null differ diff --git a/lib/__pycache__/google_search.cpython-313.pyc b/lib/__pycache__/google_search.cpython-313.pyc deleted file mode 100644 index 47f4121..0000000 Binary files a/lib/__pycache__/google_search.cpython-313.pyc and /dev/null differ diff --git a/lib/__pycache__/playwright_manager.cpython-313.pyc b/lib/__pycache__/playwright_manager.cpython-313.pyc deleted file mode 100644 index 3f46b9c..0000000 Binary files a/lib/__pycache__/playwright_manager.cpython-313.pyc and /dev/null differ diff --git a/lib/__pycache__/rate_limiter.cpython-313.pyc b/lib/__pycache__/rate_limiter.cpython-313.pyc deleted file mode 100644 index 979b813..0000000 Binary files a/lib/__pycache__/rate_limiter.cpython-313.pyc and /dev/null differ diff --git a/lib/duckduckgo_search.py b/lib/duckduckgo_search.py index e49b611..83d7eb9 100644 --- a/lib/duckduckgo_search.py +++ b/lib/duckduckgo_search.py @@ -1,6 +1,6 @@ import asyncio import logging -from typing import List, Dict, Any, Optional +from typing import Any, Dict, List from lib.playwright_manager import PlaywrightManager from lib.rate_limiter import RateLimiter @@ -76,4 +76,4 @@ class DuckDuckGoSearch: return results; }""" ) - return results[:max_results] \ No newline at end of file + return results[:max_results] diff --git a/lib/google_search.py b/lib/google_search.py index 803e821..82c9354 100644 --- a/lib/google_search.py +++ b/lib/google_search.py @@ -1,7 +1,8 @@ import asyncio import logging import re -from typing import List, Dict, Any +import urllib.parse +from typing import Any, Dict, List from lib.playwright_manager import PlaywrightManager from lib.rate_limiter import RateLimiter @@ -29,7 +30,10 @@ class GoogleSearch: try: page = await PlaywrightManager.get_page(stealth=True) - url_params = f"{GOOGLE_URL}?q={query}&num={min(num_results, 20)}&hl={language}" + url_params = ( + f"{GOOGLE_URL}?q={urllib.parse.quote(query)}" + f"&num={min(num_results, 20)}&hl={language}" + ) logger.info(f"Navigating to Google search: {query}") await page.goto(url_params, wait_until="domcontentloaded", timeout=30000) @@ -98,8 +102,6 @@ class GoogleSearch: def _clean_url(self, raw_url: str) -> str: """Decode Google redirect URL.""" - import urllib.parse - raw_url = urllib.parse.unquote(raw_url) # Remove tracking parameters url = re.sub(r"&[a-z_]+=.*$", "", raw_url) @@ -118,7 +120,13 @@ class GoogleSearch: for i, result in enumerate(results): for block in snippet_blocks: clean = re.sub(r'<[^>]+>', "", block).strip() - if 50 < len(clean) < 300 and result["url"] in html[max(0, html.find(clean) - 500):html.find(clean) + 500]: + if not 50 < len(clean) < 300: + continue + pos = html.find(clean) + if pos == -1: + continue + window = html[max(0, pos - 500): pos + 500] + if result["url"] in window: result["snippet"] = clean[:200] break return results @@ -152,4 +160,4 @@ class GoogleSearch: elif title: results.append({"title": title, "url": url, "snippet": ""}) - return results \ No newline at end of file + return results diff --git a/lib/playwright_manager.py b/lib/playwright_manager.py index b5d168f..16b4024 100644 --- a/lib/playwright_manager.py +++ b/lib/playwright_manager.py @@ -2,7 +2,7 @@ import asyncio import logging from typing import Optional -from playwright.async_api import async_playwright, Browser, BrowserContext, Page +from playwright.async_api import Browser, BrowserContext, Page, async_playwright logger = logging.getLogger(__name__) @@ -75,7 +75,7 @@ class PlaywrightManager: stealth = Stealth() await stealth.apply_stealth_async(page) logger.debug("Stealth mode applied to page") - except (ImportError, Exception) as e: + except Exception as e: logger.warning(f"Could not apply stealth: {e}") return page @@ -111,4 +111,4 @@ class PlaywrightManager: @classmethod def is_initialized(cls) -> bool: - return cls._initialized \ No newline at end of file + return cls._initialized diff --git a/lib/rate_limiter.py b/lib/rate_limiter.py index 6a48624..fa9a80f 100644 --- a/lib/rate_limiter.py +++ b/lib/rate_limiter.py @@ -29,4 +29,4 @@ class RateLimiter: f"(min interval: {self.min_interval}s)" ) await asyncio.sleep(wait_time) - self._last_request_time = time.monotonic() \ No newline at end of file + self._last_request_time = time.monotonic() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..eacbd01 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,37 @@ +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[project] +name = "mcp-search-servers" +version = "1.1.0" +description = "MCP servers exposing Google and DuckDuckGo search via SearXNG" +readme = "README.md" +license = { text = "MIT" } +requires-python = ">=3.10" +dependencies = [ + "mcp>=1.0.0,<2.0.0", + "httpx>=0.27.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=8.0", + "ruff>=0.4", +] +playwright = [ + "playwright>=1.44", +] + +[tool.setuptools] +packages = ["lib"] + +[tool.ruff] +line-length = 100 +target-version = "py310" + +[tool.ruff.lint] +select = ["E", "F", "W", "I"] + +[tool.pytest.ini_options] +testpaths = ["tests"] diff --git a/test_client.py b/test_client.py index 6858158..84da46c 100644 --- a/test_client.py +++ b/test_client.py @@ -7,10 +7,8 @@ Usage: """ import asyncio -import json import subprocess import sys -import time async def test_mcp_server(service_name: str): @@ -36,7 +34,13 @@ async def test(): await asyncio.sleep(0.5) init_req = json.loads(await read.__anext__()) # Send initialize response - init_result = {{'jsonrpc': '2.0', 'id': init_req['id'], 'result': {{'protocolVersion': '2024-11-05', 'capabilities': {{}}, 'serverInfo': {{'name': 'test', 'version': '1.0'}}}}}} + init_result = {{ + 'jsonrpc': '2.0', 'id': init_req['id'], + 'result': {{ + 'protocolVersion': '2024-11-05', 'capabilities': {{}}, + 'serverInfo': {{'name': 'test', 'version': '1.0'}}, + }}, + }} await write.send(json.dumps(init_result)) # Client sends initialized notification await read.__anext__() @@ -68,7 +72,6 @@ async def test(): async def test_with_curl(service_name: str): """Test using docker exec to run a Python test directly in the container.""" container_name = f"{service_name}-mcp" - module = service_name.replace("-", "_") test_script = f""" import sys, asyncio, os @@ -170,4 +173,4 @@ async def main(): if __name__ == "__main__": - asyncio.run(main()) \ No newline at end of file + asyncio.run(main()) diff --git a/tests/__pycache__/__init__.cpython-313.pyc b/tests/__pycache__/__init__.cpython-313.pyc deleted file mode 100644 index db12ce6..0000000 Binary files a/tests/__pycache__/__init__.cpython-313.pyc and /dev/null differ diff --git a/tests/__pycache__/integration_test.cpython-313-pytest-9.0.3.pyc b/tests/__pycache__/integration_test.cpython-313-pytest-9.0.3.pyc deleted file mode 100644 index 8402883..0000000 Binary files a/tests/__pycache__/integration_test.cpython-313-pytest-9.0.3.pyc and /dev/null differ diff --git a/tests/__pycache__/test_integration.cpython-313-pytest-9.0.3.pyc b/tests/__pycache__/test_integration.cpython-313-pytest-9.0.3.pyc deleted file mode 100644 index 49a02af..0000000 Binary files a/tests/__pycache__/test_integration.cpython-313-pytest-9.0.3.pyc and /dev/null differ diff --git a/tests/__pycache__/test_rate_limiter.cpython-313-pytest-9.0.3.pyc b/tests/__pycache__/test_rate_limiter.cpython-313-pytest-9.0.3.pyc deleted file mode 100644 index 404f22f..0000000 Binary files a/tests/__pycache__/test_rate_limiter.cpython-313-pytest-9.0.3.pyc and /dev/null differ diff --git a/tests/__pycache__/test_search_parsing.cpython-313-pytest-9.0.3.pyc b/tests/__pycache__/test_search_parsing.cpython-313-pytest-9.0.3.pyc deleted file mode 100644 index d59e56f..0000000 Binary files a/tests/__pycache__/test_search_parsing.cpython-313-pytest-9.0.3.pyc and /dev/null differ diff --git a/tests/test_rate_limiter.py b/tests/test_rate_limiter.py index cbdd6bf..cf8d625 100644 --- a/tests/test_rate_limiter.py +++ b/tests/test_rate_limiter.py @@ -1,19 +1,9 @@ import asyncio import time -import pytest -import pytest_asyncio - from lib.rate_limiter import RateLimiter -@pytest.fixture -def event_loop(): - loop = asyncio.new_event_loop() - yield loop - loop.close() - - class TestRateLimiter: """Test rate limiter enforces minimum delays between requests.""" @@ -23,9 +13,9 @@ class TestRateLimiter: start = time.monotonic() await limiter.acquire() elapsed = time.monotonic() - start - assert elapsed < 0.1 + return elapsed - asyncio.get_event_loop().run_until_complete(run()) + assert asyncio.run(run()) < 0.1 def test_enforces_minimum_interval(self): async def run(): @@ -34,27 +24,26 @@ class TestRateLimiter: start = time.monotonic() await limiter.acquire() - elapsed = time.monotonic() - start + return time.monotonic() - start - assert elapsed >= 0.25 - - asyncio.get_event_loop().run_until_complete(run()) + assert asyncio.run(run()) >= 0.25 def test_consecutive_requests_space_correctly(self): + interval = 0.2 + async def run(): - interval = 0.2 limiter = RateLimiter(min_interval_seconds=interval) times = [] for _ in range(5): await limiter.acquire() times.append(time.monotonic()) + return times - for i in range(1, len(times)): - gap = times[i] - times[i - 1] - assert gap >= interval * 0.8 - - asyncio.get_event_loop().run_until_complete(run()) + times = asyncio.run(run()) + for i in range(1, len(times)): + gap = times[i] - times[i - 1] + assert gap >= interval * 0.8 def test_custom_interval(self): async def run(): @@ -63,11 +52,9 @@ class TestRateLimiter: start = time.monotonic() await limiter.acquire() - elapsed = time.monotonic() - start + return time.monotonic() - start - assert elapsed >= 0.05 - - asyncio.get_event_loop().run_until_complete(run()) + assert asyncio.run(run()) >= 0.05 def test_no_delay_after_long_pause(self): async def run(): @@ -77,8 +64,6 @@ class TestRateLimiter: start = time.monotonic() await limiter.acquire() - elapsed = time.monotonic() - start + return time.monotonic() - start - assert elapsed < 0.1 - - asyncio.get_event_loop().run_until_complete(run()) \ No newline at end of file + assert asyncio.run(run()) < 0.1 diff --git a/tests/test_search_parsing.py b/tests/test_search_parsing.py index ddbdb52..f2ff01f 100644 --- a/tests/test_search_parsing.py +++ b/tests/test_search_parsing.py @@ -1,4 +1,3 @@ -import pytest class TestSearchResultFormatting: @@ -70,4 +69,4 @@ class TestSearchResultFormatting: if snippet: lines.append(f" {snippet}") - assert len(lines[2]) <= 203 \ No newline at end of file + assert len(lines[2]) <= 203