diff --git a/MCPServer/README.md b/MCPServer/README.md index 33fbc9d..329d137 100644 --- a/MCPServer/README.md +++ b/MCPServer/README.md @@ -53,6 +53,7 @@ The server supports configuration through environment variables: - `FLASK_ENV` - Set to 'development' or 'production' (default: 'development') - `DATABASE_URL` - URL for database connection (e.g., PostgreSQL) - `API_KEY` - API key for external services +- `AI_SERVICE_API_KEY` - API key for authenticating with the centralized AI service at http://example.com:4000 - `LOG_LEVEL` - Logging level (DEBUG, INFO, WARNING, ERROR) ## Installation diff --git a/MCPServer/server.py b/MCPServer/server.py index 4229a49..9c186a2 100644 --- a/MCPServer/server.py +++ b/MCPServer/server.py @@ -106,12 +106,22 @@ def get_embedding(text): """ try: # Using the OpenAI-compatible endpoint for embeddings + headers = { + "Content-Type": "application/json" + } + + # Add API key if available + api_key = os.getenv("AI_SERVICE_API_KEY") + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + response = requests.post( "http://example.com:4000/v1/embeddings", json={ "input": text, "model": "text-embedding-3-small" # or whatever model you're using }, + headers=headers, timeout=30 ) response.raise_for_status() diff --git a/agent.md b/agent.md index a695a94..20ee3f6 100644 --- a/agent.md +++ b/agent.md @@ -265,6 +265,12 @@ scraper/ ### Environment Variables Each project may require specific environment variables. Check individual `README.md` files for details. +### Authentication +The system now supports API key authentication for connections to the centralized AI service at `http://example.com:4000`. To enable authentication: + +1. Set the `AI_SERVICE_API_KEY` environment variable with your API key +2. All connections to the AI service will automatically include the `Authorization: Bearer {api_key}` header + ### Data Storage - Articles are stored in the scraper component - Processed data flows through the ai_processor diff --git a/ai_processor/ai_processor.py b/ai_processor/ai_processor.py index 622cb6f..435b9bb 100644 --- a/ai_processor/ai_processor.py +++ b/ai_processor/ai_processor.py @@ -12,6 +12,9 @@ AI_SERVER_HOST = os.getenv("AI_SERVER_HOST", "example.com") AI_SERVER_PORT = os.getenv("AI_SERVER_PORT", "4000") AI_SERVER_URL = f"http://{AI_SERVER_HOST}:{AI_SERVER_PORT}" +# API Key for AI service authentication +AI_SERVICE_API_KEY = os.getenv("AI_SERVICE_API_KEY") + def process_article_content(article_content, filename, source): """ Process article content and extract key facts using the centralized AI service @@ -21,6 +24,14 @@ def process_article_content(article_content, filename, source): # Use the gpt-oss model for fact extraction as specified extraction_url = f"{AI_SERVER_URL}/v1/chat/completions" + # Build headers with authentication if available + headers = { + "Content-Type": "application/json" + } + + if AI_SERVICE_API_KEY: + headers["Authorization"] = f"Bearer {AI_SERVICE_API_KEY}" + # Create a proper prompt for fact extraction prompt = f""" Extract key facts from the following article in structured JSON format. @@ -65,6 +76,7 @@ def process_article_content(article_content, filename, source): "temperature": 0.3, "max_tokens": 1000 }, + headers=headers, timeout=60 ) diff --git a/docker-compose.yml b/docker-compose.yml index 067a871..412ca6b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,6 +35,7 @@ services: - ./ai_processor/output:/app/output environment: - AI_SERVICE_URL=http://example.com:4000 + - AI_SERVICE_API_KEY=${AI_SERVICE_API_KEY} embedder: build: ./embedding platform: linux/amd64 @@ -52,6 +53,7 @@ services: - AI_SERVER_PORT=4000 - CACHE_FILE=/app/processed_articles_cache.json - LOG_LEVEL=INFO + - AI_SERVICE_API_KEY=${AI_SERVICE_API_KEY} networks: ainetwork: diff --git a/embedding/advanced_embedder.py b/embedding/advanced_embedder.py index 90a80bf..77ab0fa 100644 --- a/embedding/advanced_embedder.py +++ b/embedding/advanced_embedder.py @@ -65,6 +65,9 @@ CACHE_FILE = os.getenv("CACHE_FILE", "processed_articles_cache.json") OPENAI_API_KEY = os.getenv("OPENAI_API_KEY", "placeholder-key") OPENAI_BASE_URL = os.getenv("OPENAI_BASE_URL", f"http://{AI_SERVER_HOST}:{AI_SERVER_PORT}") +# API Key for AI service authentication +AI_SERVICE_API_KEY = os.getenv("AI_SERVICE_API_KEY") + # Batch processing configuration BATCH_SIZE = int(os.getenv("EMBEDDING_BATCH_SIZE", "50")) @@ -74,12 +77,22 @@ def get_embedding(text): """ try: embedding_url = f"{AI_SERVER_URL}/v1/embeddings" + + # Build headers with authentication if available + headers = { + "Content-Type": "application/json" + } + + if AI_SERVICE_API_KEY: + headers["Authorization"] = f"Bearer {AI_SERVICE_API_KEY}" + response = requests.post( embedding_url, json={ "input": text, "model": "qwen3:8b" }, + headers=headers, timeout=60 ) response.raise_for_status() @@ -97,6 +110,14 @@ def extract_facts_from_article(article_content, title): # Use the centralized AI endpoint for fact extraction extraction_url = f"{AI_SERVER_URL}/v1/chat/completions" + # Build headers with authentication if available + headers = { + "Content-Type": "application/json" + } + + if AI_SERVICE_API_KEY: + headers["Authorization"] = f"Bearer {AI_SERVICE_API_KEY}" + # Create a proper prompt for fact extraction prompt = f""" Extract key facts from the following article in structured JSON format. @@ -136,6 +157,7 @@ def extract_facts_from_article(article_content, title): "temperature": 0.3, "max_tokens": 1000 }, + headers=headers, timeout=60 ) diff --git a/embedding/embedder.py b/embedding/deprecated/embedder.py similarity index 100% rename from embedding/embedder.py rename to embedding/deprecated/embedder.py diff --git a/test_implementation.py b/test_implementation.py new file mode 100644 index 0000000..2d0dd52 --- /dev/null +++ b/test_implementation.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +""" +Test script to verify the enhanced cache system implementation +""" + +import json +import os +import datetime +from scraper.scraper import load_processed_cache, save_processed_cache, get_processing_progress + +def test_cache_system(): + """Test the enhanced cache system""" + print("Testing enhanced cache system...") + + # Test loading cache (should work even if file doesn't exist) + cache = load_processed_cache() + print(f"Initial cache loaded: {len(cache)} entries") + + # Test saving cache + test_entry = { + "test_article_path": { + "processed_date": datetime.datetime.now().isoformat(), + "status": "completed", + "embedding_status": "pending", + "last_updated": datetime.datetime.now().isoformat() + } + } + + save_processed_cache(test_entry) + print("Cache saved successfully") + + # Test loading again + cache = load_processed_cache() + print(f"Cache loaded after save: {len(cache)} entries") + + # Test progress tracking + progress = get_processing_progress() + print(f"Progress tracking: {progress}") + + print("Cache system test completed successfully!") + +if __name__ == "__main__": + test_cache_system() \ No newline at end of file