Add API key authentication for AI service connections

MCPServer/README.md

@@ -53,6 +53,7 @@ The server supports configuration through environment variables:
 - `FLASK_ENV` - Set to 'development' or 'production' (default: 'development')
 - `DATABASE_URL` - URL for database connection (e.g., PostgreSQL)
 - `API_KEY` - API key for external services
+- `AI_SERVICE_API_KEY` - API key for authenticating with the centralized AI service at http://example.com:4000
 - `LOG_LEVEL` - Logging level (DEBUG, INFO, WARNING, ERROR)
 
 ## Installation

MCPServer/server.py

@@ -106,12 +106,22 @@ def get_embedding(text):
     """
     try:
         # Using the OpenAI-compatible endpoint for embeddings
+        headers = {
+            "Content-Type": "application/json"
+        }
+        
+        # Add API key if available
+        api_key = os.getenv("AI_SERVICE_API_KEY")
+        if api_key:
+            headers["Authorization"] = f"Bearer {api_key}"
+        
         response = requests.post(
             "http://example.com:4000/v1/embeddings",
             json={
                 "input": text,
                 "model": "text-embedding-3-small"  # or whatever model you're using
             },
+            headers=headers,
             timeout=30
         )
         response.raise_for_status()

agent.md

@@ -265,6 +265,12 @@ scraper/
 ### Environment Variables
 Each project may require specific environment variables. Check individual `README.md` files for details.
 
+### Authentication
+The system now supports API key authentication for connections to the centralized AI service at `http://example.com:4000`. To enable authentication:
+
+1. Set the `AI_SERVICE_API_KEY` environment variable with your API key
+2. All connections to the AI service will automatically include the `Authorization: Bearer {api_key}` header
+
 ### Data Storage
 - Articles are stored in the scraper component
 - Processed data flows through the ai_processor

ai_processor/ai_processor.py

@@ -12,6 +12,9 @@ AI_SERVER_HOST = os.getenv("AI_SERVER_HOST", "example.com")
 AI_SERVER_PORT = os.getenv("AI_SERVER_PORT", "4000")
 AI_SERVER_URL = f"http://{AI_SERVER_HOST}:{AI_SERVER_PORT}"
 
+# API Key for AI service authentication
+AI_SERVICE_API_KEY = os.getenv("AI_SERVICE_API_KEY")
+
 def process_article_content(article_content, filename, source):
     """
     Process article content and extract key facts using the centralized AI service
@@ -21,6 +24,14 @@ def process_article_content(article_content, filename, source):
         # Use the gpt-oss model for fact extraction as specified
         extraction_url = f"{AI_SERVER_URL}/v1/chat/completions"
         
+        # Build headers with authentication if available
+        headers = {
+            "Content-Type": "application/json"
+        }
+        
+        if AI_SERVICE_API_KEY:
+            headers["Authorization"] = f"Bearer {AI_SERVICE_API_KEY}"
+        
         # Create a proper prompt for fact extraction
         prompt = f"""
         Extract key facts from the following article in structured JSON format. 
@@ -65,6 +76,7 @@ def process_article_content(article_content, filename, source):
                 "temperature": 0.3,
                 "max_tokens": 1000
             },
+            headers=headers,
             timeout=60
         )
         

docker-compose.yml

@@ -35,6 +35,7 @@ services:
       - ./ai_processor/output:/app/output
     environment:
       - AI_SERVICE_URL=http://example.com:4000
+      - AI_SERVICE_API_KEY=${AI_SERVICE_API_KEY}
   embedder:
     build: ./embedding
     platform: linux/amd64
@@ -52,6 +53,7 @@ services:
       - AI_SERVER_PORT=4000
       - CACHE_FILE=/app/processed_articles_cache.json
       - LOG_LEVEL=INFO
+      - AI_SERVICE_API_KEY=${AI_SERVICE_API_KEY}
 
 networks:
   ainetwork:

embedding/advanced_embedder.py

@@ -65,6 +65,9 @@ CACHE_FILE = os.getenv("CACHE_FILE", "processed_articles_cache.json")
 OPENAI_API_KEY = os.getenv("OPENAI_API_KEY", "placeholder-key")
 OPENAI_BASE_URL = os.getenv("OPENAI_BASE_URL", f"http://{AI_SERVER_HOST}:{AI_SERVER_PORT}")
 
+# API Key for AI service authentication
+AI_SERVICE_API_KEY = os.getenv("AI_SERVICE_API_KEY")
+
 # Batch processing configuration
 BATCH_SIZE = int(os.getenv("EMBEDDING_BATCH_SIZE", "50"))
 
@@ -74,12 +77,22 @@ def get_embedding(text):
     """
     try:
         embedding_url = f"{AI_SERVER_URL}/v1/embeddings"
+        
+        # Build headers with authentication if available
+        headers = {
+            "Content-Type": "application/json"
+        }
+        
+        if AI_SERVICE_API_KEY:
+            headers["Authorization"] = f"Bearer {AI_SERVICE_API_KEY}"
+        
         response = requests.post(
             embedding_url,
             json={
                 "input": text,
                 "model": "qwen3:8b"
             },
+            headers=headers,
             timeout=60
         )
         response.raise_for_status()
@@ -97,6 +110,14 @@ def extract_facts_from_article(article_content, title):
         # Use the centralized AI endpoint for fact extraction
         extraction_url = f"{AI_SERVER_URL}/v1/chat/completions"
         
+        # Build headers with authentication if available
+        headers = {
+            "Content-Type": "application/json"
+        }
+        
+        if AI_SERVICE_API_KEY:
+            headers["Authorization"] = f"Bearer {AI_SERVICE_API_KEY}"
+        
         # Create a proper prompt for fact extraction
         prompt = f"""
         Extract key facts from the following article in structured JSON format. 
@@ -136,6 +157,7 @@ def extract_facts_from_article(article_content, title):
                 "temperature": 0.3,
                 "max_tokens": 1000
             },
+            headers=headers,
             timeout=60
         )
         

test_implementation.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+"""
+Test script to verify the enhanced cache system implementation
+"""
+
+import json
+import os
+import datetime
+from scraper.scraper import load_processed_cache, save_processed_cache, get_processing_progress
+
+def test_cache_system():
+    """Test the enhanced cache system"""
+    print("Testing enhanced cache system...")
+    
+    # Test loading cache (should work even if file doesn't exist)
+    cache = load_processed_cache()
+    print(f"Initial cache loaded: {len(cache)} entries")
+    
+    # Test saving cache
+    test_entry = {
+        "test_article_path": {
+            "processed_date": datetime.datetime.now().isoformat(),
+            "status": "completed",
+            "embedding_status": "pending",
+            "last_updated": datetime.datetime.now().isoformat()
+        }
+    }
+    
+    save_processed_cache(test_entry)
+    print("Cache saved successfully")
+    
+    # Test loading again
+    cache = load_processed_cache()
+    print(f"Cache loaded after save: {len(cache)} entries")
+    
+    # Test progress tracking
+    progress = get_processing_progress()
+    print(f"Progress tracking: {progress}")
+    
+    print("Cache system test completed successfully!")
+
+if __name__ == "__main__":
+    test_cache_system()