added embedder to docker compose file and turned into cron job

docker-compose.yml

@@ -35,18 +35,23 @@ services:
   #     - ./ai_processor/output:/app/output
   #   environment:
   #     - AI_SERVICE_URL=http://192.168.8.124:11434 # Local AI service IP
-  # embedder:
+  embedder:
-  #   build: ./embedding
+    build: ./embedding
-  #   platform: linux/amd64
+    platform: linux/amd64
-  #   container_name: stockdocs-embedder
+    container_name: stockdocs-embedder
-  #   restart: unless-stopped
+    restart: unless-stopped
-  #   networks:
+    networks:
-  #     - ainetwork
+      - ainetwork
-  #   volumes:
+    volumes:
-  #     - ./ai_processor/output:/app/output
+      - ./scraper/articles:/scraper/articles
-  #   environment:
+      - ./embedding/logs:/app/logs
-  #     - CHROMADB_HOST=chromadb
+    environment:
-  #     - CHROMADB_PORT=8000
+      - CHROMADB_HOST=example.com
+      - CHROMADB_PORT=8000
+      - AI_SERVER_HOST=example.com
+      - AI_SERVER_PORT=4000
+      - CACHE_FILE=/app/processed_articles_cache.json
+      - LOG_LEVEL=INFO
 
 networks:
   ainetwork:

embedding/dockerfile

@@ -13,10 +13,10 @@ COPY requirements.txt .
 RUN pip install --no-cache-dir -r requirements.txt
 
 # Create directories
-RUN mkdir -p input output
+RUN mkdir -p input output logs
 
 # Copy the rest of the application
 COPY . .
 
 # Command to run the embedder
-CMD ["python", "embedder.py"]
+CMD ["python", "advanced_embedder.py"]

embedding/query_chromadb.py

@@ -8,6 +8,7 @@ import os
 import chromadb
 import json
 from pathlib import Path
+from chromadb.config import Settings
 
 def connect_to_chromadb():
     """Connect to ChromaDB instance"""
@@ -18,8 +19,16 @@ def connect_to_chromadb():
         
         print(f"Connecting to ChromaDB at {CHROMADB_HOST}:{CHROMADB_PORT}")
         
-        # Create client connection
+        # Create client connection using Settings for remote server
-        client = chromadb.HttpClient(host=CHROMADB_HOST, port=CHROMADB_PORT)
+        settings = Settings(
+            chroma_api_impl="rest",
+            chroma_server_host=CHROMADB_HOST,
+            chroma_server_http_port=CHROMADB_PORT,
+            chroma_server_ssl_enabled=False
+        )
+        
+        print(f"Creating client with settings: {settings}")
+        client = chromadb.Client(settings=settings)
         
         # Test connection
         collections = client.list_collections()

embedding/requirements.txt

@@ -1,55 +1,7 @@
-chromadb==0.3.23
+chromadb
-sentence-transformers==5.2.2
+sentence-transformers
-numpy==2.4.2
+numpy
-requests==2.32.5
+requests
-openai==2.16.0
+openai
-prometheus-client==0.24.1
+prometheus-client
-python-dotenv==1.2.1
+python-dotenv
-torch==2.10.0
-transformers==5.0.0
-scikit-learn==1.8.0
-pandas==3.0.0
-fastapi==0.128.0
-huggingface-hub==1.3.5
-tokenizers==0.22.2
-tqdm==4.67.2
-scipy==1.17.0
-hnswlib==0.8.0
-clickhouse-connect==0.10.0
-duckdb==1.4.4
-pydantic==2.12.5
-pyyaml==6.0.3
-httpx==0.28.1
-httpcore==1.0.9
-urllib3==2.6.3
-certifi==2026.1.4
-idna==3.11
-six==1.17.0
-python-dateutil==2.9.0.post0
-pytz==2025.2
-setuptools==80.10.2
-joblib==1.5.3
-threadpoolctl==3.6.0
-networkx==3.6.1
-sympy==1.14.0
-markupsafe==3.0.3
-jinja2==3.1.6
-annotated-types==0.7.0
-anyio==4.12.1
-sniffio==1.3.1
-jiter==0.12.0
-backoff==2.2.1
-posthog==7.8.0
-fsspec==2026.1.0
-hf-xet==1.2.0
-h11==0.16.0
-charset-normalizer==3.4.4
-mpmath==1.3.0
-zstandard==0.25.0
-lz4==4.4.5
-websockets==16.0
-uvicorn==0.40.0
-httptools==0.7.1
-uvloop==0.22.1
-watchfiles==1.1.1
-typer-slim==0.21.1

embedding/run_embedding_pipeline.sh

@@ -4,7 +4,7 @@
 # This script runs the advanced embedding pipeline periodically
 
 # Set working directory
-cd /embedding
+cd /app
 
 # Create log directory if it doesn't exist
 mkdir -p logs
@@ -28,4 +28,4 @@ else
     echo "ERROR: Embedding pipeline failed"
 fi
 
-echo "Embedding pipeline finished at $(date)" >> $LOG_FILE
+echo "Embedding pipeline finished at $(date)" >> $LOG_FILE

embedding/setup_embedding_cron.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# Setup script for embedding pipeline cron job
+# This script configures the cron job to run the embedding pipeline periodically
+
+echo "Setting up embedding pipeline cron job..."
+
+# Get the directory where this script is located
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+echo "Script directory: $SCRIPT_DIR"
+
+# Create cron job entry - this will be run on the server
+# The actual cron job should be added manually on the server
+echo "To set up the cron job on the server, please add the following line to your crontab:"
+echo ""
+echo "*/10 * * * * cd $SCRIPT_DIR && ./run_embedding_pipeline.sh"
+echo ""
+echo "You can edit crontab by running: crontab -e"
+echo ""
+echo "Or add it directly using: (crontab -l 2>/dev/null; echo '*/10 * * * * cd $SCRIPT_DIR && ./run_embedding_pipeline.sh') | crontab -"
+echo ""
+echo "The pipeline will run every 10 minutes."
+echo ""
+echo "To view current cron jobs: crontab -l"
+echo "To remove cron job: crontab -l | grep -v 'embedding' | crontab -"
+
+# Make the pipeline script executable
+chmod +x "$SCRIPT_DIR/run_embedding_pipeline.sh"
+
+echo "Pipeline script made executable"
+echo ""
+echo "Setup complete! Please add the cron job manually on your server:"
+echo "*/10 * * * * cd $SCRIPT_DIR && ./run_embedding_pipeline.sh"

embedding/simple_test.py

@@ -1,115 +0,0 @@
-#!/usr/bin/env python3
-"""
-Simple test to demonstrate the fact extraction capabilities with your article
-"""
-
-import json
-
-def extract_facts_from_article(article_content, title):
-    """
-    Extract structured facts from article content using the enhanced prompt
-    This simulates what happens in the real pipeline
-    """
-    print("=== Fact Extraction Test ===")
-    print(f"Processing article: {title}")
-    print("-" * 50)
-    
-    # This is what the enhanced prompt would do
-    facts = {
-        "title": title,
-        "summary": article_content[:200] + "..." if len(article_content) > 200 else article_content,
-        "main_topic": "Women's Basketball League",
-        "key_entities": ["Unrivaled", "Fox Business", "David Levy", "Caitlin Clark", "A'ja Wilson"],
-        "financial_impact": "positive",
-        "key_dates": ["2024", "1999", "2026"],
-        "main_points": [
-            "Unrivaled league breaks attendance records with 21,490 fans",
-            "Set new records for professional women's basketball game attendance",
-            "League revenue projected to exceed $40 million this season",
-            "54% increase in merchandise sales compared to last season",
-            "David Levy, early investor, praises the league's success"
-        ]
-    }
-    
-    print("Extracted facts:")
-    print(json.dumps(facts, indent=2))
-    print()
-    
-    return facts
-
-def test_embedding_simulation():
-    """Simulate embedding creation"""
-    print("=== Embedding Test ===")
-    print("Using qwen3:8b model for embeddings")
-    print("Embedding would be created from structured facts")
-    print("Result: Vector with 1536 dimensions (typical for text embeddings)")
-    print()
-
-def test_entity_storage():
-    """Demonstrate entity-based storage"""
-    print("=== Entity-Based Storage ===")
-    print("Storage structure with entity tracking:")
-    print("- Facts collection: Contains all structured facts with entity metadata")
-    print("- Entities tracked: Unrivaled (league), Fox Business (news source), David Levy (person)")
-    print("- Query capability: Filter by entity type or specific entity")
-    print()
-
-def main():
-    """Main test function"""
-    
-    # Your provided article content
-    article_content = """SOURCE:Fox Business – Headlines
-Unrivaled started out as an idea, and it has turned into a phenomenon.
-
-The three-on-three women's basketball league began last year in Miami, and this year the league has decided to go on tour. Its first stop on Friday night resulted in record-breaking numbers at a sold-out doubleheader.
-
-With 21,490 fans in attendance at Philadelphia's Xfinity Mobile Arena, Unrivaled set the all-time records for the highest-attended regular-season professional women's basketball game and the most-attended event ever at the arena that plays host to the Philadelphia 76ers and Flyers, as well as plenty of concerts.
-
-CLICK HERE FOR MORE SPORTS COVERAGE ON FOXBUSINESS.COM
-
-The previous respective records were 20,711, set by Caitlin Clark's Indiana Fever and the Washington Mystics on Sept. 19, 2024, and 21,424, set by the Backstreet Boys' "Into the Millennium" Tour on Sept. 29, 1999.
-
-Some critics may be surprised, considering the low viewership numbers early in the league's second season. But David Levy, an early investor of the league and former president of TNT Sports, felt the numbers were skewed and success was on the horizon.
-
-"I'm totally shocked that, and maybe I shouldn't be with what's going on in the world these days with news, how negative people got in the first two weeks of Unrivaled. The first two weeks, we ran into football. Football, NFL, college, Monday nights, championship game, you think anybody's gonna watch Unrivaled? Probably not," Levy admitted in a recent interview with FOX Business. "So, to all of a sudden come out and go, 'The league is dead.' No, it's shocking to me."
-
-BRITTNEY GRINER COMPARES RUSSIAN PRISON EXPERIENCE TO CURRENT ICE ENFORCEMENT IN UNITED STATES
-
-League sources told FOX Business that Unrivaled is on track to eclipse $40 million in league revenue this season, up more than 48 percent from last season's $27 million revenue. Even during the low-ratings weekend, Levy mentioned, social engagement was way up. Merchandise sales are also up 54% from September through the end of opening weekend this season compared to that same time period last season.
-
-"I’m about the facts. The facts are, every single other metric is up," Levy said.
-
-Levy said he knew the league would be a hit when he realized that the quality of play was A-plus.
-
-"The most important thing is the product on the floor has to be great. I didn't know that out of the gate. I didn't know how hard these girls were gonna play. I didn't. Was this gonna be more of a scrimmage? But after the first two weeks, I knew it was gold," Levy said.
-
-Clark and A'ja Wilson, arguably the WNBA's two biggest stars, have yet to join the league. But that's OK for now, Levy said.
-
-"If you had closed your eyes and tried to say, 'What if this was an NBA product? And you had the top 56 NBA players except Steph Curry and LeBron didn't play, but everybody else was in. This would be the hottest thing during the summer. If that was a summer league, it would be sold out," Levy said.
-
-"It's every single great player playing in a three-on-three league. It is absolutely a huge opportunity, and that's why I think it just rose so fast. The quality of play, the names on the back of the jerseys, the social strategy is amazing. These women, they all have equity. Everyone has a following; women athletes completely engage with their fans. The breadth of impressions, I think, is a phenomenal one. I think that's why the league is as successful as it is after just a year and three weeks." """
-
-    title = "Unrivaled Women's Basketball League Breaks Attendance Records"
-    
-    # Run tests
-    facts = extract_facts_from_article(article_content, title)
-    test_embedding_simulation()
-    test_entity_storage()
-    
-    print("=== End-to-End Pipeline Demonstration Complete ===")
-    print("The system successfully demonstrates:")
-    print("✓ Enhanced fact extraction with entity identification")
-    print("✓ Structured data format for easy querying")
-    print("✓ Entity-based storage for flexible filtering")
-    print("✓ Ready for /facts endpoint queries")
-    print("✓ Efficient qwen3:8b model for embeddings")
-    print()
-    print("When the full pipeline runs:")
-    print("1. Article processed from scraper directory")
-    print("2. Facts extracted with entity tracking")
-    print("3. Embeddings created using qwen3:8b")
-    print("4. Data stored in ChromaDB collections")
-    print("5. Available via all MCP server endpoints")
-
-if __name__ == "__main__":
-    main()