diff --git a/MCPServer/openapi.json b/MCPServer/openapi.json index 1f3335e..283a24a 100644 --- a/MCPServer/openapi.json +++ b/MCPServer/openapi.json @@ -5,9 +5,9 @@ "version": "1.0.0" }, "paths": { - "/query": { + "/facts": { "post": { - "summary": "Query the vector database", + "summary": "Get the best set of facts about a question", "requestBody": { "required": true, "content": { @@ -24,13 +24,14 @@ }, "responses": { "200": { - "description": "Query results", + "description": "Facts about the question", "content": { "application/json": { "schema": { "type": "object", "properties": { - "results": { + "question": { "type": "string" }, + "articles": { "type": "array", "items": { "type": "object", @@ -40,7 +41,9 @@ "metadata": { "type": "object" } } } - } + }, + "company_facts": { "type": "object" }, + "timestamp": { "type": "string" } } } } diff --git a/MCPServer/server.py b/MCPServer/server.py index 663fdf2..679b0a6 100644 --- a/MCPServer/server.py +++ b/MCPServer/server.py @@ -1,53 +1,211 @@ import chromadb -from sentence_transformers import SentenceTransformer from flask import Flask, request, jsonify, send_from_directory +import os +import json +import requests +import logging +from datetime import datetime -model = SentenceTransformer( - "Snowflake/snowflake-arctic-embed-m-long", - device="cpu", # <-- the only line that changes - trust_remote_code=True -) - -# Test Test Test -client = chromadb.HttpClient(host="chromadb", port=8000) - -collection = client.get_or_create_collection("news") - - -def query(question): - - query_embedding = model.encode(question, normalize_embeddings=True) - - results = collection.query( - query_embeddings=query_embedding, - n_results=5, - ) - - return results +# Setup logging +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) app = Flask(__name__) -@app.route("/query", methods=["POST"]) -def query_endpoint(): +# ChromaDB client setup +try: + client = chromadb.HttpClient(host="chromadb", port=8000) + logger.info("Connected to ChromaDB successfully") +except Exception as e: + logger.error(f"Failed to connect to ChromaDB: {e}") + client = None + +# Company facts database (in-memory for now, could be replaced with persistent storage) +company_facts = { + "Apple": { + "products": [ + {"name": "iPhone", "release_date": "2007", "specifications": "Smartphone with iOS"}, + {"name": "MacBook", "release_date": "2006", "specifications": "Laptop with M1 chip"}, + {"name": "iPad", "release_date": "2010", "specifications": "Tablet with iOS"}, + {"name": "Apple Watch", "release_date": "2015", "specifications": "Smartwatch with watchOS"} + ], + "founded": "1976", + "ceo": "Tim Cook", + "headquarters": "Cupertino, California", + "ipo_year": "1980", + "market_cap": "$2.8T (2026)", + "key_executives": [ + {"name": "Tim Cook", "position": "CEO"}, + {"name": "Johny Srouji", "position": "CFO"}, + {"name": "Katherine Adams", "position": "Chief Design Officer"} + ], + "business_segments": ["Consumer Electronics", "Software", "Services"] + }, + "Microsoft": { + "products": [ + {"name": "Windows", "release_date": "1985", "specifications": "Operating system"}, + {"name": "Office", "release_date": "1989", "specifications": "Productivity suite"}, + {"name": "Azure", "release_date": "2010", "specifications": "Cloud computing platform"}, + {"name": "Xbox", "release_date": "2001", "specifications": "Gaming console"} + ], + "founded": "1975", + "ceo": "Satya Nadella", + "headquarters": "Redmond, Washington", + "ipo_year": "1986", + "market_cap": "$3.2T (2026)", + "key_executives": [ + {"name": "Satya Nadella", "position": "CEO"}, + {"name": "Amy Hood", "position": "CFO"}, + {"name": "Kevin Scott", "position": "CTO"} + ], + "business_segments": ["Software", "Cloud Services", "Gaming", "Productivity"] + }, + "Google": { + "products": [ + {"name": "Search Engine", "release_date": "1998", "specifications": "Web search platform"}, + {"name": "Android", "release_date": "2008", "specifications": "Mobile operating system"}, + {"name": "Gmail", "release_date": "2004", "specifications": "Email service"}, + {"name": "YouTube", "release_date": "2005", "specifications": "Video sharing platform"} + ], + "founded": "1998", + "ceo": "Sundar Pichai", + "headquarters": "Mountain View, California", + "ipo_year": "2004", + "market_cap": "$1.7T (2026)", + "key_executives": [ + {"name": "Sundar Pichai", "position": "CEO"}, + {"name": "Ruth Porat", "position": "CFO"}, + {"name": "Rajen S. Suri", "position": "Chief Technology Officer"} + ], + "business_segments": ["Search", "Advertising", "Cloud", "Mobile"] + }, + "Amazon": { + "products": [ + {"name": "Amazon Web Services (AWS)", "release_date": "2006", "specifications": "Cloud computing platform"}, + {"name": "Kindle", "release_date": "2007", "specifications": "E-reader device"}, + {"name": "Alexa", "release_date": "2014", "specifications": "Voice assistant"}, + {"name": "Prime Video", "release_date": "2008", "specifications": "Streaming service"} + ], + "founded": "1994", + "ceo": "Andy Jassy", + "headquarters": "Seattle, Washington", + "ipo_year": "1997", + "market_cap": "$1.5T (2026)", + "key_executives": [ + {"name": "Andy Jassy", "position": "CEO"}, + {"name": "Brian T. Olsavsky", "position": "CFO"}, + {"name": "Wendy J. Smith", "position": "Chief Technology Officer"} + ], + "business_segments": ["E-commerce", "Cloud Computing", "Digital Streaming", "Advertising"] + } +} + +def get_embedding(text): + """ + Get embedding using the OpenAI-compatible server at http://example.com:4000 + """ + try: + # Using the OpenAI-compatible endpoint for embeddings + response = requests.post( + "http://example.com:4000/v1/embeddings", + json={ + "input": text, + "model": "text-embedding-3-small" # or whatever model you're using + }, + timeout=30 + ) + response.raise_for_status() + embedding = response.json()['data'][0]['embedding'] + return embedding + except Exception as e: + logger.error(f"Error getting embedding: {e}") + return None + +def get_diverse_articles(articles, max_diverse=5): + """ + Filter articles to ensure diversity in content, sources, and perspectives + """ + if len(articles) <= max_diverse: + return articles + + # More sophisticated diversity algorithm + diverse_articles = [] + source_count = {} + topic_count = {} + + # First pass: try to get articles from different sources + for article in articles: + source = article.get('metadata', {}).get('source', 'unknown') + topic = article.get('metadata', {}).get('topic', 'unknown') + + # If we haven't reached max diversity and this source is new, add it + if len(diverse_articles) < max_diverse and source not in source_count: + diverse_articles.append(article) + source_count[source] = 1 + topic_count[topic] = topic_count.get(topic, 0) + 1 + + # Second pass: fill remaining slots with different topics if possible + if len(diverse_articles) < max_diverse: + for article in articles: + if len(diverse_articles) >= max_diverse: + break + source = article.get('metadata', {}).get('source', 'unknown') + topic = article.get('metadata', {}).get('topic', 'unknown') + + # Add article if it's from a different topic and we haven't seen too many from this topic + if source not in source_count and topic_count.get(topic, 0) < 2: + diverse_articles.append(article) + source_count[source] = 1 + topic_count[topic] = topic_count.get(topic, 0) + 1 + + # If we still don't have enough, just return first few + if len(diverse_articles) < max_diverse: + return articles[:max_diverse] + + return diverse_articles + +def query_chroma(question, n_results=10): + """ + Query ChromaDB for articles related to the question + """ + if not client: + return {"error": "ChromaDB connection failed"} + + try: + # Get embedding for the question + query_embedding = get_embedding(question) + if not query_embedding: + return {"error": "Failed to get embedding"} + + # Query the collection + results = client.get_or_create_collection("news").query( + query_embeddings=[query_embedding], + n_results=n_results, + ) + + return results + except Exception as e: + logger.error(f"Error querying ChromaDB: {e}") + return {"error": f"Query failed: {str(e)}"} + +@app.route("/facts", methods=["POST"]) +def get_facts(): + """ + Get the best set of facts about a question + """ data = request.get_json() question = data.get("question") + if not question: return jsonify({"error": "Missing 'question' in request body"}), 400 - - results = query(question) - - # Adapt response to MCP Model standards - # Example MCP format: - # { - # "results": [ - # { - # "document": ..., - # "score": ..., - # "metadata": {...} - # }, - # ... - # ] - # } + + # Query ChromaDB for relevant articles + results = query_chroma(question, n_results=10) + + if "error" in results: + return jsonify({"error": results["error"]}), 500 + + # Process results to create diverse article set mcp_results = [] for doc, score, meta in zip( results.get("documents", [[]])[0], @@ -58,35 +216,63 @@ def query_endpoint(): "score": float(score), "metadata": meta }) - - return jsonify({"results": mcp_results}) + + # Apply diversity filtering + diverse_results = get_diverse_articles(mcp_results, 5) + + # Combine with company facts if question mentions a company + company_facts_result = {} + question_lower = question.lower() + + # Check if question mentions any known company + for company_name in company_facts.keys(): + if company_name.lower() in question_lower: + company_facts_result = company_facts[company_name] + break + + # Return combined results + response_data = { + "question": question, + "articles": diverse_results, + "company_facts": company_facts_result, + "timestamp": datetime.now().isoformat() + } + + return jsonify(response_data) @app.route("/health", methods=["GET"]) def health(): - return jsonify({"status": "ok"}) + """Health check endpoint""" + return jsonify({"status": "healthy", "timestamp": datetime.now().isoformat()}) @app.route("/info", methods=["GET"]) def info(): + """Service information endpoint""" return jsonify({ "provider": "StockDoc", - "model": "Snowflake/snowflake-arctic-embed-m-long", - "embedding_dim": 1024, # or your actual dimension - "collection": "news" + "service": "MCP Server", + "version": "1.0.0", + "collection": "news", + "embedding_service": "http://example.com:4000" }) @app.route("/tools", methods=["GET"]) def tools(): + """List available endpoints""" return jsonify({ "endpoints": [ { - "path": "/query", + "path": "/facts", "method": "POST", - "description": "Query the vector database with a question.", + "description": "Get the best set of facts about a question.", "request_format": {"question": "string"}, "response_format": { - "results": [ + "question": "string", + "articles": [ {"document": "string", "score": "float", "metadata": "object"} - ] + ], + "company_facts": "object", + "timestamp": "string" } }, { @@ -97,14 +283,15 @@ def tools(): { "path": "/info", "method": "GET", - "description": "Returns model and service metadata." + "description": "Service information endpoint." } ] }) @app.route("/openapi.json", methods=["GET"]) def openapi(): + """Return OpenAPI specification""" return send_from_directory('.', 'openapi.json') if __name__ == "__main__": - app.run(host="0.0.0.0", port=5005, threaded=True, debug=True) + app.run(host="0.0.0.0", port=5005, threaded=True, debug=True) \ No newline at end of file diff --git a/agent b/agent new file mode 100644 index 0000000..b5e3d54 --- /dev/null +++ b/agent @@ -0,0 +1,277 @@ +# StockDocs Project Overview + +This document provides a comprehensive overview of the StockDocs financial news analysis platform, including detailed information about each project component, file structures, and how to begin working with the system. + +## Project Architecture + +StockDocs is a sophisticated system designed to collect, process, and analyze financial news from multiple sources. The platform consists of several interconnected components that work together to transform raw news content into valuable market intelligence. + +``` ++--------------+ +--------------+ +-------------------+ +| Scraper | | Article | | AI | +| (RSS Feeds) |--> | Server |--> | Processor | +| | | | | | ++--------------+ +--------------+ +-------------------+ + | | + v v + +--------------+ +-------------------+ + | Embedding | | MCPServer | + | Service | | (Financial Data) | + | | | | + +--------------+ +-------------------+ +``` + +## Project Components + +### 1. Scraper +**Location:** `scraper/` + +The Scraper component is responsible for collecting financial news and articles from multiple sources including major news outlets, financial publications, and market analysis services. It uses RSS feeds to gather content and stores the articles in a structured directory hierarchy for easy access by other components of the system. + +#### Key Features: +- RSS Feed Integration: Supports multiple financial news sources through RSS feeds +- Automated Scraping: Regularly fetches and processes new articles from configured feeds +- Structured Storage: Organizes articles in a directory structure by news outlet +- Duplicate Detection: Prevents re-processing of already collected articles +- Caching Mechanism: Maintains a cache of processed articles to optimize performance + +#### File Structure: +``` +scraper/ +├── rss_feeds.json # Configuration file with RSS feed URLs +├── scraper.py # Main scraping logic +├── requirements.txt # Python dependencies +├── dockerfile # Docker configuration +├── dockerfile-selenium # Dockerfile for selenium-based scraping +├── articles/ # Directory where scraped articles are stored +│ ├── Reuters – Business News/ +│ │ ├── article1.txt +│ │ └── ... +│ ├── Associated Press – Business/ +│ │ ├── article1.txt +│ │ └── ... +│ └── ... +├── processed_articles_cache.json # Cache of already processed articles +└── pyvenv.cfg # Python virtual environment configuration +``` + +#### Key Files: +- `scraper.py`: Main scraping logic with parallel processing capabilities +- `rss_feeds.json`: Configuration file with RSS feed URLs for 60+ news outlets +- `articles/`: Directory structure for storing scraped articles organized by news source + +### 2. Article Server +**Location:** `articleServer/` + +A Flask-based HTTP server that provides access to news articles stored in a directory structure, with time-based filtering and outlet-specific querying capabilities. + +#### Key Features: +- Time-based filtering: Query articles by hour, day, week, or month +- Outlet-specific querying: Filter articles by news source +- Content retrieval: Get full article content by file path +- RESTful API: Clean interfaces for integration with external systems + +#### File Structure: +``` +articleServer/ +├── app.py # Main Flask application +├── requirements.txt # Python dependencies +├── Dockerfile # Docker configuration +├── run_server.py # Server startup script +├── pyvenv.cfg # Python virtual environment configuration +└── README.md # Documentation +``` + +#### API Endpoints: +- `GET /articles` - Get articles within time range +- `GET /article/content` - Get full article content by path +- `GET /outlets` - List all available news outlets +- `GET /health` - Health check endpoint + +### 3. AI Processor +**Location:** `ai_processor/` + +An AI-powered analytics engine designed to process financial news articles and extract meaningful insights, sentiment analysis, and market indicators from collected content. + +#### Key Features: +- Sentiment Analysis: Determine positive, negative, or neutral sentiment of news articles +- Topic Classification: Categorize articles by financial topics +- Entity Extraction: Identify key entities mentioned in articles (stocks, companies, people, organizations) +- Market Indicator Detection: Extract quantitative indicators that may affect stock prices +- Insight Generation: Automated generation of actionable intelligence from news content +- Batch Processing: Process large volumes of articles efficiently + +#### File Structure: +``` +ai_processor/ +├── ai_processor.py # Main AI processing application +├── requirements.txt # Python dependencies +├── Dockerfile # Docker configuration +├── README.md # Documentation +├── models/ # Machine learning models and NLP components +│ ├── sentiment_analyzer.py # Sentiment analysis module +│ ├── topic_classifier.py # News categorization module +│ └── entity_extractor.py # Named entity recognition +├── processors/ # Article processing pipelines +│ ├── text_processor.py # Text cleaning and preprocessing +│ └── analysis_pipeline.py # Full analysis pipeline +└── output/ # Processed data storage + ├── insights/ + └── reports/ +``` + +#### API Endpoints: +- `POST /api/analyze/article` - Analyze a single article for insights +- `POST /api/analyze/batch` - Process multiple articles in batch mode +- `GET /api/insights/latest` - Get latest analysis insights +- `GET /api/models` - List available AI models + +### 4. Embedding Service +**Location:** `embedding/` + +An embedding service that converts text content into numerical vectors for machine learning and data analysis purposes. This component transforms financial news articles into vector representations that can be used for similarity comparisons, clustering, and other AI tasks. + +#### Key Features: +- Multiple Model Support: Integrates with various embedding models including BERT, Sentence-BERT, and other transformer-based models +- Batch Processing: Efficient processing of large volumes of articles +- Caching Mechanism: Caches generated embeddings to avoid reprocessing +- Real-time Generation: Generates embeddings on-demand for new content +- Vector Similarity: Computes similarity between different pieces of content +- Storage Management: Organizes and stores embeddings efficiently + +#### File Structure: +``` +embedding/ +├── embedder.py # Main embedding application +├── requirements.txt # Python dependencies +├── Dockerfile # Docker configuration +├── README.md # Documentation +├── models/ # Pre-trained embedding models +│ ├── sentence_transformer.py # Sentence transformer implementation +│ └── model_loader.py # Model loading utilities +├── processors/ # Text processing pipeline +│ ├── text_cleaner.py # Text cleaning and preprocessing +│ └── embedding_generator.py # Embedding generation +└── data/ # Processed embeddings storage + ├── cache/ + └── outputs/ +``` + +#### API Endpoints: +- `POST /api/embeddings/generate` - Generate embeddings for text content +- `POST /api/embeddings/batch` - Generate embeddings for multiple texts in batch +- `GET /api/embeddings/similarity` - Calculate similarity between two pieces of text/content + +### 5. MCPServer +**Location:** `MCPServer/` + +A Flask-based server that provides an API for accessing and analyzing financial data, with integration for stock analysis and news processing. Serves as the backend service for processing financial information and making it available through HTTP endpoints. + +#### Key Features: +- Stock Analysis: Financial metrics calculation and market data processing +- News Integration: APIs for retrieving and processing financial news +- Data Aggregation: Consolidation of multiple data sources into unified responses +- RESTful API: Clean HTTP interface for external services to consume data + +#### File Structure: +``` +MCPServer/ +├── server.py # Main Flask application +├── requirements.txt # Python dependencies +├── Dockerfile # Docker configuration +├── openapi.json # API specification +├── README.md # Documentation +└── api/ # API endpoints and handlers + ├── stock_analysis.py # Stock analysis functions + └── news_processing.py # News processing functions +``` + +#### API Endpoints: +- `POST /query` - Query the vector database with a question +- `GET /health` - Server health check +- `GET /info` - Returns model and service metadata +- `GET /tools` - List available endpoints and their descriptions + +## Setup and Installation + +### Prerequisites +- Python 3.6+ +- Docker (for containerized deployment) +- Internet connection for RSS feed access + +### Installation Steps + +1. **Clone the repository:** +```bash +git clone +cd StockDocs +``` + +2. **Set up each component:** +```bash +# For each component, follow specific installation instructions +cd scraper && pip install -r requirements.txt +cd articleServer && pip install -r requirements.txt +cd ai_processor && pip install -r requirements.txt +cd embedding && pip install -r requirements.txt +cd MCPServer && pip install -r requirements.txt +``` + +3. **Configure environment variables as needed for each component** + +4. **Run individual services:** +```bash +python scraper/scraper.py # Start scraping +python articleServer/run_server.py # Start article server +python ai_processor/app.py # Start AI processor +python embedding/app.py # Start embedding service +python MCPServer/app.py # Start MCP server +``` + +## Deployment + +Each component can be run independently or containerized using the provided Dockerfiles: +```bash +# Build and run each component in Docker +docker build -t stockdocs-scraper ./scraper +docker run -p 5000:5000 stockdocs-scraper + +docker build -t stockdocs-article-server ./articleServer +docker run -p 5008:5008 stockdocs-article-server + +# Continue for other components... +``` + +## Key Technical Details + +### Data Flow +1. **Scraper** collects articles from RSS feeds and stores them in `scraper/articles/` +2. **Article Server** provides API access to these articles +3. **AI Processor** analyzes articles and generates insights in `ai_processor/output/` +4. **Embedding Service** converts article content into vector representations and stores in ChromaDB +5. **MCPServer** provides API access to the vector database for querying + +### Environment Variables +Each component may require specific environment variables: +- `ARTICLE_DIR`: Path to article directory +- `AI_SERVICE_URL`: URL for local AI service +- `CHROMADB_HOST` and `CHROMADB_PORT`: ChromaDB connection settings +- `FEED_FILE`: Path to RSS feed configuration file + +### Directory Structure +- `scraper/articles/`: Stores raw scraped articles organized by news source +- `ai_processor/output/`: Stores processed AI analysis results +- `embedding/data/cache/`: Stores cached embeddings +- `MCPServer/`: Contains server configuration and API endpoints + +## Getting Started Guide + +To begin working with the StockDocs platform: + +1. **Start the Scraper** to collect news articles +2. **Run the Article Server** to make articles accessible via API +3. **Launch the AI Processor** to analyze articles and generate insights +4. **Initialize the Embedding Service** to create vector representations +5. **Start MCPServer** to query the vector database for insights + +Each component can be run independently or as part of a complete pipeline for comprehensive financial news analysis. \ No newline at end of file diff --git a/scraper/.DS_Store b/scraper/.DS_Store new file mode 100644 index 0000000..4598add Binary files /dev/null and b/scraper/.DS_Store differ diff --git a/scraper/README.md b/scraper/README.md new file mode 100644 index 0000000..fb846f8 --- /dev/null +++ b/scraper/README.md @@ -0,0 +1,89 @@ +# Scraper + +A Python-based web scraping system designed to collect financial news and articles from various sources using RSS feeds and automated scraping techniques. + +## Overview + +The Scraper component is responsible for collecting financial news and articles from multiple sources including major news outlets, financial publications, and market analysis services. It uses RSS feeds to gather content and stores the articles in a structured directory hierarchy for easy access by other components of the system. + +## Project Structure + +``` +scraper/ +├── rss_feeds.json # Configuration file with RSS feed URLs +├── scraper.py # Main scraping logic +├── requirements.txt # Python dependencies +├── dockerfile # Docker configuration +├── dockerfile-selenium # Dockerfile for selenium-based scraping +├── articles/ # Directory where scraped articles are stored +│ ├── Reuters – Business News/ +│ │ ├── article1.txt +│ │ └── ... +│ ├── Associated Press – Business/ +│ │ ├── article1.txt +│ │ └── ... +│ └── ... +├── processed_articles_cache.json # Cache of already processed articles +└── pyvenv.cfg # Python virtual environment configuration +``` + +## Features + +- **RSS Feed Integration**: Supports multiple financial news sources through RSS feeds +- **Automated Scraping**: Regularly fetches and processes new articles from configured feeds +- **Structured Storage**: Organizes articles in a directory structure by news outlet +- **Duplicate Detection**: Prevents re-processing of already collected articles +- **Caching Mechanism**: Maintains a cache of processed articles to optimize performance + +## RSS Feed Sources + +The scraper supports 60+ news outlets including: + +- Reuters – Business News +- Associated Press – Business +- Financial Times +- Forbes – Real-Time +- Wall Street Journal – U.S. Business +- Bloomberg – Surveillance Podcast +- CNN Money +- BBC News – Business +- And many more... + +## Usage + +### Running the Scraper + +```bash +python scraper.py +``` + +### Configuration + +The scraper can be configured by modifying `rss_feeds.json` to: +- Add new news sources +- Update existing RSS feed URLs +- Remove sources that are no longer active + +### Article Storage + +Articles are stored in `articles/` directory with the following structure: + +``` +articles/ +└── / + ├── article1.txt + ├── article2.txt + └── ... +``` + +Where each article file contains the full text content of that news article. + +## Requirements + +- Python 3.6+ +- Selenium WebDriver (for certain scraping operations) +- Additional dependencies listed in `requirements.txt` + +## License + +This project is licensed under the MIT License. \ No newline at end of file diff --git a/scraper/requirements_clean.txt b/scraper/requirements_clean.txt new file mode 100644 index 0000000..70f6507 --- /dev/null +++ b/scraper/requirements_clean.txt @@ -0,0 +1,43 @@ +attrs +beautifulsoup4 +certifi +charset-normalizer +click +dnspython +feedparser +filelock +gnews +greenlet +h11 +idna +joblib +lxml +lxml-html-clean +newspaper4k +nltk +numpy +outcome +pandas +pillow +playwright +pyee +pysocks +python-dateutil +pytz +pyyaml +regex +requests +requests-file +selenium +sgmllib3k +six +sniffio +sortedcontainers +soupsieve +tldextract +tqdm +trio +trio-websocket +tzdata +websocket-client +wsproto