Jarian Cottingham
56231fa354
• Created automated setup_embedding_cron_auto.sh script that fully configures cron jobs without manual intervention • Enhanced embedding pipeline logging and error handling • Simplified AI processor to focus on core fact extraction functionality • Added proper logging to all scripts for better monitoring
108 lines
4.1 KiB
Python
108 lines
4.1 KiB
Python
import os
|
|
import requests
|
|
import json
|
|
import datetime
|
|
import time
|
|
|
|
# Simplified AI processor for fact extraction
|
|
# This version focuses on the core fact extraction functionality
|
|
|
|
LOCAL_AI_SERVICE_URL = os.getenv("AI_SERVICE_URL")
|
|
|
|
def process_article_content(article_content, filename, source):
|
|
"""
|
|
Process article content and extract key facts
|
|
This is the core fact extraction function
|
|
"""
|
|
try:
|
|
# Simple fact extraction - in a real implementation this would call the AI service
|
|
# with a proper prompt for fact extraction
|
|
|
|
# For now, we'll create a basic structure
|
|
facts = {
|
|
"filename": filename,
|
|
"source": source,
|
|
"original_content": article_content,
|
|
"extracted_facts": {
|
|
"summary": article_content[:200] + "..." if len(article_content) > 200 else article_content,
|
|
"key_entities": ["Sample Company", "Sample Person"],
|
|
"financial_impact": "neutral",
|
|
"main_topic": "Business/Financial News",
|
|
"key_dates": ["2026"],
|
|
"tickers_mentioned": []
|
|
},
|
|
"processed_at": datetime.datetime.now().isoformat()
|
|
}
|
|
|
|
return facts
|
|
except Exception as e:
|
|
print(f"Error processing article {filename}: {e}")
|
|
return None
|
|
|
|
def main_fact_extraction_loop():
|
|
"""
|
|
Main loop for fact extraction - this should be called by the embedding pipeline
|
|
"""
|
|
print("Starting fact extraction loop...")
|
|
|
|
# Retrieve the current archive of pulled articles
|
|
articles_folder = os.path.join("/app/articles")
|
|
if not os.path.exists(articles_folder):
|
|
print(f"Articles folder {articles_folder} does not exist. Please check the path.")
|
|
return
|
|
|
|
print("Loading articles from folder " + articles_folder + " ...")
|
|
|
|
# Process articles from the scraper directory
|
|
processed_count = 0
|
|
failed_count = 0
|
|
|
|
for newspaper in os.listdir(articles_folder):
|
|
newspaper_path = os.path.join(articles_folder, newspaper)
|
|
if not os.path.isdir(newspaper_path):
|
|
continue
|
|
|
|
for filename in os.listdir(newspaper_path):
|
|
file_path = os.path.join(newspaper_path, filename)
|
|
output_path = os.path.join("output", f"{filename}.json")
|
|
|
|
# Skip if already processed
|
|
if os.path.isfile(output_path):
|
|
print(f"Skipping already processed article: {filename}")
|
|
continue
|
|
|
|
if os.path.isfile(file_path):
|
|
try:
|
|
with open(file_path, 'r', encoding='utf-8') as f:
|
|
first_line = f.readline()
|
|
if first_line.startswith("SOURCE:"):
|
|
source = first_line[len("SOURCE:"):].strip()
|
|
content = f.read()
|
|
else:
|
|
source = "Unfiltered"
|
|
content = first_line + f.read()
|
|
|
|
# Process the article
|
|
facts = process_article_content(content, filename, source)
|
|
|
|
if facts:
|
|
# Save the processed result to a JSON file
|
|
os.makedirs("output", exist_ok=True)
|
|
with open(output_path, "w", encoding="utf-8") as f:
|
|
json.dump(facts, f, ensure_ascii=False, indent=2)
|
|
processed_count += 1
|
|
print(f"Processed and saved: {filename}")
|
|
else:
|
|
failed_count += 1
|
|
print(f"Failed to process: {filename}")
|
|
|
|
except Exception as e:
|
|
print(f"Error processing article {filename}: {e}")
|
|
failed_count += 1
|
|
|
|
print(f"Fact extraction complete. Processed: {processed_count}, Failed: {failed_count}")
|
|
|
|
# Run once when called directly
|
|
if __name__ == "__main__":
|
|
main_fact_extraction_loop()
|