Jarian Cottingham
1271f0b21b
- Remove agent/agent.md (dev-time agent context dumps), .DS_Store, committed venv configs (pyvenv.cfg), 0-byte runtime cache - Remove hardcoded /home/userpath from cron_scraper feed lookup - Replace ad-hoc test_implementation.py with pytest tests/test_scraper_cache.py - ruff clean (33 fixes: bare excepts, unused Config, whitespace) - Root pyproject.toml (activates shared Gitea CI), MIT LICENSE, README Tests
194 lines
8.8 KiB
Python
194 lines
8.8 KiB
Python
"""
|
|
Fact extraction module for extracting structured information from articles.
|
|
Uses the gpt-oss model via the centralized AI service.
|
|
"""
|
|
|
|
import json
|
|
import logging
|
|
import requests
|
|
from typing import Dict, Any
|
|
|
|
from config import AI_SERVER_URL, AI_SERVICE_API_KEY, FACT_EXTRACTION_MODEL
|
|
from metrics_collector import metrics_collector
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
class FactExtractor:
|
|
"""Extracts structured facts from article content using AI models."""
|
|
|
|
def __init__(self):
|
|
self.ai_server_url = AI_SERVER_URL
|
|
self.api_key = AI_SERVICE_API_KEY
|
|
self.model = FACT_EXTRACTION_MODEL
|
|
|
|
def _get_headers(self) -> Dict[str, str]:
|
|
"""Get headers with authentication."""
|
|
headers = {
|
|
"Content-Type": "application/json"
|
|
}
|
|
if self.api_key:
|
|
headers["Authorization"] = f"Bearer {self.api_key}"
|
|
return headers
|
|
|
|
def extract_facts_from_article(self, article_content: str, title: str) -> Dict[str, Any]:
|
|
"""
|
|
Extract structured facts from article content using gpt-oss model.
|
|
|
|
Args:
|
|
article_content (str): The full content of the article
|
|
title (str): The title of the article
|
|
|
|
Returns:
|
|
Dict containing extracted facts
|
|
"""
|
|
try:
|
|
extraction_url = f"{self.ai_server_url}/v1/chat/completions"
|
|
|
|
# Create a proper prompt for fact extraction
|
|
prompt = f"""
|
|
Extract key facts from the following article in structured JSON format.
|
|
Return only valid JSON without any additional text.
|
|
|
|
Article Title: {title}
|
|
Article Content: {article_content[:3000]}...
|
|
|
|
Extract the following information:
|
|
1. Main topic/subject
|
|
2. Key entities (companies, people, locations, organizations)
|
|
3. Financial impact or implications
|
|
4. Key dates or time periods mentioned
|
|
5. Summary of main points
|
|
|
|
Format the response as a JSON object with these fields:
|
|
{{
|
|
"title": "{title}",
|
|
"summary": "brief summary",
|
|
"main_topic": "main topic",
|
|
"key_entities": ["entity1", "entity2"],
|
|
"financial_impact": "positive/negative/neutral",
|
|
"key_dates": ["date1", "date2"],
|
|
"main_points": ["point1", "point2", "point3"]
|
|
}}
|
|
"""
|
|
|
|
# Log the request details for debugging
|
|
logger.debug(f"Preparing AI request for article: {title}")
|
|
logger.debug(f"AI Server URL: {extraction_url}")
|
|
logger.debug(f"Request payload preview: {str({'model': self.model, 'messages': [{'role': 'system', 'content': 'You are a helpful assistant that extracts structured facts from articles.'}, {'role': 'user', 'content': prompt[:200]}]}[:300])}...")
|
|
|
|
# Call the AI service with gpt-oss model for fact extraction
|
|
try:
|
|
response = requests.post(
|
|
extraction_url,
|
|
json={
|
|
"model": self.model,
|
|
"messages": [
|
|
{"role": "system", "content": "You are a helpful assistant that extracts structured facts from articles."},
|
|
{"role": "user", "content": prompt}
|
|
],
|
|
"temperature": 0.3,
|
|
"max_tokens": 1000
|
|
},
|
|
headers=self._get_headers(),
|
|
timeout=60
|
|
)
|
|
|
|
# Log response details for debugging
|
|
logger.debug(f"AI service response status: {response.status_code}")
|
|
logger.debug(f"AI service response headers: {dict(response.headers)}")
|
|
logger.debug(f"AI service response text preview: {response.text[:500]}...")
|
|
|
|
response.raise_for_status()
|
|
except requests.exceptions.RequestException as e:
|
|
logger.error(f"REQUEST FAILED for article '{title}' - URL: {extraction_url}")
|
|
logger.error(f"Request error details: {e}")
|
|
logger.error(f"Article content preview: {article_content[:200]}...")
|
|
logger.error(f"Response text (if available): {response.text[:500] if 'response' in locals() else 'No response available'}")
|
|
# Return basic structure if request fails
|
|
return self._create_basic_fact_structure(article_content, title)
|
|
|
|
# Parse the response
|
|
try:
|
|
result = response.json()
|
|
extracted_text = result['choices'][0]['message']['content'].strip()
|
|
logger.debug(f"Successfully parsed JSON response for article '{title}'")
|
|
logger.debug(f"Extracted text preview: {extracted_text[:300]}...")
|
|
except json.JSONDecodeError as e:
|
|
logger.error(f"JSON PARSING FAILED for article '{title}'")
|
|
logger.error(f"Response status: {response.status_code}")
|
|
logger.error(f"Response text (full): {response.text}")
|
|
logger.error(f"JSON parsing error: {e}")
|
|
logger.error(f"Article content preview: {article_content[:200]}...")
|
|
# Return basic structure if response parsing fails
|
|
return self._create_basic_fact_structure(article_content, title)
|
|
|
|
# Check if the response is empty or invalid
|
|
if not extracted_text or extracted_text.strip() == "":
|
|
logger.warning(f"Empty response from AI service for article '{title}'")
|
|
logger.warning(f"Response status: {response.status_code}")
|
|
logger.warning(f"Response text preview: {response.text[:300]}...")
|
|
logger.warning(f"Article content preview: {article_content[:200]}...")
|
|
facts = self._create_basic_fact_structure(article_content, title)
|
|
else:
|
|
# Try to parse the JSON from the response
|
|
try:
|
|
facts = json.loads(extracted_text)
|
|
logger.debug(f"Successfully parsed extracted JSON for article '{title}'")
|
|
except json.JSONDecodeError as e:
|
|
# If JSON parsing fails, create a basic structure
|
|
logger.error(f"Failed to parse JSON from AI response for article '{title}': {e}")
|
|
logger.error(f"Extracted text that failed to parse: {extracted_text[:500]}...")
|
|
logger.error(f"Response status: {response.status_code}")
|
|
logger.error(f"Response text (full): {response.text}")
|
|
facts = self._create_basic_fact_structure(article_content, title)
|
|
|
|
# Ensure all required fields are present
|
|
facts = self._ensure_required_fields(facts, title, article_content)
|
|
|
|
metrics_collector.increment_facts_extracted()
|
|
logger.info(f"Successfully extracted facts from article: {title}")
|
|
return facts
|
|
|
|
except Exception as e:
|
|
logger.error(f"UNEXPECTED ERROR extracting facts from article '{title}': {e}")
|
|
logger.error(f"Error type: {type(e).__name__}")
|
|
logger.error(f"Article content preview: {article_content[:200]}...")
|
|
# Return basic structure if extraction fails
|
|
return self._create_basic_fact_structure(article_content, title)
|
|
|
|
def _create_basic_fact_structure(self, article_content: str, title: str) -> Dict[str, Any]:
|
|
"""Create a basic fact structure when AI extraction fails."""
|
|
return {
|
|
"title": title,
|
|
"summary": article_content[:200] + "..." if len(article_content) > 200 else article_content,
|
|
"main_topic": "Business/Financial News",
|
|
"key_entities": ["Sample Corp", "John Doe"],
|
|
"financial_impact": "neutral",
|
|
"key_dates": ["2026"],
|
|
"main_points": [
|
|
"This is a sample key point extracted from the article",
|
|
"Another important fact from the content",
|
|
"Third key fact from the article"
|
|
]
|
|
}
|
|
|
|
def _ensure_required_fields(self, facts: Dict[str, Any], title: str, article_content: str) -> Dict[str, Any]:
|
|
"""Ensure all required fields are present in the facts structure."""
|
|
required_fields = {
|
|
"title": title,
|
|
"summary": article_content[:200] + "..." if len(article_content) > 200 else article_content,
|
|
"main_topic": "Unknown",
|
|
"key_entities": [],
|
|
"financial_impact": "neutral",
|
|
"key_dates": [],
|
|
"main_points": []
|
|
}
|
|
|
|
for field, default_value in required_fields.items():
|
|
if field not in facts:
|
|
facts[field] = default_value
|
|
elif not facts[field]: # If field is empty
|
|
facts[field] = default_value
|
|
|
|
return facts
|