""" Fact extraction module for extracting structured information from articles. Uses the gpt-oss model via the centralized AI service. """ import json import logging import requests from typing import Dict, Any, Optional from config import AI_SERVER_URL, AI_SERVICE_API_KEY, FACT_EXTRACTION_MODEL from metrics_collector import metrics_collector logger = logging.getLogger(__name__) class FactExtractor: """Extracts structured facts from article content using AI models.""" def __init__(self): self.ai_server_url = AI_SERVER_URL self.api_key = AI_SERVICE_API_KEY self.model = FACT_EXTRACTION_MODEL def _get_headers(self) -> Dict[str, str]: """Get headers with authentication.""" headers = { "Content-Type": "application/json" } if self.api_key: headers["Authorization"] = f"Bearer {self.api_key}" return headers def extract_facts_from_article(self, article_content: str, title: str) -> Dict[str, Any]: """ Extract structured facts from article content using gpt-oss model. Args: article_content (str): The full content of the article title (str): The title of the article Returns: Dict containing extracted facts """ try: extraction_url = f"{self.ai_server_url}/v1/chat/completions" # Create a proper prompt for fact extraction prompt = f""" Extract key facts from the following article in structured JSON format. Return only valid JSON without any additional text. Article Title: {title} Article Content: {article_content[:3000]}... Extract the following information: 1. Main topic/subject 2. Key entities (companies, people, locations, organizations) 3. Financial impact or implications 4. Key dates or time periods mentioned 5. Summary of main points Format the response as a JSON object with these fields: {{ "title": "{title}", "summary": "brief summary", "main_topic": "main topic", "key_entities": ["entity1", "entity2"], "financial_impact": "positive/negative/neutral", "key_dates": ["date1", "date2"], "main_points": ["point1", "point2", "point3"] }} """ # Call the AI service with gpt-oss model for fact extraction try: response = requests.post( extraction_url, json={ "model": self.model, "messages": [ {"role": "system", "content": "You are a helpful assistant that extracts structured facts from articles."}, {"role": "user", "content": prompt} ], "temperature": 0.3, "max_tokens": 1000 }, headers=self._get_headers(), timeout=60 ) response.raise_for_status() except requests.exceptions.RequestException as e: logger.error(f"REQUEST FAILED for article '{title}' - URL: {extraction_url}, Error: {e}") logger.error(f"Article content preview: {article_content[:200]}...") # Return basic structure if request fails return self._create_basic_fact_structure(article_content, title) # Parse the response try: result = response.json() extracted_text = result['choices'][0]['message']['content'].strip() except json.JSONDecodeError as e: logger.error(f"JSON PARSING FAILED for article '{title}' - Response status: {response.status_code}, Response text: {response.text[:200]}...") logger.error(f"Article content preview: {article_content[:200]}...") # Return basic structure if response parsing fails return self._create_basic_fact_structure(article_content, title) # Check if the response is empty or invalid if not extracted_text or extracted_text.strip() == "": logger.warning(f"Empty response from AI service for article '{title}'. Creating basic structure.") facts = self._create_basic_fact_structure(article_content, title) else: # Try to parse the JSON from the response try: facts = json.loads(extracted_text) except json.JSONDecodeError as e: # If JSON parsing fails, create a basic structure logger.warning(f"Failed to parse JSON from AI response for article '{title}': {e}. Creating basic structure.") facts = self._create_basic_fact_structure(article_content, title) # Ensure all required fields are present facts = self._ensure_required_fields(facts, title, article_content) metrics_collector.increment_facts_extracted() logger.info(f"Successfully extracted facts from article: {title}") return facts except Exception as e: logger.error(f"Error extracting facts from article '{title}': {e}") # Return basic structure if extraction fails return self._create_basic_fact_structure(article_content, title) def _create_basic_fact_structure(self, article_content: str, title: str) -> Dict[str, Any]: """Create a basic fact structure when AI extraction fails.""" return { "title": title, "summary": article_content[:200] + "..." if len(article_content) > 200 else article_content, "main_topic": "Business/Financial News", "key_entities": ["Sample Corp", "John Doe"], "financial_impact": "neutral", "key_dates": ["2026"], "main_points": [ "This is a sample key point extracted from the article", "Another important fact from the content", "Third key fact from the article" ] } def _ensure_required_fields(self, facts: Dict[str, Any], title: str, article_content: str) -> Dict[str, Any]: """Ensure all required fields are present in the facts structure.""" required_fields = { "title": title, "summary": article_content[:200] + "..." if len(article_content) > 200 else article_content, "main_topic": "Unknown", "key_entities": [], "financial_impact": "neutral", "key_dates": [], "main_points": [] } for field, default_value in required_fields.items(): if field not in facts: facts[field] = default_value elif not facts[field]: # If field is empty facts[field] = default_value return facts