Add webhook functionality to send new articles to external endpoint

scraper/cron_scraper.py

@@ -10,6 +10,7 @@ import json
 import feedparser
 import time
 import os
+import requests
 from selenium import webdriver
 from selenium.webdriver.firefox.options import Options as FirefoxOptions
 from selenium.webdriver.common.by import By
@@ -286,6 +287,58 @@ def safe_pull_articles(article_list):
     return results, errors
 
 
+def gather_new_articles():
+    """
+    Gather list of all newly downloaded articles and format them for webhook.
+    """
+    new_articles = []
+    
+    # Walk through all article directories
+    for root, dirs, files in os.walk("articles"):
+        for file in files:
+            if file != "processed_articles_cache.json":  # Skip cache file
+                # Get the full file path
+                file_path = os.path.join(root, file)
+                
+                # Get the outlet name from the directory path
+                outlet = os.path.basename(root)
+                
+                # Create the relative path for the article
+                relative_path = os.path.relpath(file_path, "scraper")
+                
+                # Create article data structure
+                article_data = {
+                    "created_at": time.strftime("%Y-%m-%dT%H:%M:%S.%f", time.localtime(os.path.getctime(file_path))),
+                    "name": file,
+                    "outlet": outlet,
+                    "path": f"../{relative_path}"
+                }
+                
+                new_articles.append(article_data)
+    
+    return new_articles
+
+
+def send_to_webhook(articles):
+    """
+    Send list of articles to the webhook URL.
+    """
+    webhook_url = "http://agents.example.com/webhook-test/49c5b169-c68c-4f8c-90c2-0fcca6e2d387"
+    headers = {
+        "StockDocsN8NAuthToken": "ganvT4gsgRjWpGE8FMw9uCzFjZrTx8RZCoVm2Dh7skbZecov"
+    }
+    
+    try:
+        response = requests.post(webhook_url, json=articles, headers=headers, timeout=30)
+        if response.status_code == 200:
+            print(f"Successfully sent {len(articles)} articles to webhook")
+        else:
+            print(f"Webhook request failed with status code: {response.status_code}")
+            print(f"Response: {response.text}")
+    except Exception as e:
+        print(f"Error sending to webhook: {e}")
+
+
 def main():
     """
     Main scraping function for cron execution.
@@ -334,6 +387,13 @@ def main():
                 f.write(result + "\n")
         print("All articles pulled successfully.")
 
+        # Gather and send new articles to webhook
+        new_articles = gather_new_articles()
+        if new_articles:
+            send_to_webhook(new_articles)
+        else:
+            print("No new articles to send to webhook")
+
     except Exception as e:
         print(f"Major error in main execution: {e}")
         import traceback
@@ -344,4 +404,4 @@ def main():
 
 
 if __name__ == "__main__":
-    main()
+    main()