StockDocs/scraper/rss_feeds.json
Jarian Cottingham 6b8a87f9ef fix: close #1 - add bot-detection evasion for Reuters scraping
- Add rotating User-Agent pool (5 browser profiles) to all scrapers
- Fix Playwright: use browser context for UA instead of broken set_extra_http_headers
- Fix Selenium: set general.useragent.override preference
- Fix newspaper4k: pass browser_user_agent to bypass bot detection
- Add random delays (0.5-2s pre-fetch, 1-4s post-load) to mimic human behavior
- Switch Reuters RSS from Google News proxy to official Reuters agency feed
- Add legitimacy headers (Accept, Accept-Language, Connection) to Playwright
- Apply all fixes to both scraper.py and cron_scraper.py
2026-07-05 05:21:48 +00:00

197 lines
6.6 KiB
JSON
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

{
"rss_feeds": {
"Reuters Business News": {
"source_website": "reuters.com",
"rss_url": "https://www.reutersagency.com/feed/"
},
"Associated Press Business": {
"source_website": "apnews.com",
"rss_url": "https://rsshub.app/apnews/topics/apf-topnews"
},
"Financial Times": {
"source_website": "ft.com",
"rss_url": "https://www.ft.com/rss/home"
},
"Fortune Top Stories": {
"source_website": "fortune.com",
"rss_url": "https://fortune.com/feed/fortune-feeds/?id=3230629"
},
"Seeking Alpha Market News": {
"source_website": "seekingalpha.com",
"rss_url": "https://seekingalpha.com/feed.xml"
},
"The Motley Fool Stock News & Analysis": {
"source_website": "fool.com",
"rss_url": "https://www.fool.com/a/feeds/partner/googlechromefollow?apikey=5e092c1f-c5f9-4428-9219-908a47d2e2de"
},
"Business Standard Latest News": {
"source_website": "business-standard.com",
"rss_url": "https://www.business-standard.com/rss/latest.rss"
},
"TheStreet Full Articles": {
"source_website": "thestreet.com",
"rss_url": "https://www.thestreet.com/.rss/full"
},
"Benzinga Financial News": {
"source_website": "benzinga.com",
"rss_url": "https://feeds.benzinga.com/benzinga"
},
"MarketBeat Market News": {
"source_website": "marketbeat.com",
"rss_url": "https://www.marketbeat.com/feed/"
},
"Money (Time) Personal Finance": {
"source_website": "money.com",
"rss_url": "https://money.com/money/feed/"
},
"Global Finance Magazine": {
"source_website": "gfmag.com",
"rss_url": "https://www.gfmag.com/feed"
},
"Financial Samurai": {
"source_website": "financialsamurai.com",
"rss_url": "https://www.financialsamurai.com/feed/"
},
"MoneyWeek": {
"source_website": "moneyweek.com",
"rss_url": "https://moneyweek.com/feed/all"
},
"Finance Monthly": {
"source_website": "finance-monthly.com",
"rss_url": "https://www.finance-monthly.com/feed/"
},
"European Financial Review": {
"source_website": "europeanfinancialreview.com",
"rss_url": "https://www.europeanfinancialreview.com/feed"
},
"Money Morning": {
"source_website": "moneymorning.com",
"rss_url": "https://moneymorning.com/feed"
},
"Dealbreaker": {
"source_website": "dealbreaker.com",
"rss_url": "https://dealbreaker.com/.rss/full"
},
"World Finance": {
"source_website": "worldfinance.com",
"rss_url": "https://www.worldfinance.com/feed"
},
"Fox Business Headlines": {
"source_website": "foxbusiness.com",
"rss_url": "https://moxie.foxbusiness.com/google-publisher/latest.xml"
},
"FinanceAsia": {
"source_website": "financeasia.com",
"rss_url": "https://www.financeasia.com/rss/latest"
},
"CNBC Business": {
"source_website": "cnbc.com",
"rss_url": "https://www.cnbc.com/id/100003114/device/rss/rss.html"
},
"CNN Money": {
"source_website": "cnn.com",
"rss_url": "http://rss.cnn.com/rss/money_topstories.rss"
},
"Markets Insider": {
"source_website": "markets.businessinsider.com",
"rss_url": "https://markets.businessinsider.com/rss/news"
},
"The Economist Business & Finance": {
"source_website": "economist.com",
"rss_url": "https://www.economist.com/business/rss.xml"
},
"Barchart News": {
"source_website": "barchart.com",
"rss_url": "http://feeds.feedburner.com/BarchartNews"
},
"The Guardian Business": {
"source_website": "theguardian.com",
"rss_url": "http://feeds.theguardian.com/theguardian/uk/business/rss"
},
"Economy Watch": {
"source_website": "economywatch.com",
"rss_url": "https://www.economywatch.com/feed"
},
"CFI.co": {
"source_website": "cfi.co",
"rss_url": "https://cfi.co/feed"
},
"BBC News Business": {
"source_website": "bbc.co.uk",
"rss_url": "http://feeds.bbci.co.uk/news/business/rss.xml"
},
"Investors Business Daily": {
"source_website": "investors.com",
"rss_url": "https://www.investors.com/feed/"
},
"Forbes Real-Time": {
"source_website": "forbes.com",
"rss_url": "https://www.forbes.com/real-time/feed2/"
},
"The Financial Express": {
"source_website": "financialexpress.com",
"rss_url": "https://www.financialexpress.com/feed/"
},
"MarketWatch Top Stories": {
"source_website": "marketwatch.com",
"rss_url": "http://feeds.marketwatch.com/marketwatch/topstories/"
},
"Wall Street Journal U.S. Business": {
"source_website": "wsj.com",
"rss_url": "https://feeds.a.dj.com/rss/WSJcomUSBusiness.xml"
},
"Sky News Business": {
"source_website": "news.sky.com",
"rss_url": "http://news.sky.com/feeds/rss/business.xml"
},
"Bloomberg Surveillance Podcast": {
"source_website": "bloomberg.com",
"rss_url": "https://www.bloomberg.com/feed/podcast/bloomberg-surveillance.xml"
},
"Barrons Markets": {
"source_website": "barrons.com",
"rss_url": "https://www.barrons.com/xml/rss/markets.xml"
},
"Yahoo Finance": {
"source_website": "finance.yahoo.com",
"rss_url": "https://www.yahoo.com/news/rss/finance"
},
"Investing.com News": {
"source_website": "investing.com",
"rss_url": "https://www.investing.com/rss/news.rss"
},
"Investopedia Headlines": {
"source_website": "investopedia.com",
"rss_url": "https://www.investopedia.com/feedbuilder/feed/getfeed/?feedName=rss_headline"
},
"NerdWallet Finance": {
"source_website": "nerdwallet.com",
"rss_url": "https://www.nerdwallet.com/news/finance/feed"
},
"Newsmax Finance": {
"source_website": "newsmax.com",
"rss_url": "https://www.newsmax.com/rss/finance"
},
"Bankrate News": {
"source_website": "bankrate.com",
"rss_url": "https://www.bankrate.com/rss/"
},
"Morningstar Articles": {
"source_website": "morningstar.com",
"rss_url": "https://www.morningstar.com/articles.rss"
},
"Kiplinger": {
"source_website": "kiplinger.com",
"rss_url": "https://www.kiplinger.com/kiplinger.rss"
},
"International Business Times": {
"source_website": "ibtimes.com",
"rss_url": "https://www.ibtimes.com/rss"
},
"Policygenius News": {
"source_website": "policygenius.com",
"rss_url": "https://www.policygenius.com/news/feed/"
}
}
}