- Add rotating User-Agent pool (5 browser profiles) to all scrapers - Fix Playwright: use browser context for UA instead of broken set_extra_http_headers - Fix Selenium: set general.useragent.override preference - Fix newspaper4k: pass browser_user_agent to bypass bot detection - Add random delays (0.5-2s pre-fetch, 1-4s post-load) to mimic human behavior - Switch Reuters RSS from Google News proxy to official Reuters agency feed - Add legitimacy headers (Accept, Accept-Language, Connection) to Playwright - Apply all fixes to both scraper.py and cron_scraper.py
197 lines
6.6 KiB
JSON
197 lines
6.6 KiB
JSON
{
|
||
"rss_feeds": {
|
||
"Reuters – Business News": {
|
||
"source_website": "reuters.com",
|
||
"rss_url": "https://www.reutersagency.com/feed/"
|
||
},
|
||
"Associated Press – Business": {
|
||
"source_website": "apnews.com",
|
||
"rss_url": "https://rsshub.app/apnews/topics/apf-topnews"
|
||
},
|
||
"Financial Times": {
|
||
"source_website": "ft.com",
|
||
"rss_url": "https://www.ft.com/rss/home"
|
||
},
|
||
"Fortune – Top Stories": {
|
||
"source_website": "fortune.com",
|
||
"rss_url": "https://fortune.com/feed/fortune-feeds/?id=3230629"
|
||
},
|
||
"Seeking Alpha – Market News": {
|
||
"source_website": "seekingalpha.com",
|
||
"rss_url": "https://seekingalpha.com/feed.xml"
|
||
},
|
||
"The Motley Fool – Stock News & Analysis": {
|
||
"source_website": "fool.com",
|
||
"rss_url": "https://www.fool.com/a/feeds/partner/googlechromefollow?apikey=5e092c1f-c5f9-4428-9219-908a47d2e2de"
|
||
},
|
||
"Business Standard – Latest News": {
|
||
"source_website": "business-standard.com",
|
||
"rss_url": "https://www.business-standard.com/rss/latest.rss"
|
||
},
|
||
"TheStreet – Full Articles": {
|
||
"source_website": "thestreet.com",
|
||
"rss_url": "https://www.thestreet.com/.rss/full"
|
||
},
|
||
"Benzinga – Financial News": {
|
||
"source_website": "benzinga.com",
|
||
"rss_url": "https://feeds.benzinga.com/benzinga"
|
||
},
|
||
"MarketBeat – Market News": {
|
||
"source_website": "marketbeat.com",
|
||
"rss_url": "https://www.marketbeat.com/feed/"
|
||
},
|
||
"Money (Time) – Personal Finance": {
|
||
"source_website": "money.com",
|
||
"rss_url": "https://money.com/money/feed/"
|
||
},
|
||
"Global Finance Magazine": {
|
||
"source_website": "gfmag.com",
|
||
"rss_url": "https://www.gfmag.com/feed"
|
||
},
|
||
"Financial Samurai": {
|
||
"source_website": "financialsamurai.com",
|
||
"rss_url": "https://www.financialsamurai.com/feed/"
|
||
},
|
||
"MoneyWeek": {
|
||
"source_website": "moneyweek.com",
|
||
"rss_url": "https://moneyweek.com/feed/all"
|
||
},
|
||
"Finance Monthly": {
|
||
"source_website": "finance-monthly.com",
|
||
"rss_url": "https://www.finance-monthly.com/feed/"
|
||
},
|
||
"European Financial Review": {
|
||
"source_website": "europeanfinancialreview.com",
|
||
"rss_url": "https://www.europeanfinancialreview.com/feed"
|
||
},
|
||
"Money Morning": {
|
||
"source_website": "moneymorning.com",
|
||
"rss_url": "https://moneymorning.com/feed"
|
||
},
|
||
"Dealbreaker": {
|
||
"source_website": "dealbreaker.com",
|
||
"rss_url": "https://dealbreaker.com/.rss/full"
|
||
},
|
||
"World Finance": {
|
||
"source_website": "worldfinance.com",
|
||
"rss_url": "https://www.worldfinance.com/feed"
|
||
},
|
||
"Fox Business – Headlines": {
|
||
"source_website": "foxbusiness.com",
|
||
"rss_url": "https://moxie.foxbusiness.com/google-publisher/latest.xml"
|
||
},
|
||
"FinanceAsia": {
|
||
"source_website": "financeasia.com",
|
||
"rss_url": "https://www.financeasia.com/rss/latest"
|
||
},
|
||
"CNBC – Business": {
|
||
"source_website": "cnbc.com",
|
||
"rss_url": "https://www.cnbc.com/id/100003114/device/rss/rss.html"
|
||
},
|
||
"CNN Money": {
|
||
"source_website": "cnn.com",
|
||
"rss_url": "http://rss.cnn.com/rss/money_topstories.rss"
|
||
},
|
||
"Markets Insider": {
|
||
"source_website": "markets.businessinsider.com",
|
||
"rss_url": "https://markets.businessinsider.com/rss/news"
|
||
},
|
||
"The Economist – Business & Finance": {
|
||
"source_website": "economist.com",
|
||
"rss_url": "https://www.economist.com/business/rss.xml"
|
||
},
|
||
"Barchart News": {
|
||
"source_website": "barchart.com",
|
||
"rss_url": "http://feeds.feedburner.com/BarchartNews"
|
||
},
|
||
"The Guardian – Business": {
|
||
"source_website": "theguardian.com",
|
||
"rss_url": "http://feeds.theguardian.com/theguardian/uk/business/rss"
|
||
},
|
||
"Economy Watch": {
|
||
"source_website": "economywatch.com",
|
||
"rss_url": "https://www.economywatch.com/feed"
|
||
},
|
||
"CFI.co": {
|
||
"source_website": "cfi.co",
|
||
"rss_url": "https://cfi.co/feed"
|
||
},
|
||
"BBC News – Business": {
|
||
"source_website": "bbc.co.uk",
|
||
"rss_url": "http://feeds.bbci.co.uk/news/business/rss.xml"
|
||
},
|
||
"Investor’s Business Daily": {
|
||
"source_website": "investors.com",
|
||
"rss_url": "https://www.investors.com/feed/"
|
||
},
|
||
"Forbes – Real-Time": {
|
||
"source_website": "forbes.com",
|
||
"rss_url": "https://www.forbes.com/real-time/feed2/"
|
||
},
|
||
"The Financial Express": {
|
||
"source_website": "financialexpress.com",
|
||
"rss_url": "https://www.financialexpress.com/feed/"
|
||
},
|
||
"MarketWatch – Top Stories": {
|
||
"source_website": "marketwatch.com",
|
||
"rss_url": "http://feeds.marketwatch.com/marketwatch/topstories/"
|
||
},
|
||
"Wall Street Journal – U.S. Business": {
|
||
"source_website": "wsj.com",
|
||
"rss_url": "https://feeds.a.dj.com/rss/WSJcomUSBusiness.xml"
|
||
},
|
||
"Sky News – Business": {
|
||
"source_website": "news.sky.com",
|
||
"rss_url": "http://news.sky.com/feeds/rss/business.xml"
|
||
},
|
||
"Bloomberg – Surveillance Podcast": {
|
||
"source_website": "bloomberg.com",
|
||
"rss_url": "https://www.bloomberg.com/feed/podcast/bloomberg-surveillance.xml"
|
||
},
|
||
"Barron’s – Markets": {
|
||
"source_website": "barrons.com",
|
||
"rss_url": "https://www.barrons.com/xml/rss/markets.xml"
|
||
},
|
||
"Yahoo Finance": {
|
||
"source_website": "finance.yahoo.com",
|
||
"rss_url": "https://www.yahoo.com/news/rss/finance"
|
||
},
|
||
"Investing.com – News": {
|
||
"source_website": "investing.com",
|
||
"rss_url": "https://www.investing.com/rss/news.rss"
|
||
},
|
||
"Investopedia – Headlines": {
|
||
"source_website": "investopedia.com",
|
||
"rss_url": "https://www.investopedia.com/feedbuilder/feed/getfeed/?feedName=rss_headline"
|
||
},
|
||
"NerdWallet – Finance": {
|
||
"source_website": "nerdwallet.com",
|
||
"rss_url": "https://www.nerdwallet.com/news/finance/feed"
|
||
},
|
||
"Newsmax Finance": {
|
||
"source_website": "newsmax.com",
|
||
"rss_url": "https://www.newsmax.com/rss/finance"
|
||
},
|
||
"Bankrate – News": {
|
||
"source_website": "bankrate.com",
|
||
"rss_url": "https://www.bankrate.com/rss/"
|
||
},
|
||
"Morningstar – Articles": {
|
||
"source_website": "morningstar.com",
|
||
"rss_url": "https://www.morningstar.com/articles.rss"
|
||
},
|
||
"Kiplinger": {
|
||
"source_website": "kiplinger.com",
|
||
"rss_url": "https://www.kiplinger.com/kiplinger.rss"
|
||
},
|
||
"International Business Times": {
|
||
"source_website": "ibtimes.com",
|
||
"rss_url": "https://www.ibtimes.com/rss"
|
||
},
|
||
"Policygenius – News": {
|
||
"source_website": "policygenius.com",
|
||
"rss_url": "https://www.policygenius.com/news/feed/"
|
||
}
|
||
}
|
||
}
|