Phase 2: ingest, embed, cluster
Pipeline: fetch -> normalise -> embed (CPU) -> cluster. `news poll` runs a cycle; `news sources` and `news clusters` inspect the result. Replay and the live service share run_cycle so tuning and production cannot diverge -- the old system's replay harness hardcoded default params and could never sweep the thresholds it existed to tune. Calibrated, not guessed: - Cluster threshold 0.82, from reading real output. 0.70 merged "Solheim Cup capsules" with "FIBA Women's World Cup Glance"; 0.82 gives clean groups (76 items on the AfD Saxony-Anhalt election, 27 on the Nepal tunnel rescue). The old system produced 515 clusters of which 514 were singletons. - Corroboration counts DISTINCT buckets, not feeds, so ten outlets reprinting one wire story is one confirmation rather than ten. Removed after measurement: a SimHash near-duplicate stage. It gave +1.2 bits of separation on titles and -0.2 with summaries -- news duplicates are paraphrases, which is the same lexical trap that sank the old Jaccard clustering. Sources: all 39 fetched against the live web, not assumed. Dropped CBC (every feed URL times out from this host) and Tech Policy Press (serves HTML at every documented feed path). Fixed WVXU (index.rss is an empty stub; local-news.rss works). Reddit moved to .rss because .json now 403s without OAuth, and its intervals raised because it 429s after three quick requests. Deutsche Welle is back: it ships RSS 1.0/RDF, which feedparser reads and the old hand-rolled quick-xml parser could not.
This commit is contained in:
connor committed
1 parent
c89a723603
commit
64690ecfe0
18 files changed
+1338
No files matched your search
@@ -0,0 +1,268 @@
|
||||
# Sources.
|
||||
#
|
||||
# `bucket` is the editorial-independence group, and it is the important field.
|
||||
# Importance counts DISTINCT buckets, never feeds: two BBC feeds are one bucket,
|
||||
# and ten outlets reprinting one wire story must not read as ten independent
|
||||
# confirmations. Getting this wrong is what made the old system fire priority-5
|
||||
# pushes on anything two feeds happened to carry.
|
||||
#
|
||||
# `prominence = 1` marks a publisher's front-page/top-stories feed. Subscribing
|
||||
# to both it and the section feed is a free editorial-promotion signal; the old
|
||||
# config dropped BBC's top-news feed and threw that away.
|
||||
|
||||
[defaults]
|
||||
poll_interval_secs = 900
|
||||
|
||||
# ------------------------------------------------------------- world / wire --
|
||||
[[source]]
|
||||
name = "Al Jazeera"
|
||||
url = "https://www.aljazeera.com/xml/rss/all.xml"
|
||||
bucket = "aljazeera"
|
||||
weight = 1.2
|
||||
|
||||
[[source]]
|
||||
name = "BBC Top Stories"
|
||||
url = "https://feeds.bbci.co.uk/news/rss.xml"
|
||||
bucket = "bbc"
|
||||
prominence = 1
|
||||
weight = 1.1
|
||||
|
||||
[[source]]
|
||||
name = "BBC World"
|
||||
url = "https://feeds.bbci.co.uk/news/world/rss.xml"
|
||||
bucket = "bbc"
|
||||
weight = 1.1
|
||||
|
||||
[[source]]
|
||||
name = "Guardian World"
|
||||
url = "https://www.theguardian.com/world/rss"
|
||||
bucket = "guardian"
|
||||
|
||||
[[source]]
|
||||
name = "NYT World"
|
||||
url = "https://rss.nytimes.com/services/xml/rss/nyt/World.xml"
|
||||
bucket = "nyt"
|
||||
|
||||
[[source]]
|
||||
name = "NPR News"
|
||||
url = "https://feeds.npr.org/1001/rss.xml"
|
||||
bucket = "npr"
|
||||
|
||||
[[source]]
|
||||
name = "France24"
|
||||
url = "https://www.france24.com/en/rss"
|
||||
bucket = "france24"
|
||||
|
||||
# RSS 1.0/RDF. The old hand-rolled quick-xml parser could not read this format
|
||||
# and DW was excluded with the note that it "would yield zero items silently".
|
||||
# feedparser handles RDF, so DW is back.
|
||||
[[source]]
|
||||
name = "Deutsche Welle"
|
||||
url = "https://rss.dw.com/rdf/rss-en-all"
|
||||
bucket = "dw"
|
||||
|
||||
[[source]]
|
||||
name = "ABC Australia"
|
||||
url = "https://www.abc.net.au/news/feed/51120/rss.xml"
|
||||
bucket = "abc-au"
|
||||
|
||||
[[source]]
|
||||
name = "PBS NewsHour"
|
||||
url = "https://www.pbs.org/newshour/feeds/rss/headlines"
|
||||
bucket = "pbs"
|
||||
poll_interval_secs = 1800
|
||||
|
||||
# --------------------------------------------------------------- mideast ----
|
||||
[[source]]
|
||||
name = "Middle East Eye"
|
||||
url = "https://www.middleeasteye.net/rss"
|
||||
bucket = "middle-east-eye"
|
||||
|
||||
[[source]]
|
||||
name = "+972 Magazine"
|
||||
url = "https://www.972mag.com/feed/"
|
||||
bucket = "972mag"
|
||||
poll_interval_secs = 3600
|
||||
|
||||
# ----------------------------------------------------------- tech / policy --
|
||||
[[source]]
|
||||
name = "Ars Technica"
|
||||
url = "https://feeds.arstechnica.com/arstechnica/index"
|
||||
bucket = "ars-technica"
|
||||
|
||||
[[source]]
|
||||
name = "EFF Deeplinks"
|
||||
url = "https://www.eff.org/rss/updates.xml"
|
||||
bucket = "eff"
|
||||
poll_interval_secs = 3600
|
||||
|
||||
# Global-South technology and economics; covers two interest blocks at once.
|
||||
[[source]]
|
||||
name = "Rest of World"
|
||||
url = "https://restofworld.org/feed/latest/"
|
||||
bucket = "rest-of-world"
|
||||
poll_interval_secs = 3600
|
||||
|
||||
# ------------------------------------------------------ software / systems --
|
||||
[[source]]
|
||||
name = "Hacker News"
|
||||
url = "https://hn.algolia.com/api/v1/search_by_date"
|
||||
kind = "hn"
|
||||
bucket = "hacker-news"
|
||||
poll_interval_secs = 1800
|
||||
|
||||
[[source]]
|
||||
name = "Lobsters"
|
||||
url = "https://lobste.rs/rss"
|
||||
bucket = "lobsters"
|
||||
poll_interval_secs = 1800
|
||||
|
||||
# Reddit's .json endpoints return 403 without OAuth as of 2026-09; the .rss
|
||||
# views work unauthenticated but 429 aggressively -- three requests in quick
|
||||
# succession from one IP were enough to get throttled during validation, and the
|
||||
# block persisted across serialized retries. Hence the long intervals here.
|
||||
# The fetcher treats 429 as a state to back off from, not an error.
|
||||
#
|
||||
# Cost of the .rss route: the Atom feed carries no score, so these contribute no
|
||||
# aggregator-maturity signal. Hacker News is the source that supplies it.
|
||||
[[source]]
|
||||
name = "r/LocalLLaMA"
|
||||
url = "https://www.reddit.com/r/LocalLLaMA/new/.rss"
|
||||
bucket = "reddit"
|
||||
poll_interval_secs = 5400
|
||||
|
||||
[[source]]
|
||||
name = "r/selfhosted"
|
||||
url = "https://www.reddit.com/r/selfhosted/new/.rss"
|
||||
bucket = "reddit"
|
||||
poll_interval_secs = 7200
|
||||
|
||||
[[source]]
|
||||
name = "r/rust"
|
||||
url = "https://www.reddit.com/r/rust/new/.rss"
|
||||
bucket = "reddit"
|
||||
poll_interval_secs = 7200
|
||||
|
||||
# Release feeds for the stack Connor actually runs. GitHub releases are Atom,
|
||||
# so they need no adapter of their own.
|
||||
[[source]]
|
||||
name = "llama.cpp releases"
|
||||
url = "https://github.com/ggml-org/llama.cpp/releases.atom"
|
||||
bucket = "github-releases"
|
||||
poll_interval_secs = 7200
|
||||
|
||||
[[source]]
|
||||
name = "LiteLLM releases"
|
||||
url = "https://github.com/BerriAI/litellm/releases.atom"
|
||||
bucket = "github-releases"
|
||||
poll_interval_secs = 7200
|
||||
|
||||
[[source]]
|
||||
name = "llama-swap releases"
|
||||
url = "https://github.com/mostlygeek/llama-swap/releases.atom"
|
||||
bucket = "github-releases"
|
||||
poll_interval_secs = 7200
|
||||
|
||||
[[source]]
|
||||
name = "Podman releases"
|
||||
url = "https://github.com/containers/podman/releases.atom"
|
||||
bucket = "github-releases"
|
||||
poll_interval_secs = 7200
|
||||
|
||||
[[source]]
|
||||
name = "Rust releases"
|
||||
url = "https://github.com/rust-lang/rust/releases.atom"
|
||||
bucket = "github-releases"
|
||||
poll_interval_secs = 7200
|
||||
|
||||
# arXiv announces once per weekday (~00:00 UTC); the feed is legitimately EMPTY
|
||||
# between announcement windows, which is not a fault. Verified 2026-09-08 that
|
||||
# the endpoint serves valid RSS -- cs.CL, cs.LG and cs.AI were all empty at the
|
||||
# time of checking, consistent with being between windows.
|
||||
[[source]]
|
||||
name = "arXiv cs.CL"
|
||||
url = "https://rss.arxiv.org/rss/cs.CL"
|
||||
bucket = "arxiv"
|
||||
poll_interval_secs = 21600
|
||||
|
||||
# ------------------------------------------------------------------ labor ---
|
||||
[[source]]
|
||||
name = "Labor Notes"
|
||||
url = "https://labornotes.org/feed"
|
||||
bucket = "labor-notes"
|
||||
poll_interval_secs = 3600
|
||||
|
||||
[[source]]
|
||||
name = "Jacobin"
|
||||
url = "https://jacobin.com/feed/"
|
||||
bucket = "jacobin"
|
||||
poll_interval_secs = 3600
|
||||
|
||||
# --------------------------------------------------------------- urbanism ---
|
||||
[[source]]
|
||||
name = "Streetsblog USA"
|
||||
url = "https://usa.streetsblog.org/feed"
|
||||
bucket = "streetsblog"
|
||||
poll_interval_secs = 3600
|
||||
|
||||
[[source]]
|
||||
name = "Strong Towns"
|
||||
url = "https://www.strongtowns.org/journal?format=rss"
|
||||
bucket = "strong-towns"
|
||||
poll_interval_secs = 7200
|
||||
|
||||
# --------------------------------------------------------- energy / climate --
|
||||
[[source]]
|
||||
name = "Canary Media"
|
||||
url = "https://www.canarymedia.com/rss"
|
||||
bucket = "canary-media"
|
||||
poll_interval_secs = 3600
|
||||
|
||||
# --------------------------------------------------------------- US / local --
|
||||
[[source]]
|
||||
name = "ProPublica"
|
||||
url = "https://www.propublica.org/feeds/propublica/main"
|
||||
bucket = "propublica"
|
||||
poll_interval_secs = 3600
|
||||
|
||||
[[source]]
|
||||
name = "NPR Politics"
|
||||
url = "https://feeds.npr.org/1014/rss.xml"
|
||||
bucket = "npr"
|
||||
|
||||
[[source]]
|
||||
name = "Ohio Capital Journal"
|
||||
url = "https://ohiocapitaljournal.com/feed/"
|
||||
bucket = "ohio-capital-journal"
|
||||
poll_interval_secs = 3600
|
||||
|
||||
[[source]]
|
||||
name = "WVXU Cincinnati"
|
||||
url = "https://www.wvxu.org/local-news.rss"
|
||||
bucket = "wvxu"
|
||||
poll_interval_secs = 3600
|
||||
|
||||
[[source]]
|
||||
name = "WCPO Cincinnati"
|
||||
url = "https://www.wcpo.com/news.rss"
|
||||
bucket = "wcpo"
|
||||
poll_interval_secs = 3600
|
||||
|
||||
[[source]]
|
||||
name = "Kentucky Lantern"
|
||||
url = "https://kentuckylantern.com/feed/"
|
||||
bucket = "kentucky-lantern"
|
||||
poll_interval_secs = 3600
|
||||
|
||||
# ------------------------------------------------------------ italy / mexico --
|
||||
[[source]]
|
||||
name = "ANSA English"
|
||||
url = "https://www.ansa.it/english/english_rss.xml"
|
||||
bucket = "ansa"
|
||||
poll_interval_secs = 3600
|
||||
|
||||
[[source]]
|
||||
name = "Mexico News Daily"
|
||||
url = "https://mexiconewsdaily.com/feed/"
|
||||
bucket = "mexico-news-daily"
|
||||
poll_interval_secs = 3600
|
||||
Reference in new issue
Block a user