diff --git a/news/cluster.py b/news/cluster.py index df51d89..4f4098e 100644 --- a/news/cluster.py +++ b/news/cluster.py @@ -62,6 +62,23 @@ def assign( out: list[Assignment] = [] ordered = sorted(items, key=lambda r: r["published_at"]) + # Centroids are held in memory for the whole batch rather than re-queried + # and re-deserialised per item. The naive version issued one SELECT and + # rebuilt a numpy matrix for every item, which took ~15 minutes for 1,258 + # items; this keeps a single matrix and appends to it. + warm: list[dict] = [] # {id, vec, size, first, last} + for row in con.execute( + """SELECT id, centroid, size, first_seen_at, last_seen_at FROM clusters + WHERE last_seen_at >= ?""", + ((_parse(ordered[0]["published_at"]) - timedelta(hours=window_hours)).isoformat(),) + if ordered else ("",), + ): + vec = store.from_blob(row["centroid"]) + warm.append({"id": row["id"], "vec": vec.astype(np.float32), "size": row["size"], + "first": row["first_seen_at"], "last": row["last_seen_at"]}) + + dirty: dict[str, dict] = {} + for item in ordered: vec = vectors.get(item["content_hash"]) if vec is None: @@ -76,50 +93,37 @@ def assign( if existing is not None: out.append(Assignment(item["id"], existing["cluster_id"], False, 1.0)) continue + published = _parse(item["published_at"]) - cutoff = (published - timedelta(hours=window_hours)).isoformat() + lo = (published - timedelta(hours=window_hours)).isoformat() + hi = (published + timedelta(hours=2)).isoformat() - # Only clusters still warm can absorb the item. A 2h grace on the upper - # bound lets a late-arriving report join a story it predates. - rows = con.execute( - """SELECT id, centroid, size FROM clusters - WHERE last_seen_at >= ? AND first_seen_at <= ?""", - (cutoff, (published + timedelta(hours=2)).isoformat()), - ).fetchall() - - best_id, best_sim = None, -1.0 - if rows: - mat = np.stack([store.from_blob(r["centroid"]) for r in rows]) + candidates = [c for c in warm if c["last"] >= lo and c["first"] <= hi] + best, best_sim = None, -1.0 + if candidates: + mat = np.stack([c["vec"] for c in candidates]) mat /= np.maximum(np.linalg.norm(mat, axis=1, keepdims=True), 1e-12) sims = mat @ vec k = int(np.argmax(sims)) - best_id, best_sim = rows[k]["id"], float(sims[k]) + best, best_sim = candidates[k], float(sims[k]) - if best_id is not None and best_sim >= threshold: - row = next(r for r in rows if r["id"] == best_id) - size = row["size"] - centroid = (store.from_blob(row["centroid"]) * size + vec) / (size + 1) - con.execute( - """UPDATE clusters SET centroid=?, size=?, last_seen_at=max(last_seen_at,?) - WHERE id=?""", - (store.to_blob(centroid), size + 1, item["published_at"], best_id), - ) - cluster_id, created = best_id, False + if best is not None and best_sim >= threshold: + best["vec"] = (best["vec"] * best["size"] + vec) / (best["size"] + 1) + best["size"] += 1 + best["last"] = max(best["last"], item["published_at"]) + cluster_id, created = best["id"], False + dirty[cluster_id] = best else: cluster_id = stable_id("cluster", item["id"]) con.execute( """INSERT OR IGNORE INTO clusters (id,centroid,size,first_seen_at,last_seen_at,lead_item_id) VALUES (?,?,?,?,?,?)""", - ( - cluster_id, - store.to_blob(vec), - 1, - item["published_at"], - item["published_at"], - item["id"], - ), + (cluster_id, store.to_blob(vec), 1, + item["published_at"], item["published_at"], item["id"]), ) + warm.append({"id": cluster_id, "vec": vec.astype(np.float32), "size": 1, + "first": item["published_at"], "last": item["published_at"]}) created = True con.execute( @@ -128,6 +132,13 @@ def assign( (cluster_id, item["id"]), ) out.append(Assignment(item["id"], cluster_id, created, best_sim)) + + # One write per touched cluster at the end, instead of one per absorbed item. + for cid, c in dirty.items(): + con.execute( + "UPDATE clusters SET centroid=?, size=?, last_seen_at=? WHERE id=?", + (store.to_blob(c["vec"]), c["size"], c["last"], cid), + ) return out diff --git a/profile/profile.md b/profile/profile.md index 12fe061..406da2a 100644 --- a/profile/profile.md +++ b/profile/profile.md @@ -1,108 +1,139 @@ # Connor's news profile -The thing the previous system never had. `news_triage` v0.2's entire model of taste was -three keywords and two blocked substrings; everything else was ranking machinery with -nothing meaningful to rank. This file is the replacement, and `profile.toml` is its -machine-readable form. +The thing the previous system never had. `news_triage` v0.2's entire model of taste +was three keywords and two blocked substrings; everything else was ranking +machinery with nothing meaningful to rank. This file is the replacement, and +`profile.toml` is its machine-readable form. -Written 2026-09-08, from a structured interview plus evidence from 32 real feedback -clicks in the v0.2 database. +Written 2026-09-08 from a structured interview plus evidence from 40 real feedback +clicks in the v0.2 database. Schema v2. ## How this is used -Every `description` and `probe` below is embedded (bge-small-en-v1.5, CPU) into a vector. -Item relevance is the **max** cosine similarity across probes, never the mean — averaging -"Middle East geopolitics" with "grid-scale energy storage" produces a vector near neither. -Feedback then moves these vectors over time (Rocchio); the written text is the cold-start -seed so the system is useful on day one instead of after 200 clicks. +Every `description` and `probe` is embedded (bge-small-en-v1.5, CPU) into a vector. +An item's score for a block is the **max** cosine across that block's probes, never +the mean — averaging "Middle East geopolitics" with "grid-scale energy storage" +gives a vector near neither. Blocks are then compared by **z-score within block**, +because raw cosine is not comparable across blocks; see `CALIBRATION.md`. -Vetoes work the same way, as *regions* rather than strings. This is the whole point: the old -system could not express "sports" as a concept, only as the literal characters `s-p-o-r-t-s`, -which is why 18 of 32 feedback clicks were Connor hand-rejecting AP baseball recaps that a -`blocklist = ["sports"]` never matched. +Feedback moves these vectors over time. The written text is the cold-start seed, so +the system is useful on day one rather than after 200 clicks. ## Interests -Ordered roughly by strength. Weights are multipliers on the similarity score. +**Foreign affairs is split in two.** Connor's own framing: *"my actual war interest +is low, but unfortunately all wars are important."* The click history looked +war-heavy only because world-news feeds are war-heavy. So statecraft — +`diplomacy-foreign-policy`, weight 0.9 — is the standing interest, while armed +conflict is weighted lower and routed mainly through importance. -1. **Geopolitics and conflict** (1.0) — the strongest and most consistent signal. Middle East - above all, plus great-power dynamics: China's expanding global reach, US foreign policy, - war and escalation, sanctions, the UN. -2. **Tech and science policy** (0.9) — AI regulation, platform governance, surveillance and - privacy law, research funding. Policy *about* technology. -3. **Labor, inequality and social policy** (0.9) — the "democratic socialism" of the old - config, read properly: unions, strikes, wages, housing, healthcare, education, welfare - states. -4. **Software and systems, deep** (0.9) — Connor runs a self-hosted inference stack and a - substantial homelab. Rust and WASM, systems programming, self-hosting, local model - inference, and release notes for things he actually runs. **Eligible for push**, not - digest-only — this was an explicit choice. -5. **Urbanism and city planning** (0.8) — transit, zoning, housing density, street design, - public space. New in this profile; entirely absent from v0.2. -6. **Energy, climate and infrastructure** (0.6) — grid buildout, storage, renewables, climate - policy. Real but explicitly weaker than the others. -7. **Global South** (0.6) — Africa, South Asia, Latin America. Not selected in the interview, - but three of the fourteen starred items were Uber exiting Nigeria and Uganda, India's grid - and storage crunch, and explosions at a Bolivian military base. Carried at a modest weight - on that evidence. **Flagged for review** — drop it if it turns out to be noise. +Within conflict, his stated ordering became five weighted blocks: -### Geography +| block | weight | | +|---|---|---| +| `conflict-anti-imperial` | 0.90 | anti-colonial and socialist-revolutionary wars | +| `conflict-us-involved` | 0.85 | wars the US is actively engaged in | +| `conflict-mideast-israel` | 0.80 | Middle East, especially engagements with Israel | +| `conflict-undercovered` | 0.75 | Sudan, DRC, Myanmar, the Sahel, Yemen | +| `conflict-general` | 0.45 | everything else — low interest, still important | -- **Cincinnati** — where he lives. Local politics and city stories, at a much lower - importance bar than world news. -- **Louisville / Kentucky** — where he is from. Still wanted, weaker than Cincinnati. -- **Ohio** — state politics. -- **US national politics** — federal policy, courts, elections, legislation. -- **Italy** and **Mexico** — specific standing country interests. +**Space and aerospace** (0.95) is the single highest-weighted block: Connor is an +aerospace engineer. He asked for launches and missions, deep technical coverage +(propulsion, materials, orbital mechanics, failure post-mortems), and +astronomy/planetary science — deliberately *not* the industry-and-contracts beat, +which his policy blocks already catch. -## Vetoes +**Software and systems** (0.9, push-eligible) covers local model inference, Rust and +WebAssembly, self-hosting and infrastructure, and — in his words — *"Linux news in +general. The ricing world/neovim configs/etc."* Notably he did **not** pick security +and vulnerabilities, so that is absent rather than assumed. -- **US big-three sports** — professional and college basketball, American football, baseball, - and hockey. Game recaps, scores, trades, drafts, salary-cap news, betting lines. -- **Royal and celebrity gossip** — the British royal family, red carpets, box office, - influencer drama. -- **Horoscopes, listicles, product-roundup content.** +**Political economy** spans `labor-inequality` (0.85), `left-movements` (0.85), +`anticolonial-sovereignty` (0.8), `far-right-watch` (0.75, tracked as a threat +rather than out of sympathy), and `political-theory` (0.7). -### Explicitly NOT vetoed +**Place**: `cincinnati` (0.9 — where he lives, so a lower bar), `ohio-state-politics` +(0.75), `kentucky-louisville` (0.6 — where he's from), `us-national` (0.8), and +`italy-mexico` (0.75). -Connor is not anti-sport; he is anti *US-big-three-wire-recap*. These stay: +Italy and Mexico are a special case. He follows both for politics but also because +he is learning the languages and expects to travel there, so **culture and popular +culture count as signal in this block, not noise** — the one place where lifestyle +content is wanted. The `consumer-lifestyle` veto must not swallow it. -- **Rock climbing, chess, the Olympics, oddball sports** — actively wanted. -- **Soccer** — he is trying to get into it. Match previews and live threads are noise, but - the sport is not blocked as a concept. This is the first thing to loosen if soccer feels - under-covered. -- **Sports governance and politics** — FIFA/UEFA corruption, Olympic host-city displacement, - Qatar migrant labour. These read as politics, not sport, and the old blocklist's - collisions with them were a bug. +Also carried: `science-discovery` (0.8), `environment-ecology` (0.75), +`tech-science-policy` (0.9), `urbanism` (0.8), `global-south` (0.7), +`energy-climate` (0.6). + +## Vetoes, and why some things are demotes instead + +Hard vetoes are reserved for categories that never overlap a real interest: US +big-three sports recaps, royal and celebrity gossip, consumer/lifestyle filler. + +Anything that *might* overlap is a **demote** — a penalty on the score rather than +exclusion. This distinction exists because a hard veto on a sports-adjacent region +killed *"Belgian mayor blocks club from hosting Israeli opponent in Europa League"*, +which is Middle East politics. Overlap is the norm, not the exception: + +- `crypto-ai-hype` sits next to AI regulation and local-inference news +- `culture-war-outrage` sits next to `far-right-watch` +- `true-crime-blotter` sits next to criminal-justice policy + +### Explicitly not vetoed + +Connor is not anti-sport; he is anti *US-big-three-wire-recap*. **Rock climbing, +chess, the Olympics and oddball sports are actively wanted.** Soccer is not blocked +as a concept — he is trying to get into it — though match previews and live threads +are noise. Sports *governance* (FIFA/UEFA corruption, Olympic host-city displacement, +Qatar migrant labour) reads as politics and stays. ## The importance lane -Independent of interest. This is the "I need to know this even though it isn't my topic" -channel, and it is half the reason the project exists. Its own small daily budget. +Independent of interest, and half the reason the project exists: war and escalation, +mass-casualty events, US federal shocks, economic breaks, and major US political +events (decisive elections, deaths of political figures). -- War and escalation — new military action between states, major escalation in an ongoing - conflict, nuclear or WMD events. -- Mass-casualty events and disasters — earthquakes, major attacks, industrial and transport - catastrophes. -- US federal shocks — constitutional crises, major Supreme Court rulings, shutdowns, sudden - sweeping policy moves. -- Economic breaks — market crashes, bank failures, currency crises, central bank surprises. -- Major US political events — significant election results, deaths of political figures. - -**Vetoes still apply on this lane.** In v0.2 the bypass lane ignored relevance entirely -(`gate.rs:302`), so a blocklisted item could still fire at priority 5. That must not recur. +**Vetoes still apply on this lane.** In v0.2 the bypass lane ignored relevance +entirely (`gate.rs:302`), so a blocklisted item could still fire at priority 5. ## Delivery -- 2–5 pushes per day total, across both lanes. High bar. -- The daily digest is a **web page**, not a push payload. v0.2 sent a long markdown blob that - ntfy turned into a `.md` attachment, which is miserable to read on a phone. The push is - three lines and a link. -- ntfy feedback buttons **must work**. They never have — `NEWS_API_BASE_URL` was unset, so - they were silently dropped at boot. That is the entire training signal for everything above. +- Digest at **08:45**, quiet hours **22:00–07:00**. The importance lane **may** wake + him; the interest lane never may. +- 2–5 pushes/day across both lanes. +- Conflict outside categories A–D notifies only on a major development — a new + offensive, a ceasefire, mass casualties, a state entering. Routine front-line + reporting goes to the digest. +- HN items are ignored below **150 points**: the crowd does the first filtering pass. +- The digest is a **complete record** of the day, including already-notified items, + and the weekend edition runs longer and leans longform. +- **Three serendipity slots** in every digest for items that scored well on quality + and importance but poorly on the profile, labelled "outside your usual". A system + that only shows you what you already like tightens its own bubble as it learns. +- No format filter: he reads analysis, wire copy, investigations *and* opinion, which + also confirms the old config's `notify_opinions = true` was deliberate. +- Feedback stays **binary**. The old system's failure was collecting 32 labels, not + collecting shallow ones. + +## Source trust + +`weight` in `sources.toml` is editorial trust, carried over from the old config where +Connor had deliberately set Al Jazeera highest (1.2) and NYT lowest (0.8), and +extended to weight independent outlets (The Intercept, Drop Site, Tribune, African +Arguments, Middle East Eye, +972) at 1.1. It affects importance and tie-breaking, +never whether an item is visible. + +## Language + +Embedding is English-only. Measured: the same story in Spanish and English scored +0.758 and 0.600 cosine against a random-pair p99 of 0.634 — one of two translation +pairs was indistinguishable from an unrelated article. Native-language feeds would +rank erratically, so Italy and Mexico are covered by English-language sources. +Revisit with a multilingual model only if that becomes worth re-calibrating every +threshold for. ## Open questions -- Digest hour, and the exact quiet-hours window. -- Whether the importance lane may ever override quiet hours, and at what score. -- Whether "Global South" is a real standing interest or an artifact of a small sample. +- Whether `global-south` is a genuine standing interest or an artifact of a small + sample. Confirmed wanted by Connor, but worth re-checking against real feedback. +- Exact score at which the importance lane may override quiet hours. diff --git a/profile/profile.toml b/profile/profile.toml index b25b2a6..a2b25e2 100644 --- a/profile/profile.toml +++ b/profile/profile.toml @@ -3,24 +3,127 @@ # never the mean. Probes are written as prose resembling article text, because # that is what they are compared against. # -# Weights multiply the similarity. Feedback moves these vectors over time; the -# text here is the cold-start seed. +# Blocks are compared by z-score WITHIN block, then multiplied by `weight`. +# See profile/CALIBRATION.md -- raw cosine is not comparable across blocks. -schema_version = 1 +schema_version = 2 -# ---------------------------------------------------------------- interests -- +# =========================================================== interests ======= + +# --- foreign affairs, split from armed conflict ------------------------------ +# Connor: "my actual war interest is low, but unfortunately all wars are +# important." The click history looked war-heavy only because world-news feeds +# are war-heavy. Statecraft is the standing interest; combat reporting is +# routed by importance instead. Keeping them in one block conflated the two. [[interest]] -id = "geopolitics" -weight = 1.0 +id = "diplomacy-foreign-policy" +weight = 0.9 probes = [ - "Escalation in the Middle East as strikes continue and diplomatic efforts stall.", "China expands its influence abroad through investment, diplomacy and security agreements.", "US foreign policy shifts as the administration reconsiders its posture toward an ally.", "The United Nations Security Council debates sanctions and an arms embargo.", - "Ceasefire negotiations falter amid disputes over withdrawal and prisoner exchange.", + "Two governments sign a treaty realigning trade and security ties in the region.", + "A diplomatic crisis erupts as one country expels another's ambassador.", ] +# The A-E ordering Connor gave, encoded as weights. Even the lowest band still +# reaches him -- but through importance, on major developments only. +[[interest]] +id = "conflict-anti-imperial" # (A) top of his ordering +weight = 0.9 +probes = [ + "An armed liberation movement fights a colonial or occupying power for self-determination.", + "A socialist or revolutionary force gains ground against a US-backed government.", + "Anti-imperialist resistance movements coordinate against foreign military presence.", +] + +[[interest]] +id = "conflict-us-involved" # (B) +weight = 0.85 +probes = [ + "US forces carry out strikes and the administration defends the legal basis for the war.", + "American troops, weapons or funding are central to an ongoing foreign conflict.", +] + +[[interest]] +id = "conflict-mideast-israel" # (C) +weight = 0.8 +probes = [ + "Israeli forces strike targets in Gaza, Lebanon or Syria amid continuing escalation.", + "Ceasefire negotiations falter amid disputes over withdrawal and prisoner exchange.", + "Palestinians face displacement, demolition and occupation in the West Bank.", +] + +[[interest]] +id = "conflict-undercovered" # (D) +weight = 0.75 +probes = [ + "Fighting intensifies in Sudan as famine spreads and aid convoys are blocked.", + "Armed groups clash in the eastern Democratic Republic of Congo, displacing thousands.", + "Conflict in Myanmar, the Sahel or Yemen continues with little international attention.", +] + +[[interest]] +id = "conflict-general" # (E) -- low interest, still important +weight = 0.45 +probes = [ + "Front-line fighting continues as both sides exchange drone and artillery strikes.", + "A long-running interstate war grinds on with incremental territorial changes.", +] + +# --- space and aerospace ----------------------------------------------------- +# Connor is an aerospace engineer. He asked for launches/missions/spacecraft, +# deep technical, and astronomy/planetary science -- deliberately NOT the +# industry-and-contracts beat, which his policy blocks would catch anyway. +[[interest]] +id = "space-aerospace" +weight = 0.95 +probes = [ + "The rocket lifted off carrying the spacecraft to orbit; the booster landed and the mission reached its next milestone.", + "Engineers trace a launch failure to a propulsion anomaly and publish the post-mortem.", + "Propulsion, materials and orbital mechanics choices shape the vehicle's design trade-offs.", + "A planetary probe returns data from its flyby, reshaping models of the body's formation.", + "Astronomers report observations of exoplanets, galaxies or a transient event from a space telescope.", + "A crewed mission's schedule slips as the spacecraft completes qualification testing.", +] + +[[interest]] +id = "science-discovery" +weight = 0.8 +probes = [ + "Researchers report a finding that overturns the prevailing explanation in the field.", + "A study in physics, biology or archaeology reveals something previously unknown.", + "New evidence changes scientists' understanding of a long-standing question.", +] + +[[interest]] +id = "environment-ecology" +weight = 0.75 +probes = [ + "Protections for an endangered species are rolled back, alarming conservation biologists.", + "An ecosystem shifts as habitat loss and invasive species reshape which animals survive.", + "Public land and wildlife management rules are rewritten.", +] + +# --- software and systems (push-eligible) ------------------------------------ +[[interest]] +id = "software-systems" +weight = 0.9 +push_eligible = true +probes = [ + "Running large language models locally: quantization, throughput, context length and memory bandwidth on consumer hardware.", + "A new open-weights model is released with benchmarks and quantized builds available.", + "A new release of the systems programming language changes the compiler, borrow checker or toolchain.", + "WebAssembly gains new capabilities for running compiled code in the browser.", + "Self-hosting a service stack with containers, reverse proxies and automated deployment.", + "A widely used open source project announces a breaking change, a license change, or a governance dispute.", + # Connor's words: "Linux news in general. The ricing world/neovim configs/etc" + "A Linux desktop release brings changes to the compositor, window manager and display stack.", + "Customizing a desktop setup: window manager configuration, terminal theming, dotfiles and Neovim plugins.", +] + +# --- policy and political economy -------------------------------------------- [[interest]] id = "tech-science-policy" weight = 0.9 @@ -34,7 +137,7 @@ probes = [ [[interest]] id = "labor-inequality" -weight = 0.9 +weight = 0.85 probes = [ "Workers vote to unionize after a long organizing campaign; the employer contests the result.", "A strike over wages and staffing shuts down operations across the sector.", @@ -44,67 +147,39 @@ probes = [ ] [[interest]] -id = "software-systems" -weight = 0.9 -push_eligible = true +id = "left-movements" +weight = 0.85 probes = [ - "A new release of the systems programming language brings major changes to the compiler and toolchain.", - "Running large language models locally on consumer hardware: quantization, throughput and memory bandwidth.", - "Self-hosting a service stack with containers, reverse proxies and automated deployment.", - "A widely used open source project announces a breaking change, a license change, or a governance dispute.", - "WebAssembly gains new capabilities for running compiled code in the browser.", - "A serious vulnerability is disclosed in widely deployed infrastructure software.", + "A socialist party wins a decisive election, shifting the governing coalition leftward.", + "A mass movement mobilizes in the streets demanding redistribution and public ownership.", + "Cooperatives, mutual aid networks and tenant unions expand their organizing base.", ] [[interest]] -id = "urbanism" +id = "anticolonial-sovereignty" weight = 0.8 probes = [ - "The city council votes to reform zoning rules and allow denser housing near transit.", - "A new bus rapid transit and light rail line reshapes how residents move through the city.", - "Planners redesign streets to reduce traffic deaths and reclaim space from cars.", - "A downtown redevelopment project raises questions about displacement and public space.", + "Indigenous communities win recognition of land rights against extractive industry.", + "A former colony renegotiates resource contracts and asserts control over its minerals.", + "IMF and World Bank conditionality drives austerity, prompting debt-relief demands.", ] +# Tracked as a threat, not out of sympathy. Connor's AfD and Tate clicks fit here. [[interest]] -id = "energy-climate" -weight = 0.6 +id = "far-right-watch" +weight = 0.75 probes = [ - "Grid operators struggle to connect new renewable generation as storage capacity lags demand.", - "A national climate policy sets emissions targets and restructures energy subsidies.", - "Conservation rules protecting endangered species and public land are rolled back.", + "A far-right party surges in a regional election, unsettling the political mainstream.", + "An authoritarian government consolidates power over courts, press and elections.", + "Extremist organizing and street violence grow as mainstream parties adopt their rhetoric.", ] -# Confirmed by Connor 2026-09-08: wanted, at a level that keeps him up to date -# rather than saturating the feed. Also supported by three of eighteen starred -# items (Uber exiting Nigeria and Uganda, India's grid crunch, the Bolivian base -# explosions). [[interest]] -id = "global-south" +id = "political-theory" weight = 0.7 probes = [ - "An international company withdraws from several African markets amid regulatory pressure.", - "Elections and political upheaval reshape governments across Latin America.", - "Infrastructure and economic development across South Asia strains under rapid growth.", -] - -# ---------------------------------------------------------------- geography -- - -[[interest]] -id = "cincinnati" -weight = 0.9 -local = true -probes = [ - "Cincinnati city council debates a development, budget or policing decision affecting neighborhoods.", - "Local news from Cincinnati and Hamilton County, Ohio.", -] - -[[interest]] -id = "kentucky-louisville" -weight = 0.6 -local = true -probes = [ - "Louisville and Kentucky state politics, legislation and local government decisions.", + "An essay argues about socialism, imperialism and political economy, drawing on history rather than reporting a news event.", + "A critical analysis of capitalism's structure and its alternatives.", ] [[interest]] @@ -116,19 +191,93 @@ probes = [ "Federal agencies are restructured by executive order, prompting legal challenges.", ] +# --- place ------------------------------------------------------------------- +[[interest]] +id = "urbanism" +weight = 0.8 +probes = [ + "The city council votes to reform zoning rules and allow denser housing near transit.", + "A new bus rapid transit and light rail line reshapes how residents move through the city.", + "Planners redesign streets to reduce traffic deaths and reclaim space from cars.", + "A downtown redevelopment project raises questions about displacement and public space.", +] + +[[interest]] +id = "cincinnati" +weight = 0.9 +local = true +probes = [ + "Cincinnati city council votes on a budget, a zoning change or a development project downtown.", + "A regional transit plan or street redesign changes how people move around Cincinnati.", + "Rent, eviction and homelessness policy in Hamilton County; a local union drive or strike.", + "Neighborhood institutions, local arts, food and community life in Cincinnati.", + "Local news from Cincinnati and Hamilton County, Ohio.", +] + +[[interest]] +id = "kentucky-louisville" +weight = 0.6 +local = true +probes = [ + "Louisville and Kentucky state politics, legislation and local government decisions.", +] + +# The level above city hall: Connor asked for Ohio and Kentucky statehouse +# coverage alongside Cincinnati city government. +[[interest]] +id = "ohio-state-politics" +weight = 0.75 +local = true +probes = [ + "The Ohio statehouse passes legislation over the governor's objection.", + "Redistricting, election law and statewide ballot measures in Ohio.", +] + +# Connor follows both for politics, but also because he is learning the +# languages and expects to travel there -- so culture and popular culture count +# as signal here, not noise. He said: "pop culture stuff from Mexico/Italy is +# even ok. Learning the culture helps me learn the language and travel there +# better." That makes this the one block where lifestyle content is wanted, so +# the consumer-lifestyle veto must not swallow it. [[interest]] id = "italy-mexico" -weight = 0.7 +weight = 0.75 probes = [ "Italian politics, government coalitions and social policy in Italy.", "Mexican politics, security policy and the relationship with the United States.", + "Regional life, cost of living and transport across Italy; what is changing in its cities.", + "Mexican and Italian music, film, television and food culture; what people are talking about.", ] -# ------------------------------------------------------------------- vetoes -- +# Confirmed by Connor 2026-09-08: wanted, at a level that keeps him up to date +# rather than saturating the feed. +[[interest]] +id = "global-south" +weight = 0.7 +probes = [ + "An international company withdraws from several African markets amid regulatory pressure.", + "Elections and political upheaval reshape governments across Latin America.", + "Infrastructure and economic development across South Asia strains under rapid growth.", +] + +[[interest]] +id = "energy-climate" +weight = 0.6 +probes = [ + "Grid operators struggle to connect new renewable generation as storage capacity lags demand.", + "A national climate policy sets emissions targets and restructures energy subsidies.", +] + +# =========================================================== vetoes ========== +# Hard vetoes only for categories that never overlap a real interest. Anything +# that *might* overlap is a demote instead -- see below. This split exists +# because a hard veto on a sports-adjacent region killed "Belgian mayor blocks +# club from hosting Israeli opponent in Europa League", which is Middle East +# politics. Overlap is the norm, not the exception. [[veto]] id = "us-big-three-sports" -threshold = 0.55 +threshold = 0.60 probes = [ "The team beat their rivals in a game recap with final scores, statistics and player performances.", "A player is traded, drafted or signs a contract; the league fines a franchise over salary cap rules.", @@ -139,20 +288,57 @@ probes = [ [[veto]] id = "royal-celebrity-gossip" -threshold = 0.55 +threshold = 0.60 probes = [ "The royal family attends an event; speculation continues about the couple's status and future.", "Celebrity red carpet appearances, box office returns and influencer controversy.", ] [[veto]] -id = "listicle-filler" +id = "consumer-lifestyle" threshold = 0.60 probes = [ - "Horoscopes, quizzes, and a roundup of the best products to buy this week.", + "The best products to buy this week, ranked, with deals and discount codes.", + "Horoscopes, quizzes, recipes and a travel guide to the perfect weekend away.", + "Personal finance tips: how to budget, save and pick a credit card.", ] -# ------------------------------------------------------- the importance lane -- +# =========================================================== demotes ========= +# Penalty applied to the interest score, not a hard exclusion. These categories +# each sit adjacent to something Connor genuinely wants, so removing them +# outright would take real stories with them. + +# Adjacent to: tech-science-policy (AI regulation) and software-systems (local +# inference). A hard veto here would be actively harmful. +[[demote]] +id = "crypto-ai-hype" +penalty = 1.0 +probes = [ + "The token surged after the exchange listing as investors piled into the crypto rally.", + "The startup raised a funding round at a multibillion-dollar valuation to build AI agents.", + "A company announces its new AI-powered product will transform the industry.", +] + +# Adjacent to: far-right-watch. "X said Y and Z reacted" is noise; a far-right +# party winning a state is not. +[[demote]] +id = "culture-war-outrage" +penalty = 1.0 +probes = [ + "A politician's remark sparked backlash online and critics demanded an apology.", + "A manufactured controversy over a advertisement or a celebrity's comment dominates social media for a day.", +] + +# Adjacent to: policing and criminal-justice policy, which Connor does want. +[[demote]] +id = "true-crime-blotter" +penalty = 0.8 +probes = [ + "Police arrested a suspect in the killing; the trial opens with jury selection this week.", + "A missing person case takes a turn as investigators search a rural property.", +] + +# ============================================= the importance lane =========== # Independent of interest. Vetoes still apply here -- v0.2's bypass lane ignored # relevance entirely (gate.rs:302), which let blocklisted items fire at priority 5. @@ -162,6 +348,7 @@ weight = 1.0 probes = [ "One state launches military strikes on another, marking a major escalation of the conflict.", "Nuclear facilities are targeted or a state signals willingness to use weapons of mass destruction.", + "A major new offensive opens a front, or a ceasefire is signed and takes effect.", ] [[importance]] @@ -198,34 +385,59 @@ probes = [ "A prominent political figure dies or resigns unexpectedly.", ] -# ------------------------------------------------------- calibrated scoring -- -# These constants are not guesses. They were fitted on 2026-09-08 against all -# 3,885 items in the v0.2 database and Connor's 40 real feedback labels. -# See profile/CALIBRATION.md for the measurements. +# =========================================================== delivery ======== [delivery] -# Settled with Connor 2026-09-08. digest_hour = "08:45" quiet_hours = { start = "22:00", end = "07:00" } # The importance lane MAY override quiet hours. The interest lane never may. importance_may_wake = true +# No format filter: Connor reads analysis, wire copy, investigations AND opinion, +# so writing style is deliberately not a scoring axis. This also confirms the old +# config's notify_opinions = true was intentional. +format_filter = false +# Filter-bubble counterweight. The digest always reserves slots for items that +# scored well on quality and importance but POORLY on the interest profile, +# labelled "outside your usual". Notifications stay strictly on-profile; this is +# a reading surface, not an interruption. +serendipity_slots = 3 + +# The digest is a complete record of the day, including items that already +# fired a notification -- not just the leftovers. +digest_includes_notified = true +# Saturday and Sunday editions run longer and lean toward longform, analysis and +# the serendipity slots. +weekend_digest_multiplier = 2.0 + +# Aggregator maturation. An HN post at 3 points is noise; the same post at 300 +# is news. Items below this score are ingested and clustered but never scored +# for routing, so the system waits for the crowd instead of firing on arrival. +hn_min_score = 150 + +# Feedback stays binary: interested / not interested. Low friction matters more +# than richness here -- the model wants many labels, and the old system's +# failure was collecting 32 of them, not collecting shallow ones. +feedback_actions = ["interested", "not_interested"] + +# Embedding stays English-only (bge-small-en-v1.5). Measured 2026-09-08: the +# same story in Spanish and English scored 0.758 and 0.600 cosine, against a +# random-pair p99 of 0.634 -- so one of two translation pairs was +# indistinguishable from an unrelated article. Native-language Italian and +# Spanish feeds would rank erratically, so Italy and Mexico are covered by +# English-language sources (ANSA English, Mexico News Daily) instead. Revisit +# with multilingual-e5-small only if native sources become worth re-calibrating +# every threshold for. +embedding_language = "en" +# Conflict outside categories A-D notifies only on a major development -- a new +# offensive, a ceasefire, mass casualties, a state entering. Routine front-line +# reporting goes to the digest. +conflict_general_requires_importance = true + +# =========================================================== scoring ========= +# Fitted 2026-09-08 against 3,885 items and 40 real feedback labels. +# See profile/CALIBRATION.md. [scoring] -# Compare blocks by z-score WITHIN each block, never by raw cosine. Raw cosine -# is not comparable across blocks -- "geopolitics" averages 0.501 similarity to -# world-news text while "kentucky-louisville" averages 0.441 -- so a plain max -# let geopolitics win 78% of all items. Per-block z-scoring drops that to 19% -# and moves the median rank of a known-liked item from 1246 to 293 of 3885. normalize = "zscore-per-block" - -# Veto is deliberately NOT a plain similarity threshold. A story can be -# genuinely sports-adjacent and still be wanted: "Belgian mayor blocks club from -# hosting Israeli opponent in Europa League" is Middle East politics, and both a -# substring blocklist AND a 0.55 cosine threshold falsely killed it. -# -# So: veto on unambiguous similarity, OR when the veto signal beats the best -# interest signal by a margin. Measured at these values: 17/22 of Connor's real -# rejections caught, 0/18 of his likes lost, ~21% of the corpus vetoed, and all -# four sports-adjacent news canaries survive. veto_absolute = 0.60 veto_margin_over_interest = 0.3 diff --git a/profile/sources.toml b/profile/sources.toml index 5d270e7..78821fe 100644 --- a/profile/sources.toml +++ b/profile/sources.toml @@ -6,6 +6,10 @@ # confirmations. Getting this wrong is what made the old system fire priority-5 # pushes on anything two feeds happened to carry. # +# `weight` is editorial trust, carried over from Connor's old config where he +# had deliberately set Al Jazeera highest (1.2) and NYT lowest (0.8). It affects +# importance and tie-breaking, never whether an item is visible at all. +# # `prominence = 1` marks a publisher's front-page/top-stories feed. Subscribing # to both it and the section feed is a free editorial-promotion signal; the old # config dropped BBC's top-news feed and threw that away. @@ -42,6 +46,7 @@ bucket = "guardian" name = "NYT World" url = "https://rss.nytimes.com/services/xml/rss/nyt/World.xml" bucket = "nyt" +weight = 0.8 [[source]] name = "NPR News" @@ -75,11 +80,13 @@ poll_interval_secs = 1800 # --------------------------------------------------------------- mideast ---- [[source]] name = "Middle East Eye" +weight = 1.1 url = "https://www.middleeasteye.net/rss" bucket = "middle-east-eye" [[source]] name = "+972 Magazine" +weight = 1.1 url = "https://www.972mag.com/feed/" bucket = "972mag" poll_interval_secs = 3600 @@ -266,3 +273,121 @@ name = "Mexico News Daily" url = "https://mexiconewsdaily.com/feed/" bucket = "mexico-news-daily" poll_interval_secs = 3600 + +# ============================================================ space / aerospace +# Connor is an aerospace engineer and asked for launches and missions, deep +# technical coverage, and astronomy/planetary science. All verified 2026-09-08. + +[[source]] +name = "SpaceNews" +url = "https://spacenews.com/feed/" +bucket = "spacenews" +poll_interval_secs = 3600 + +[[source]] +name = "Spaceflight Now" +url = "https://spaceflightnow.com/feed/" +bucket = "spaceflight-now" +poll_interval_secs = 3600 + +[[source]] +name = "NASASpaceflight" +url = "https://www.nasaspaceflight.com/feed/" +bucket = "nasaspaceflight" +poll_interval_secs = 3600 + +[[source]] +name = "NASA Breaking" +url = "https://www.nasa.gov/rss/dyn/breaking_news.rss" +bucket = "nasa" +poll_interval_secs = 3600 + +[[source]] +name = "ESA" +url = "https://www.esa.int/rssfeed/Our_Activities/Space_News" +bucket = "esa" +poll_interval_secs = 7200 + +[[source]] +name = "Ars Technica Science" +url = "https://feeds.arstechnica.com/arstechnica/science" +bucket = "ars-technica" +poll_interval_secs = 3600 + +[[source]] +name = "Phys.org Space" +url = "https://phys.org/rss-feed/space-news/" +bucket = "phys-org" +poll_interval_secs = 7200 + +# Deliberately NOT added: nature.rss (mostly "Author Correction:" journal +# metadata, not news) and sciencedaily.com (a university press-release mill, +# 60 items a poll). Both were probed and rejected on the content they returned. +[[source]] +name = "Quanta Magazine" +url = "https://www.quantamagazine.org/feed/" +bucket = "quanta" +poll_interval_secs = 21600 + +# ================================================================ linux / ricing +# Connor: "Linux news in general. The ricing world/neovim configs/etc" + +[[source]] +name = "Phoronix" +url = "https://www.phoronix.com/rss.php" +bucket = "phoronix" +poll_interval_secs = 3600 + +[[source]] +name = "LWN" +url = "https://lwn.net/headlines/newrss" +bucket = "lwn" +poll_interval_secs = 7200 + +[[source]] +name = "It's FOSS" +url = "https://news.itsfoss.com/rss/" +bucket = "its-foss" +poll_interval_secs = 7200 + +[[source]] +name = "r/unixporn" +url = "https://www.reddit.com/r/unixporn/top/.rss" +bucket = "reddit" +poll_interval_secs = 10800 + +[[source]] +name = "Neovim releases" +url = "https://github.com/neovim/neovim/releases.atom" +bucket = "github-releases" +poll_interval_secs = 21600 + +# ==================================================== left / anti-colonial ===== + +[[source]] +name = "African Arguments" +weight = 1.1 +url = "https://africanarguments.org/feed/" +bucket = "african-arguments" +poll_interval_secs = 21600 + +[[source]] +name = "Tribune" +weight = 1.1 +url = "https://tribunemag.co.uk/feed" +bucket = "tribune" +poll_interval_secs = 21600 + +[[source]] +name = "The Intercept" +weight = 1.1 +url = "https://theintercept.com/feed/?rss" +bucket = "the-intercept" +poll_interval_secs = 7200 + +[[source]] +name = "Drop Site News" +weight = 1.1 +url = "https://www.dropsitenews.com/feed" +bucket = "drop-site" +poll_interval_secs = 7200