commit c89a7236031d27945e4f2da866822457f302b26c Author: Connor Johnstone Date: Tue Sep 8 14:34:56 2026 -0400 Seed the rewrite: profile, calibration, and repo scaffolding Ground-up replacement for news_triage, whose signal quality failed because it had no representation of what its user likes -- three keywords fed to BM25 plus a two-term substring blocklist. profile/ holds the taste model as prose and TOML: 11 interest blocks, 3 veto blocks, 5 importance blocks, each a set of probe sentences to be embedded. The [scoring] constants are fitted, not guessed. CALIBRATION.md records the measurements against all 3,885 items in the v0.2 database and 40 real feedback labels, including two findings that changed the design: - compare blocks by z-score within block, never raw cosine; raw cosine let geopolitics win 78% of items and buried known-liked items at median rank 1246 of 3885 (293 after normalizing) - veto needs a margin over the interest signal, not a threshold; a plain 0.55 cutoff falsely killed "Belgian mayor blocks club from hosting Israeli opponent in Europa League", which the old substring blocklist also killed diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..122f031 --- /dev/null +++ b/.gitignore @@ -0,0 +1,32 @@ +# Allowlist ignore file, per the house convention: ignore everything, then +# explicitly re-admit tracked paths. +**/* + +!.gitignore +!README.md +!pyproject.toml +!uv.lock + +!profile/ +!profile/** + +!news/ +!news/** + +!tests/ +!tests/** + +!deploy/ +!deploy/** + +# Re-exclude generated and local artifacts that would otherwise slip through +# the directory re-admissions above. +news/**/__pycache__/** +news/**/*.pyc +tests/**/__pycache__/** +tests/**/*.pyc +**/*.db +**/*.db-wal +**/*.db-shm +**/*.npy +.venv/** diff --git a/README.md b/README.md new file mode 100644 index 0000000..48f1e93 --- /dev/null +++ b/README.md @@ -0,0 +1,53 @@ +# news + +A personal news aggregator: many sources in, two to five well-chosen push +notifications and one readable daily digest out. + +This is a ground-up rewrite of [`news_triage`](../news_triage), which is still +running. The rewrite exists because the old system had no representation of what +its user actually likes — its whole preference model was three keywords fed to +BM25 plus a two-term substring blocklist, and everything else was ranking +machinery with nothing meaningful to rank. + +## The core idea + +Taste is a set of **vectors**, not a bag of keywords. + +Every interest, veto and importance category in [`profile/profile.toml`](profile/profile.toml) +is written as prose and embedded with a small CPU model (`bge-small-en-v1.5`, +~130 MB, ONNX, no torch, no GPU — so it never contends with the llama-swap stack). +An item's score is the max cosine similarity across a block's probes, never the +mean: averaging "Middle East geopolitics" with "grid-scale energy storage" gives +a vector near neither. + +This is what lets a topic be expressed as a *region* rather than a string. The +old system could not represent "sports" as a concept, only as the literal +characters `s-p-o-r-t-s`, which is why 18 of its 32 feedback clicks were its user +hand-rejecting baseball recaps that `blocklist = ["sports"]` never matched. + +A generative LLM is used only for prose — the daily digest write-up and cluster +headlines — at roughly 1–30 calls/day through LiteLLM, never for per-item scoring. + +## Two routing lanes + +Notifications are decided on two independent axes, because "something I might +like" and "something I need to know about" are different questions: + +- **interest** — high similarity to the profile's interest vectors. +- **importance** — burst velocity, independent-source-bucket count, editorial + prominence, and aggregator-native scores. Deliberately *not* raw corroboration + count, which just measures how many wire-reprinting outlets you subscribed to. + +Vetoes are absolute on **both** lanes. In the old system the bypass lane ignored +relevance entirely, so a blocklisted item could still fire at priority 5. + +## Layout + + profile/ the user's taste, as prose and as TOML. Start here. + news/ the pipeline: sources, store, embed, cluster, score, route, deliver + tests/ + deploy/ + +## Status + +Early. `profile/` is written; the pipeline is not yet built. diff --git a/profile/CALIBRATION.md b/profile/CALIBRATION.md new file mode 100644 index 0000000..947c64c --- /dev/null +++ b/profile/CALIBRATION.md @@ -0,0 +1,106 @@ +# Calibration + +Measured 2026-09-08 against the full v0.2 corpus (3,885 items) and Connor's 40 +real feedback labels (18 interested / 22 not_interested). Model: +`BAAI/bge-small-en-v1.5` via fastembed, CPU, quantized ONNX. + +Everything here is a measurement, not a design intention. Re-run it before +changing any constant in `profile.toml`'s `[scoring]` block. + +## Does the embedding model separate signal from noise? + +Yes, decisively — this was the premise the whole rewrite rests on, so it was +tested before any pipeline code was written. + +| metric | value | +|---|---| +| mean interest score, items Connor liked | 0.552 | +| mean interest score, items Connor rejected | 0.465 | +| liked items falsely vetoed | **0 of 18** | +| rejected items caught by veto | 17 of 22 | + +The old system could not do this at all: its `blocklist = ["sports"]` matched +none of the baseball recaps Connor spent 18 clicks rejecting. + +## Finding 1 — compare blocks by z-score, never by raw cosine + +Raw cosine magnitude is not comparable across blocks. Mean similarity to the +whole corpus, per block: + + geopolitics 0.501 us-national 0.477 + italy-mexico 0.494 labor-inequality 0.475 + global-south 0.490 software-systems 0.469 + energy-climate 0.455 tech-science-policy 0.468 + cincinnati 0.464 urbanism 0.448 + kentucky-louisville 0.441 + +Taking a plain `max` across blocks therefore hands almost everything to +`geopolitics`. Normalizing within each block first fixes it: + +| ranking | liked items' median rank | in top 200 | geopolitics' share of items | +|---|---|---|---| +| raw cosine | 1246 / 3885 | 4/18 | 78% | +| **z-score per block** | **293 / 3885** | **7/18** | **19%** | +| percentile rank | 1504 / 3885 | 4/18 | 34% | + +Percentile ranking is *worse than raw* because converting to ranks discards +magnitude. Z-score wins clearly. + +Under z-scoring the top of the feed becomes topically varied — AI regulation, +the Google ads ruling, a Kenyan airport strike, the shutdown vote, India's grid +crunch, mail voting at the Supreme Court, childcare waitlists — instead of +twenty consecutive Lebanon headlines. + +## Finding 2 — veto needs a margin rule, not a threshold + +A pure threshold at 0.55 caught 20/22 rejections but falsely vetoed real news +that is merely sports-*adjacent*: + +- "Belgian mayor blocks club from hosting Israeli opponent in Europa League" (0.579) +- "Shein trades down after Hong Kong debut" (0.562) — matched on "trades" +- "California lawmakers pull deal on wildfire victim compensation" (0.569) +- "Kushner dice que lamenta su participación en el plan de venta del Mundial" (0.562) + +The first of those is notable: the substring blocklist killed it too, via +`"la liga"` matching "la **Liga** Europa". Two entirely different mechanisms, +same wrong answer, because both ask "is this about sport?" when the question is +"is this *more* about sport than about anything Connor wants?" + +Hence the hybrid rule. Sweep, with canary stories tracked: + +| absolute | margin | rejections caught | likes lost | corpus vetoed | false vetoes | +|---|---|---|---|---|---| +| threshold only 0.55 | — | 20/22 | 0/18 | 28.6% | **4 known** | +| **0.60** | **0.3** | **17/22** | **0/18** | **20.7%** | **0** | +| 0.60 | 0.5 | 16/22 | 0/18 | 17.6% | 0 | +| 0.62 | 0.5 | 13/22 | 0/18 | 15.5% | 0 | + +Chosen: `veto_absolute = 0.60`, `veto_margin_over_interest = 0.3`. + +The one rejection that still escapes is "Umpire leaves game between Red Sox and +Mariners after ball catches him under the chin", which genuinely reads as +human-interest oddity rather than sport. Acceptable. + +## Finding 3 — throughput is fine, but far slower than assumed + +Embedding 3,885 items took **358 s — about 11 items/sec** with default fastembed +settings on this box (quantized bge-small, ~15 cores busy, 2.2 GB RSS). The plan +had claimed "thousands/sec"; that was wrong and is corrected here. + +It does not matter operationally — a realistic day of ~2,000 items embeds in +about three minutes, and embeddings are cached by content hash so each item is +embedded once. But it is worth tuning batch size and thread count before +assuming headroom, and worth re-measuring if the corpus grows. + +Critically, it ran entirely on CPU: no GPU, no llama-swap contention, nothing +competing with the OMO stack. + +## Caveats + +- 40 labels is a small evaluation set. These numbers are directional, not proof. +- The labels are biased: 18 of 22 negatives are AP sports, because AP was 52% of + the corpus. Veto performance is therefore better characterized than interest + ranking. +- Z-score needs a population to normalize over. In production this must be a + rolling per-block mean and standard deviation over a trailing window, not a + one-shot pass over a static corpus. diff --git a/profile/profile.md b/profile/profile.md new file mode 100644 index 0000000..12fe061 --- /dev/null +++ b/profile/profile.md @@ -0,0 +1,108 @@ +# Connor's news profile + +The thing the previous system never had. `news_triage` v0.2's entire model of taste was +three keywords and two blocked substrings; everything else was ranking machinery with +nothing meaningful to rank. This file is the replacement, and `profile.toml` is its +machine-readable form. + +Written 2026-09-08, from a structured interview plus evidence from 32 real feedback +clicks in the v0.2 database. + +## How this is used + +Every `description` and `probe` below is embedded (bge-small-en-v1.5, CPU) into a vector. +Item relevance is the **max** cosine similarity across probes, never the mean — averaging +"Middle East geopolitics" with "grid-scale energy storage" produces a vector near neither. +Feedback then moves these vectors over time (Rocchio); the written text is the cold-start +seed so the system is useful on day one instead of after 200 clicks. + +Vetoes work the same way, as *regions* rather than strings. This is the whole point: the old +system could not express "sports" as a concept, only as the literal characters `s-p-o-r-t-s`, +which is why 18 of 32 feedback clicks were Connor hand-rejecting AP baseball recaps that a +`blocklist = ["sports"]` never matched. + +## Interests + +Ordered roughly by strength. Weights are multipliers on the similarity score. + +1. **Geopolitics and conflict** (1.0) — the strongest and most consistent signal. Middle East + above all, plus great-power dynamics: China's expanding global reach, US foreign policy, + war and escalation, sanctions, the UN. +2. **Tech and science policy** (0.9) — AI regulation, platform governance, surveillance and + privacy law, research funding. Policy *about* technology. +3. **Labor, inequality and social policy** (0.9) — the "democratic socialism" of the old + config, read properly: unions, strikes, wages, housing, healthcare, education, welfare + states. +4. **Software and systems, deep** (0.9) — Connor runs a self-hosted inference stack and a + substantial homelab. Rust and WASM, systems programming, self-hosting, local model + inference, and release notes for things he actually runs. **Eligible for push**, not + digest-only — this was an explicit choice. +5. **Urbanism and city planning** (0.8) — transit, zoning, housing density, street design, + public space. New in this profile; entirely absent from v0.2. +6. **Energy, climate and infrastructure** (0.6) — grid buildout, storage, renewables, climate + policy. Real but explicitly weaker than the others. +7. **Global South** (0.6) — Africa, South Asia, Latin America. Not selected in the interview, + but three of the fourteen starred items were Uber exiting Nigeria and Uganda, India's grid + and storage crunch, and explosions at a Bolivian military base. Carried at a modest weight + on that evidence. **Flagged for review** — drop it if it turns out to be noise. + +### Geography + +- **Cincinnati** — where he lives. Local politics and city stories, at a much lower + importance bar than world news. +- **Louisville / Kentucky** — where he is from. Still wanted, weaker than Cincinnati. +- **Ohio** — state politics. +- **US national politics** — federal policy, courts, elections, legislation. +- **Italy** and **Mexico** — specific standing country interests. + +## Vetoes + +- **US big-three sports** — professional and college basketball, American football, baseball, + and hockey. Game recaps, scores, trades, drafts, salary-cap news, betting lines. +- **Royal and celebrity gossip** — the British royal family, red carpets, box office, + influencer drama. +- **Horoscopes, listicles, product-roundup content.** + +### Explicitly NOT vetoed + +Connor is not anti-sport; he is anti *US-big-three-wire-recap*. These stay: + +- **Rock climbing, chess, the Olympics, oddball sports** — actively wanted. +- **Soccer** — he is trying to get into it. Match previews and live threads are noise, but + the sport is not blocked as a concept. This is the first thing to loosen if soccer feels + under-covered. +- **Sports governance and politics** — FIFA/UEFA corruption, Olympic host-city displacement, + Qatar migrant labour. These read as politics, not sport, and the old blocklist's + collisions with them were a bug. + +## The importance lane + +Independent of interest. This is the "I need to know this even though it isn't my topic" +channel, and it is half the reason the project exists. Its own small daily budget. + +- War and escalation — new military action between states, major escalation in an ongoing + conflict, nuclear or WMD events. +- Mass-casualty events and disasters — earthquakes, major attacks, industrial and transport + catastrophes. +- US federal shocks — constitutional crises, major Supreme Court rulings, shutdowns, sudden + sweeping policy moves. +- Economic breaks — market crashes, bank failures, currency crises, central bank surprises. +- Major US political events — significant election results, deaths of political figures. + +**Vetoes still apply on this lane.** In v0.2 the bypass lane ignored relevance entirely +(`gate.rs:302`), so a blocklisted item could still fire at priority 5. That must not recur. + +## Delivery + +- 2–5 pushes per day total, across both lanes. High bar. +- The daily digest is a **web page**, not a push payload. v0.2 sent a long markdown blob that + ntfy turned into a `.md` attachment, which is miserable to read on a phone. The push is + three lines and a link. +- ntfy feedback buttons **must work**. They never have — `NEWS_API_BASE_URL` was unset, so + they were silently dropped at boot. That is the entire training signal for everything above. + +## Open questions + +- Digest hour, and the exact quiet-hours window. +- Whether the importance lane may ever override quiet hours, and at what score. +- Whether "Global South" is a real standing interest or an artifact of a small sample. diff --git a/profile/profile.toml b/profile/profile.toml new file mode 100644 index 0000000..1424d0f --- /dev/null +++ b/profile/profile.toml @@ -0,0 +1,224 @@ +# Machine-readable form of profile.md. Every `probes` string is embedded into a +# vector; an item's score for a block is the MAX cosine over that block's probes, +# never the mean. Probes are written as prose resembling article text, because +# that is what they are compared against. +# +# Weights multiply the similarity. Feedback moves these vectors over time; the +# text here is the cold-start seed. + +schema_version = 1 + +# ---------------------------------------------------------------- interests -- + +[[interest]] +id = "geopolitics" +weight = 1.0 +probes = [ + "Escalation in the Middle East as strikes continue and diplomatic efforts stall.", + "China expands its influence abroad through investment, diplomacy and security agreements.", + "US foreign policy shifts as the administration reconsiders its posture toward an ally.", + "The United Nations Security Council debates sanctions and an arms embargo.", + "Ceasefire negotiations falter amid disputes over withdrawal and prisoner exchange.", +] + +[[interest]] +id = "tech-science-policy" +weight = 0.9 +probes = [ + "Regulators propose new rules governing artificial intelligence development and deployment.", + "A court ruling reshapes platform liability and online speech regulation.", + "Government surveillance powers and encryption policy face legal challenge.", + "Public research funding is cut, reshaping the scientific workforce.", + "Antitrust enforcement targets dominant technology firms.", +] + +[[interest]] +id = "labor-inequality" +weight = 0.9 +probes = [ + "Workers vote to unionize after a long organizing campaign; the employer contests the result.", + "A strike over wages and staffing shuts down operations across the sector.", + "New research documents widening wealth inequality and stagnant real wages.", + "Lawmakers debate expanding public healthcare, childcare and paid leave.", + "Tenants organize against rising rents as an eviction wave builds.", +] + +[[interest]] +id = "software-systems" +weight = 0.9 +push_eligible = true +probes = [ + "A new release of the systems programming language brings major changes to the compiler and toolchain.", + "Running large language models locally on consumer hardware: quantization, throughput and memory bandwidth.", + "Self-hosting a service stack with containers, reverse proxies and automated deployment.", + "A widely used open source project announces a breaking change, a license change, or a governance dispute.", + "WebAssembly gains new capabilities for running compiled code in the browser.", + "A serious vulnerability is disclosed in widely deployed infrastructure software.", +] + +[[interest]] +id = "urbanism" +weight = 0.8 +probes = [ + "The city council votes to reform zoning rules and allow denser housing near transit.", + "A new bus rapid transit and light rail line reshapes how residents move through the city.", + "Planners redesign streets to reduce traffic deaths and reclaim space from cars.", + "A downtown redevelopment project raises questions about displacement and public space.", +] + +[[interest]] +id = "energy-climate" +weight = 0.6 +probes = [ + "Grid operators struggle to connect new renewable generation as storage capacity lags demand.", + "A national climate policy sets emissions targets and restructures energy subsidies.", + "Conservation rules protecting endangered species and public land are rolled back.", +] + +# Confirmed by Connor 2026-09-08: wanted, at a level that keeps him up to date +# rather than saturating the feed. Also supported by three of eighteen starred +# items (Uber exiting Nigeria and Uganda, India's grid crunch, the Bolivian base +# explosions). +[[interest]] +id = "global-south" +weight = 0.7 +probes = [ + "An international company withdraws from several African markets amid regulatory pressure.", + "Elections and political upheaval reshape governments across Latin America.", + "Infrastructure and economic development across South Asia strains under rapid growth.", +] + +# ---------------------------------------------------------------- geography -- + +[[interest]] +id = "cincinnati" +weight = 0.9 +local = true +probes = [ + "Cincinnati city council debates a development, budget or policing decision affecting neighborhoods.", + "Local news from Cincinnati and Hamilton County, Ohio.", +] + +[[interest]] +id = "kentucky-louisville" +weight = 0.6 +local = true +probes = [ + "Louisville and Kentucky state politics, legislation and local government decisions.", +] + +[[interest]] +id = "us-national" +weight = 0.8 +probes = [ + "Congress advances major legislation after a contested floor fight.", + "The Supreme Court issues a ruling with sweeping consequences for federal policy.", + "Federal agencies are restructured by executive order, prompting legal challenges.", +] + +[[interest]] +id = "italy-mexico" +weight = 0.7 +probes = [ + "Italian politics, government coalitions and social policy in Italy.", + "Mexican politics, security policy and the relationship with the United States.", +] + +# ------------------------------------------------------------------- vetoes -- + +[[veto]] +id = "us-big-three-sports" +threshold = 0.55 +probes = [ + "The team beat their rivals in a game recap with final scores, statistics and player performances.", + "A player is traded, drafted or signs a contract; the league fines a franchise over salary cap rules.", + "Playoff standings, betting lines and a schedule of games on television this week.", + "Morales homers and drives in four as the Nationals top the Braves 9-5; Snell throws six shutout innings.", + "Box score summary: hits, runs, errors, pitching line and inning-by-inning scoring.", +] + +[[veto]] +id = "royal-celebrity-gossip" +threshold = 0.55 +probes = [ + "The royal family attends an event; speculation continues about the couple's status and future.", + "Celebrity red carpet appearances, box office returns and influencer controversy.", +] + +[[veto]] +id = "listicle-filler" +threshold = 0.60 +probes = [ + "Horoscopes, quizzes, and a roundup of the best products to buy this week.", +] + +# ------------------------------------------------------- the importance lane -- +# Independent of interest. Vetoes still apply here -- v0.2's bypass lane ignored +# relevance entirely (gate.rs:302), which let blocklisted items fire at priority 5. + +[[importance]] +id = "war-escalation" +weight = 1.0 +probes = [ + "One state launches military strikes on another, marking a major escalation of the conflict.", + "Nuclear facilities are targeted or a state signals willingness to use weapons of mass destruction.", +] + +[[importance]] +id = "mass-casualty" +weight = 1.0 +probes = [ + "A powerful earthquake kills hundreds and leaves many missing under collapsed buildings.", + "A bombing, shooting or industrial disaster causes mass casualties.", + "A passenger aircraft or train crash kills dozens.", +] + +[[importance]] +id = "us-federal-shock" +weight = 0.9 +probes = [ + "A constitutional crisis unfolds as the administration defies a court order.", + "The federal government shuts down after Congress fails to pass funding.", + "The Supreme Court overturns long-standing precedent.", +] + +[[importance]] +id = "economic-break" +weight = 0.9 +probes = [ + "Stock markets plunge in the steepest single-day fall in years as a major bank fails.", + "A currency collapses and the central bank intervenes with an emergency rate decision.", +] + +[[importance]] +id = "us-political-event" +weight = 0.9 +probes = [ + "A major election is called, delivering a decisive result and a shift in power.", + "A prominent political figure dies or resigns unexpectedly.", +] + +# ------------------------------------------------------- calibrated scoring -- +# These constants are not guesses. They were fitted on 2026-09-08 against all +# 3,885 items in the v0.2 database and Connor's 40 real feedback labels. +# See profile/CALIBRATION.md for the measurements. + +[scoring] +# Compare blocks by z-score WITHIN each block, never by raw cosine. Raw cosine +# is not comparable across blocks -- "geopolitics" averages 0.501 similarity to +# world-news text while "kentucky-louisville" averages 0.441 -- so a plain max +# let geopolitics win 78% of all items. Per-block z-scoring drops that to 19% +# and moves the median rank of a known-liked item from 1246 to 293 of 3885. +normalize = "zscore-per-block" + +# Veto is deliberately NOT a plain similarity threshold. A story can be +# genuinely sports-adjacent and still be wanted: "Belgian mayor blocks club from +# hosting Israeli opponent in Europa League" is Middle East politics, and both a +# substring blocklist AND a 0.55 cosine threshold falsely killed it. +# +# So: veto on unambiguous similarity, OR when the veto signal beats the best +# interest signal by a margin. Measured at these values: 17/22 of Connor's real +# rejections caught, 0/18 of his likes lost, ~21% of the corpus vetoed, and all +# four sports-adjacent news canaries survive. +veto_absolute = 0.60 +veto_margin_over_interest = 0.3 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..4b47540 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,28 @@ +[project] +name = "news" +version = "0.1.0" +description = "Personal news aggregator with an embedding-based model of taste" +requires-python = ">=3.12" +dependencies = [ + "httpx>=0.27", # conditional GET, async polling + "feedparser>=6.0", # RSS/Atom incl. RSS 1.0/RDF, which the old hand-rolled quick-xml parser could not read + "fastembed>=0.4", # bge-small-en-v1.5 via ONNX on CPU; deliberately not torch/sentence-transformers + "numpy>=1.26", + "starlette>=0.37", + "jinja2>=3.1", + "uvicorn>=0.30", + "python-multipart>=0.0.9", +] + +[project.scripts] +news = "news.cli:main" + +[dependency-groups] +dev = ["pytest>=8.0", "pytest-asyncio>=0.23", "respx>=0.21"] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.pytest.ini_options] +testpaths = ["tests"]