diff --git a/news/score.py b/news/score.py index 259f139..098f240 100644 --- a/news/score.py +++ b/news/score.py @@ -14,6 +14,7 @@ so a plain max hands one block 78% of all items. See profile/CALIBRATION.md. from __future__ import annotations import math +import re import sqlite3 from datetime import datetime from dataclasses import dataclass, field @@ -109,6 +110,20 @@ class ProfileVectors: return {} +def _gate_terms(block: dict) -> list[str]: + return [t.lower() for t in block.get("requires_any", [])] + + +def _mentions(text: str, terms: list[str]) -> bool: + """Word-boundary match, never substring. + + The old system's blocklist matched substrings and so vetoed every Iran-war + story because "nfl" appears inside "conflict". Same trap here: "rome" is + inside "chrome", "mali" inside "normalise". + """ + return any(re.search(rf"\b{re.escape(t)}\b", text) for t in terms) + + def _z(values: np.ndarray) -> np.ndarray: """Z-score, safe when a block is flat across the population.""" std = float(values.std()) @@ -197,7 +212,47 @@ def score_clusters( weights[g] = max(weights.get(g, 0.0), profile_vectors.weight(kind, block_id)) return {g: norm(v) * weights[g] for g, v in by_group.items()} + # Place gate. Embeddings model meaning well and named entities poorly: + # "Italian politics" and "Australian politics" sit almost on top of each + # other, so 66% of the winners of a place-based block never named the place + # -- Glastonbury and Anthony Albanese were scoring as italy-mexico. + # + # Place is identity, not meaning, so it is gated lexically. A block with + # `requires_any` cannot win for an item whose text names none of its terms. + gated: dict[str, set[int]] = {} + texts: list[str] = [] + ids = [r["id"] for r in rows] + member_titles: dict[str, list[str]] = {} + if ids: + marks = ",".join("?" * len(ids)) + for m in con.execute( + f"""SELECT cm.cluster_id AS cid, i.title FROM cluster_members cm + JOIN items i ON i.id = cm.item_id + WHERE cm.cluster_id IN ({marks})""", ids): + member_titles.setdefault(m["cid"], []).append(m["title"]) + for r in rows: + texts.append(" ".join([r["lead"] or ""] + member_titles.get(r["id"], [])).lower()) + for block_id, _, block in profile_vectors.blocks["interest"]: + terms = _gate_terms(block) + if terms: + gated[block_id] = {i for i, t in enumerate(texts) if not _mentions(t, terms)} + interest_z = normalise("interest", interest_raw) + + # The gate is applied AFTER normalisation, on purpose. Setting the raw score + # to a sentinel first does not work: when most items are gated, that + # sentinel becomes the block's own MEAN, so a gated item z-scores to roughly + # zero and can still win. Normalisation erases a pre-normalisation gate. + for block_id, blocked in gated.items(): + group = profile_vectors.group("interest", block_id) + # Only gate a group whose members ALL require a mention; otherwise an + # ungated sibling in the same group would be silenced along with it. + siblings = [b for b, _, _ in profile_vectors.blocks["interest"] + if profile_vectors.group("interest", b) == group] + if not all(_gate_terms(profile_vectors.block("interest", b)) for b in siblings): + continue + for i in blocked: + interest_z[group][i] = -np.inf importance_z = normalise("importance", importance_raw) # Veto stays on z-scores: it is a comparison against the interest signal, # and both sides must be on the same scale. diff --git a/profile/profile.toml b/profile/profile.toml index 2d01b02..d713c93 100644 --- a/profile/profile.toml +++ b/profile/profile.toml @@ -204,6 +204,13 @@ probes = [ "Federal agencies are restructured by executive order, prompting legal challenges.", ] +# `requires_any` gates a block on a literal, word-boundary mention. Embeddings +# model meaning well and named entities poorly -- "Italian politics" and +# "Australian politics" are nearly the same vector -- so 66% of the winners of a +# place block named no place at all: Glastonbury and Anthony Albanese were +# scoring as italy-mexico, and a singing-apraxia story as ohio-state-politics. +# Place is identity, not meaning, so it is matched lexically. +# # --- place ------------------------------------------------------------------- [[interest]] id = "urbanism" @@ -217,6 +224,7 @@ probes = [ [[interest]] id = "cincinnati" +requires_any = ["cincinnati", "hamilton county", "northern kentucky", "covington", "newport", "over-the-rhine", "ohio"] group = "local" weight = 0.9 local = true @@ -230,6 +238,7 @@ probes = [ [[interest]] id = "kentucky-louisville" +requires_any = ["kentucky", "louisville", "frankfort", "lexington"] group = "local" weight = 0.6 local = true @@ -241,6 +250,7 @@ probes = [ # coverage alongside Cincinnati city government. [[interest]] id = "ohio-state-politics" +requires_any = ["ohio", "columbus", "cleveland", "toledo", "dayton", "akron", "statehouse"] group = "local" weight = 0.75 local = true @@ -257,6 +267,7 @@ probes = [ # the consumer-lifestyle veto must not swallow it. [[interest]] id = "italy-mexico" +requires_any = ["italy", "italian", "rome", "milan", "naples", "sicily", "meloni", "mexico", "mexican", "mexico city", "sheinbaum", "guadalajara", "monterrey"] weight = 0.75 probes = [ "Italian politics, government coalitions and social policy in Italy.", diff --git a/tests/test_place_gate.py b/tests/test_place_gate.py new file mode 100644 index 0000000..1a8d485 --- /dev/null +++ b/tests/test_place_gate.py @@ -0,0 +1,105 @@ +"""Place gating. + +Embeddings model meaning well and named entities poorly: "Italian politics" and +"Australian politics" sit almost on top of each other. Measured on live data, +66% of the winners of a place-based block named no place at all -- Glastonbury +and Anthony Albanese were scoring as `italy-mexico`. + +Place is identity, not meaning, so it is matched lexically. +""" + +from __future__ import annotations + +import numpy as np +import pytest + +from news import store +from news.score import ProfileVectors, _mentions, score_clusters + + +def test_word_boundaries_not_substrings(): + """The lesson from "nfl" matching co-NFL-ict, which vetoed every Iran story.""" + assert _mentions("meloni visits rome today", ["rome"]) + assert not _mentions("open the chrome browser", ["rome"]) + assert not _mentions("efforts to normalise ties", ["mali"]) + assert _mentions("fighting spreads in mali", ["mali"]) + + +class _FakeVectors: + """Two blocks in different groups: one gated on a place, one ungated.""" + + def __init__(self): + near = np.array([[1.0, 0.0]], dtype=np.float32) + self.blocks = { + "interest": [ + ("italy-mexico", near, {"id": "italy-mexico", "weight": 1.0, + "requires_any": ["italy", "mexico"]}), + ("urbanism", np.array([[0.0, 1.0]], dtype=np.float32), + {"id": "urbanism", "weight": 1.0}), + ], + "veto": [], "demote": [], "importance": [ + ("x", np.array([[0.0, 0.0]], dtype=np.float32), {"id": "x", "weight": 1.0})], + } + + def raw_matrix(self, kind, vecs): + return {bid: (vecs @ m.T).max(axis=1) for bid, m, _ in self.blocks[kind]} + + def weight(self, kind, bid): + return 1.0 + + def group(self, kind, bid): + return bid + + def block(self, kind, bid): + return next(b for i, _, b in self.blocks[kind] if i == bid) + + +@pytest.fixture +def con(tmp_path): + c = store.connect(tmp_path / "g.db") + store.upsert_source(c, id="s1", name="A", url="http://a", kind="rss", bucket="a", + weight=1.0, prominence=0, enabled=1, poll_interval_secs=900) + return c + + +def _add(con, cid, title): + when = "2026-01-01T00:00:00+00:00" + con.execute("""INSERT INTO items (id,source_id,title,summary,url,canonical_url, + published_at,first_seen_at,content_hash) VALUES (?,?,?,'',?,?,?,?,?)""", + (cid, "s1", title, f"http://x/{cid}", f"http://x/{cid}", when, when, cid)) + con.execute("""INSERT INTO clusters (id,centroid,size,first_seen_at,last_seen_at,lead_item_id) + VALUES (?,?,1,?,?,?)""", + (cid, store.to_blob(np.array([1.0, 0.0], dtype=np.float32)), when, when, cid)) + con.execute("INSERT INTO cluster_members VALUES (?,?,?)", (cid, cid, when)) + + +CFG = {"veto_absolute": 0.9, "veto_margin_over_interest": 9.0, + "corroboration_pickup_weight": 0.3, "corroboration_depth_weight": 0.2} +CLUST = {"syndication_threshold": 0.97} + + +def test_a_gated_block_cannot_win_without_a_mention(con): + """Both items point at the italy-mexico probe; only one names a place.""" + _add(con, "yes", "Meloni's government becomes Italy's longest-lasting") + _add(con, "no", "Anthony Albanese held an undeclared honorary membership") + scores = {s.cluster_id: s for s in score_clusters(con, _FakeVectors(), CFG, CLUST)} + assert scores["yes"].interest.block == "italy-mexico" + assert scores["no"].interest.block != "italy-mexico" + + +def test_gate_survives_normalisation(con): + """Regression for a real bug. + + The gate was first applied by writing a sentinel into the RAW score before + normalisation. When most items are gated that sentinel becomes the block's + own mean, so a gated item z-scored to roughly zero and could still win -- + the normalisation erased the gate. With 30 gated items against 1 mention, + a pre-normalisation gate lets the gated ones through. + """ + _add(con, "mention", "Sheinbaum speaks in Mexico") + for i in range(30): + _add(con, f"other{i}", f"Unrelated story number {i}") + scores = score_clusters(con, _FakeVectors(), CFG, CLUST) + winners = [s for s in scores if s.interest.block == "italy-mexico"] + assert [s.cluster_id for s in winners] == ["mention"], ( + "only the item naming a place may win the gated block")