connor
e99b7c9fb7
Connor spotted a Missouri redistricting story labelled ohio-state-politics. Measuring showed that was not an isolated slip: 66% of the winners of a place-based block named no place at all. Glastonbury, Anthony Albanese and a Tim Montgomerie story were all scoring as italy-mexico; a singing-apraxia story was ohio-state-politics. Embeddings model meaning well and named entities poorly -- "Italian politics" and "Australian politics" are nearly the same vector. Place is identity, not meaning, so it is matched lexically, which is the same division of labour that makes lexical comparison right for syndication detection and wrong for clustering. Blocks carry `requires_any`; a block cannot win for an item whose text names none of its terms. Word boundaries, not substrings: "rome" is inside "chrome" and "mali" inside "normalise". That is the trap the old blocklist fell into with "nfl" inside "conflict". The first implementation was wrong in a way worth recording, because it looked correct and measured almost unchanged (66% -> 30%). It wrote a sentinel into the RAW score before normalisation -- but when most items are gated, that sentinel becomes the block's own mean, so a gated item z-scored to roughly zero and could still win. Normalisation erased the gate. Applied after normalisation instead: 66% -> 0%, and place-block winners fell from 140 to 10, so 130 items now land in the block they actually belong to.
106 lines
4.3 KiB
Python
106 lines
4.3 KiB
Python
"""Place gating.
|
|
|
|
Embeddings model meaning well and named entities poorly: "Italian politics" and
|
|
"Australian politics" sit almost on top of each other. Measured on live data,
|
|
66% of the winners of a place-based block named no place at all -- Glastonbury
|
|
and Anthony Albanese were scoring as `italy-mexico`.
|
|
|
|
Place is identity, not meaning, so it is matched lexically.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import numpy as np
|
|
import pytest
|
|
|
|
from news import store
|
|
from news.score import ProfileVectors, _mentions, score_clusters
|
|
|
|
|
|
def test_word_boundaries_not_substrings():
|
|
"""The lesson from "nfl" matching co-NFL-ict, which vetoed every Iran story."""
|
|
assert _mentions("meloni visits rome today", ["rome"])
|
|
assert not _mentions("open the chrome browser", ["rome"])
|
|
assert not _mentions("efforts to normalise ties", ["mali"])
|
|
assert _mentions("fighting spreads in mali", ["mali"])
|
|
|
|
|
|
class _FakeVectors:
|
|
"""Two blocks in different groups: one gated on a place, one ungated."""
|
|
|
|
def __init__(self):
|
|
near = np.array([[1.0, 0.0]], dtype=np.float32)
|
|
self.blocks = {
|
|
"interest": [
|
|
("italy-mexico", near, {"id": "italy-mexico", "weight": 1.0,
|
|
"requires_any": ["italy", "mexico"]}),
|
|
("urbanism", np.array([[0.0, 1.0]], dtype=np.float32),
|
|
{"id": "urbanism", "weight": 1.0}),
|
|
],
|
|
"veto": [], "demote": [], "importance": [
|
|
("x", np.array([[0.0, 0.0]], dtype=np.float32), {"id": "x", "weight": 1.0})],
|
|
}
|
|
|
|
def raw_matrix(self, kind, vecs):
|
|
return {bid: (vecs @ m.T).max(axis=1) for bid, m, _ in self.blocks[kind]}
|
|
|
|
def weight(self, kind, bid):
|
|
return 1.0
|
|
|
|
def group(self, kind, bid):
|
|
return bid
|
|
|
|
def block(self, kind, bid):
|
|
return next(b for i, _, b in self.blocks[kind] if i == bid)
|
|
|
|
|
|
@pytest.fixture
|
|
def con(tmp_path):
|
|
c = store.connect(tmp_path / "g.db")
|
|
store.upsert_source(c, id="s1", name="A", url="http://a", kind="rss", bucket="a",
|
|
weight=1.0, prominence=0, enabled=1, poll_interval_secs=900)
|
|
return c
|
|
|
|
|
|
def _add(con, cid, title):
|
|
when = "2026-01-01T00:00:00+00:00"
|
|
con.execute("""INSERT INTO items (id,source_id,title,summary,url,canonical_url,
|
|
published_at,first_seen_at,content_hash) VALUES (?,?,?,'',?,?,?,?,?)""",
|
|
(cid, "s1", title, f"http://x/{cid}", f"http://x/{cid}", when, when, cid))
|
|
con.execute("""INSERT INTO clusters (id,centroid,size,first_seen_at,last_seen_at,lead_item_id)
|
|
VALUES (?,?,1,?,?,?)""",
|
|
(cid, store.to_blob(np.array([1.0, 0.0], dtype=np.float32)), when, when, cid))
|
|
con.execute("INSERT INTO cluster_members VALUES (?,?,?)", (cid, cid, when))
|
|
|
|
|
|
CFG = {"veto_absolute": 0.9, "veto_margin_over_interest": 9.0,
|
|
"corroboration_pickup_weight": 0.3, "corroboration_depth_weight": 0.2}
|
|
CLUST = {"syndication_threshold": 0.97}
|
|
|
|
|
|
def test_a_gated_block_cannot_win_without_a_mention(con):
|
|
"""Both items point at the italy-mexico probe; only one names a place."""
|
|
_add(con, "yes", "Meloni's government becomes Italy's longest-lasting")
|
|
_add(con, "no", "Anthony Albanese held an undeclared honorary membership")
|
|
scores = {s.cluster_id: s for s in score_clusters(con, _FakeVectors(), CFG, CLUST)}
|
|
assert scores["yes"].interest.block == "italy-mexico"
|
|
assert scores["no"].interest.block != "italy-mexico"
|
|
|
|
|
|
def test_gate_survives_normalisation(con):
|
|
"""Regression for a real bug.
|
|
|
|
The gate was first applied by writing a sentinel into the RAW score before
|
|
normalisation. When most items are gated that sentinel becomes the block's
|
|
own mean, so a gated item z-scored to roughly zero and could still win --
|
|
the normalisation erased the gate. With 30 gated items against 1 mention,
|
|
a pre-normalisation gate lets the gated ones through.
|
|
"""
|
|
_add(con, "mention", "Sheinbaum speaks in Mexico")
|
|
for i in range(30):
|
|
_add(con, f"other{i}", f"Unrelated story number {i}")
|
|
scores = score_clusters(con, _FakeVectors(), CFG, CLUST)
|
|
winners = [s for s in scores if s.interest.block == "italy-mexico"]
|
|
assert [s.cluster_id for s in winners] == ["mention"], (
|
|
"only the item naming a place may win the gated block")
|