Gate place-based blocks on a literal mention

Connor spotted a Missouri redistricting story labelled ohio-state-politics.
Measuring showed that was not an isolated slip: 66% of the winners of a
place-based block named no place at all. Glastonbury, Anthony Albanese and a
Tim Montgomerie story were all scoring as italy-mexico; a singing-apraxia story
was ohio-state-politics.

Embeddings model meaning well and named entities poorly -- "Italian politics"
and "Australian politics" are nearly the same vector. Place is identity, not
meaning, so it is matched lexically, which is the same division of labour that
makes lexical comparison right for syndication detection and wrong for
clustering. Blocks carry `requires_any`; a block cannot win for an item whose
text names none of its terms.

Word boundaries, not substrings: "rome" is inside "chrome" and "mali" inside
"normalise". That is the trap the old blocklist fell into with "nfl" inside
"conflict".

The first implementation was wrong in a way worth recording, because it looked
correct and measured almost unchanged (66% -> 30%). It wrote a sentinel into the
RAW score before normalisation -- but when most items are gated, that sentinel
becomes the block's own mean, so a gated item z-scored to roughly zero and could
still win. Normalisation erased the gate. Applied after normalisation instead:
66% -> 0%, and place-block winners fell from 140 to 10, so 130 items now land in
the block they actually belong to.

news/score.py

@@ -14,6 +14,7 @@ so a plain max hands one block 78% of all items. See profile/CALIBRATION.md.
 from __future__ import annotations
 
 import math
+import re
 import sqlite3
 from datetime import datetime
 from dataclasses import dataclass, field
@@ -109,6 +110,20 @@ class ProfileVectors:
         return {}
 
 
+def _gate_terms(block: dict) -> list[str]:
+    return [t.lower() for t in block.get("requires_any", [])]
+
+
+def _mentions(text: str, terms: list[str]) -> bool:
+    """Word-boundary match, never substring.
+
+    The old system's blocklist matched substrings and so vetoed every Iran-war
+    story because "nfl" appears inside "conflict". Same trap here: "rome" is
+    inside "chrome", "mali" inside "normalise".
+    """
+    return any(re.search(rf"\b{re.escape(t)}\b", text) for t in terms)
+
+
 def _z(values: np.ndarray) -> np.ndarray:
     """Z-score, safe when a block is flat across the population."""
     std = float(values.std())
@@ -197,7 +212,47 @@ def score_clusters(
             weights[g] = max(weights.get(g, 0.0), profile_vectors.weight(kind, block_id))
         return {g: norm(v) * weights[g] for g, v in by_group.items()}
 
+    # Place gate. Embeddings model meaning well and named entities poorly:
+    # "Italian politics" and "Australian politics" sit almost on top of each
+    # other, so 66% of the winners of a place-based block never named the place
+    # -- Glastonbury and Anthony Albanese were scoring as italy-mexico.
+    #
+    # Place is identity, not meaning, so it is gated lexically. A block with
+    # `requires_any` cannot win for an item whose text names none of its terms.
+    gated: dict[str, set[int]] = {}
+    texts: list[str] = []
+    ids = [r["id"] for r in rows]
+    member_titles: dict[str, list[str]] = {}
+    if ids:
+        marks = ",".join("?" * len(ids))
+        for m in con.execute(
+            f"""SELECT cm.cluster_id AS cid, i.title FROM cluster_members cm
+                JOIN items i ON i.id = cm.item_id
+                WHERE cm.cluster_id IN ({marks})""", ids):
+            member_titles.setdefault(m["cid"], []).append(m["title"])
+    for r in rows:
+        texts.append(" ".join([r["lead"] or ""] + member_titles.get(r["id"], [])).lower())
+    for block_id, _, block in profile_vectors.blocks["interest"]:
+        terms = _gate_terms(block)
+        if terms:
+            gated[block_id] = {i for i, t in enumerate(texts) if not _mentions(t, terms)}
+
     interest_z = normalise("interest", interest_raw)
+
+    # The gate is applied AFTER normalisation, on purpose. Setting the raw score
+    # to a sentinel first does not work: when most items are gated, that
+    # sentinel becomes the block's own MEAN, so a gated item z-scores to roughly
+    # zero and can still win. Normalisation erases a pre-normalisation gate.
+    for block_id, blocked in gated.items():
+        group = profile_vectors.group("interest", block_id)
+        # Only gate a group whose members ALL require a mention; otherwise an
+        # ungated sibling in the same group would be silenced along with it.
+        siblings = [b for b, _, _ in profile_vectors.blocks["interest"]
+                    if profile_vectors.group("interest", b) == group]
+        if not all(_gate_terms(profile_vectors.block("interest", b)) for b in siblings):
+            continue
+        for i in blocked:
+            interest_z[group][i] = -np.inf
     importance_z = normalise("importance", importance_raw)
     # Veto stays on z-scores: it is a comparison against the interest signal,
     # and both sides must be on the same scale.

profile/profile.toml

@@ -204,6 +204,13 @@ probes = [
   "Federal agencies are restructured by executive order, prompting legal challenges.",
 ]
 
+# `requires_any` gates a block on a literal, word-boundary mention. Embeddings
+# model meaning well and named entities poorly -- "Italian politics" and
+# "Australian politics" are nearly the same vector -- so 66% of the winners of a
+# place block named no place at all: Glastonbury and Anthony Albanese were
+# scoring as italy-mexico, and a singing-apraxia story as ohio-state-politics.
+# Place is identity, not meaning, so it is matched lexically.
+#
 # --- place -------------------------------------------------------------------
 [[interest]]
 id = "urbanism"
@@ -217,6 +224,7 @@ probes = [
 
 [[interest]]
 id = "cincinnati"
+requires_any = ["cincinnati", "hamilton county", "northern kentucky", "covington", "newport", "over-the-rhine", "ohio"]
 group = "local"
 weight = 0.9
 local = true
@@ -230,6 +238,7 @@ probes = [
 
 [[interest]]
 id = "kentucky-louisville"
+requires_any = ["kentucky", "louisville", "frankfort", "lexington"]
 group = "local"
 weight = 0.6
 local = true
@@ -241,6 +250,7 @@ probes = [
 # coverage alongside Cincinnati city government.
 [[interest]]
 id = "ohio-state-politics"
+requires_any = ["ohio", "columbus", "cleveland", "toledo", "dayton", "akron", "statehouse"]
 group = "local"
 weight = 0.75
 local = true
@@ -257,6 +267,7 @@ probes = [
 # the consumer-lifestyle veto must not swallow it.
 [[interest]]
 id = "italy-mexico"
+requires_any = ["italy", "italian", "rome", "milan", "naples", "sicily", "meloni", "mexico", "mexican", "mexico city", "sheinbaum", "guadalajara", "monterrey"]
 weight = 0.75
 probes = [
   "Italian politics, government coalitions and social policy in Italy.",

tests/test_place_gate.py

@@ -0,0 +1,105 @@
+"""Place gating.
+
+Embeddings model meaning well and named entities poorly: "Italian politics" and
+"Australian politics" sit almost on top of each other. Measured on live data,
+66% of the winners of a place-based block named no place at all -- Glastonbury
+and Anthony Albanese were scoring as `italy-mexico`.
+
+Place is identity, not meaning, so it is matched lexically.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+import pytest
+
+from news import store
+from news.score import ProfileVectors, _mentions, score_clusters
+
+
+def test_word_boundaries_not_substrings():
+    """The lesson from "nfl" matching co-NFL-ict, which vetoed every Iran story."""
+    assert _mentions("meloni visits rome today", ["rome"])
+    assert not _mentions("open the chrome browser", ["rome"])
+    assert not _mentions("efforts to normalise ties", ["mali"])
+    assert _mentions("fighting spreads in mali", ["mali"])
+
+
+class _FakeVectors:
+    """Two blocks in different groups: one gated on a place, one ungated."""
+
+    def __init__(self):
+        near = np.array([[1.0, 0.0]], dtype=np.float32)
+        self.blocks = {
+            "interest": [
+                ("italy-mexico", near, {"id": "italy-mexico", "weight": 1.0,
+                                        "requires_any": ["italy", "mexico"]}),
+                ("urbanism", np.array([[0.0, 1.0]], dtype=np.float32),
+                 {"id": "urbanism", "weight": 1.0}),
+            ],
+            "veto": [], "demote": [], "importance": [
+                ("x", np.array([[0.0, 0.0]], dtype=np.float32), {"id": "x", "weight": 1.0})],
+        }
+
+    def raw_matrix(self, kind, vecs):
+        return {bid: (vecs @ m.T).max(axis=1) for bid, m, _ in self.blocks[kind]}
+
+    def weight(self, kind, bid):
+        return 1.0
+
+    def group(self, kind, bid):
+        return bid
+
+    def block(self, kind, bid):
+        return next(b for i, _, b in self.blocks[kind] if i == bid)
+
+
+@pytest.fixture
+def con(tmp_path):
+    c = store.connect(tmp_path / "g.db")
+    store.upsert_source(c, id="s1", name="A", url="http://a", kind="rss", bucket="a",
+                        weight=1.0, prominence=0, enabled=1, poll_interval_secs=900)
+    return c
+
+
+def _add(con, cid, title):
+    when = "2026-01-01T00:00:00+00:00"
+    con.execute("""INSERT INTO items (id,source_id,title,summary,url,canonical_url,
+                   published_at,first_seen_at,content_hash) VALUES (?,?,?,'',?,?,?,?,?)""",
+                (cid, "s1", title, f"http://x/{cid}", f"http://x/{cid}", when, when, cid))
+    con.execute("""INSERT INTO clusters (id,centroid,size,first_seen_at,last_seen_at,lead_item_id)
+                   VALUES (?,?,1,?,?,?)""",
+                (cid, store.to_blob(np.array([1.0, 0.0], dtype=np.float32)), when, when, cid))
+    con.execute("INSERT INTO cluster_members VALUES (?,?,?)", (cid, cid, when))
+
+
+CFG = {"veto_absolute": 0.9, "veto_margin_over_interest": 9.0,
+       "corroboration_pickup_weight": 0.3, "corroboration_depth_weight": 0.2}
+CLUST = {"syndication_threshold": 0.97}
+
+
+def test_a_gated_block_cannot_win_without_a_mention(con):
+    """Both items point at the italy-mexico probe; only one names a place."""
+    _add(con, "yes", "Meloni's government becomes Italy's longest-lasting")
+    _add(con, "no", "Anthony Albanese held an undeclared honorary membership")
+    scores = {s.cluster_id: s for s in score_clusters(con, _FakeVectors(), CFG, CLUST)}
+    assert scores["yes"].interest.block == "italy-mexico"
+    assert scores["no"].interest.block != "italy-mexico"
+
+
+def test_gate_survives_normalisation(con):
+    """Regression for a real bug.
+
+    The gate was first applied by writing a sentinel into the RAW score before
+    normalisation. When most items are gated that sentinel becomes the block's
+    own mean, so a gated item z-scored to roughly zero and could still win --
+    the normalisation erased the gate. With 30 gated items against 1 mention,
+    a pre-normalisation gate lets the gated ones through.
+    """
+    _add(con, "mention", "Sheinbaum speaks in Mexico")
+    for i in range(30):
+        _add(con, f"other{i}", f"Unrelated story number {i}")
+    scores = score_clusters(con, _FakeVectors(), CFG, CLUST)
+    winners = [s for s in scores if s.interest.block == "italy-mexico"]
+    assert [s.cluster_id for s in winners] == ["mention"], (
+        "only the item naming a place may win the gated block")