diff --git a/news/route.py b/news/route.py index 3ad9a85..456b73a 100644 --- a/news/route.py +++ b/news/route.py @@ -98,8 +98,15 @@ def route( group_used: dict[str, int] = {} def group_of(s: ClusterScore, kind: str) -> str: - block = s.interest.block if kind == "interest" else s.importance.block - return groups.get(block, block) + """Quota is per SUBJECT, so both lanes share one counter. + + Importance blocks describe a KIND of importance (war escalation, + economic break), not a topic -- so without this, two stories about the + same subject could each take a slot by matching different importance + blocks, which is exactly what happened: both importance alerts were + Middle East, one via war-escalation and one via economic-break. + """ + return groups.get(s.interest.block, s.interest.block) def group_full(s: ClusterScore, kind: str) -> bool: g = group_of(s, kind) diff --git a/news/score.py b/news/score.py index fb9972d..259f139 100644 --- a/news/score.py +++ b/news/score.py @@ -95,6 +95,13 @@ class ProfileVectors: return float(block.get("weight", 1.0)) return 1.0 + def group(self, kind: str, block_id: str) -> str: + """Quota/normalisation group for a block, defaulting to its own id.""" + for bid, _, block in self.blocks[kind]: + if bid == block_id: + return block.get("group", bid) + return block_id + def block(self, kind: str, block_id: str) -> dict: for bid, _, b in self.blocks[kind]: if bid == block_id: @@ -166,12 +173,32 @@ def score_clusters( demote_raw = profile_vectors.raw_matrix("demote", mat) norm = NORMALIZERS.get(scoring_cfg.get("normalize", "zscore-per-block"), _z) - interest_z = { - b: norm(v) * profile_vectors.weight("interest", b) for b, v in interest_raw.items() - } - importance_z = { - b: norm(v) * profile_vectors.weight("importance", b) for b, v in importance_raw.items() - } + + def normalise(kind: str, raw: dict[str, np.ndarray]) -> dict[str, np.ndarray]: + """Normalise per GROUP, not per block. + + Taking the max across blocks gives a subject one draw per block, so a + topic split into five blocks beats one covered by a single block purely + by having more chances. Measured: this reader has six foreign-affairs + blocks out of 23, and foreign affairs took 70% of the digest while + software-systems -- a quarter of the corpus -- took none. + + Collapsing to the group first, THEN normalising, prices that in: the + group's distribution already reflects being a max over its members, so + normalisation removes the inflation instead of rewarding it. + """ + by_group: dict[str, np.ndarray] = {} + for block_id, values in raw.items(): + g = profile_vectors.group(kind, block_id) + by_group[g] = values if g not in by_group else np.maximum(by_group[g], values) + weights = {} + for block_id in raw: + g = profile_vectors.group(kind, block_id) + weights[g] = max(weights.get(g, 0.0), profile_vectors.weight(kind, block_id)) + return {g: norm(v) * weights[g] for g, v in by_group.items()} + + interest_z = normalise("interest", interest_raw) + importance_z = normalise("importance", importance_raw) # Veto stays on z-scores: it is a comparison against the interest signal, # and both sides must be on the same scale. veto_z = {b: norm(v) for b, v in veto_raw.items()} @@ -180,9 +207,15 @@ def score_clusters( return max(((b, float(v[i])) for b, v in per_block.items()), key=lambda kv: kv[1]) out: list[ClusterScore] = [] + def best_block_in(kind: str, group: str, raw: dict[str, np.ndarray], i: int) -> str: + members = [b for b in raw if profile_vectors.group(kind, b) == group] + return max(members, key=lambda b: float(raw[b][i])) if members else group + for i, row in enumerate(rows): - i_block, i_z = argbest(interest_z, i) - m_block, m_z = argbest(importance_z, i) + i_group, i_z = argbest(interest_z, i) + m_group, m_z = argbest(importance_z, i) + i_block = best_block_in("interest", i_group, interest_raw, i) + m_block = best_block_in("importance", m_group, importance_raw, i) # --- veto: absolute similarity OR a margin over the interest signal --- veto_hit = None diff --git a/profile/config.toml b/profile/config.toml index 1394800..ce96677 100644 --- a/profile/config.toml +++ b/profile/config.toml @@ -50,6 +50,15 @@ syndication_threshold = 0.97 # Compare blocks by z-score WITHIN each block, never by raw cosine: cosine # magnitude is not comparable across blocks, and a plain max let one block win # 78% of all items. See CALIBRATION.md. +# "zscore-per-block" or "top-anchored". Both normalise per GROUP (see `group` +# in profile.toml), not per block: taking the max across blocks gives a subject +# one draw per block, so a topic split into five blocks beats one covered by a +# single block purely by having more chances. +# +# NOTE the thresholds in [routing] are in z-score units. "top-anchored" puts +# scores on a different scale (1.0 = that group's 99th percentile), so switching +# normalizer REQUIRES re-tuning them -- verified: leaving them at 2.0 silences +# the interest lane completely. normalize = "zscore-per-block" # Veto is not a plain similarity threshold. A story can be genuinely adjacent to diff --git a/profile/profile.toml b/profile/profile.toml index b391ba9..2d01b02 100644 --- a/profile/profile.toml +++ b/profile/profile.toml @@ -369,6 +369,17 @@ probes = [ "This week in review: the stories we covered, collected.", ] +# Release feeds publish nightly and prerelease builds continuously. They match +# software-systems correctly and are worthless as notifications -- the first +# alert this block ever won was "Nvim development (prerelease) build". +[[demote]] +id = "prerelease-build" +penalty = 1.5 +probes = [ + "Nightly development prerelease build, automatically generated from the main branch.", + "Release candidate rc1: pre-release for testing, not recommended for production.", +] + # ============================================= the importance lane =========== # Independent of interest. Vetoes still apply here -- v0.2's bypass lane ignored # relevance entirely (gate.rs:302), which let blocklisted items fire at priority 5.