diff --git a/news/cli.py b/news/cli.py index 9a50c24..1fffadb 100644 --- a/news/cli.py +++ b/news/cli.py @@ -113,7 +113,7 @@ def cmd_simulate(args) -> int: routing["important_threshold"] = args.important_threshold now = datetime.now(UTC) - decisions = route(con, scores, routing, cfg.delivery, now) + decisions = route(con, scores, routing, cfg.delivery, now, cfg.block_groups()) by_lane = {} for d in decisions: by_lane.setdefault(d.lane, []).append(d) diff --git a/news/config.py b/news/config.py index b70ab7f..18ab40c 100644 --- a/news/config.py +++ b/news/config.py @@ -106,6 +106,7 @@ class Config: r = self.section("routing") return { "interest_per_day": r.get("interest_per_day", 3), + "max_alerts_per_group_per_day": r.get("max_alerts_per_group_per_day", 1), "important_per_day": r.get("important_per_day", 2), "important_per_night": r.get("important_per_night", 1), "interest_threshold": r.get("interest_threshold", 2.0), @@ -136,6 +137,15 @@ class Config: def profile(self) -> dict[str, Any]: return _load(self.profile_path) + def block_groups(self) -> dict[str, str]: + """block id -> quota group, defaulting to the block's own id.""" + profile = self.profile() + out: dict[str, str] = {} + for kind in ("interest", "importance"): + for b in profile.get(kind, []): + out[b["id"]] = b.get("group", b["id"]) + return out + @property def delivery(self) -> dict[str, Any]: return self.profile().get("delivery", {}) diff --git a/news/route.py b/news/route.py index 4002fe9..3ad9a85 100644 --- a/news/route.py +++ b/news/route.py @@ -64,6 +64,7 @@ def route( routing: dict, delivery: dict, now: datetime, + groups: dict[str, str] | None = None, ) -> list[Decision]: """Decide a lane for every scored cluster. @@ -71,6 +72,7 @@ def route( budget is spent on the best candidates rather than on whatever arrived first -- the ordering bug that let the old system's budget go to noise. """ + groups = groups or {} quiet = delivery.get("quiet_hours", {}) quiet_now = in_quiet_hours(now, quiet) may_wake = bool(delivery.get("importance_may_wake", True)) @@ -89,6 +91,24 @@ def route( ) } + # One subject must not take every slot. Measured before this existed: five of + # five alerts were Middle East, though Middle East blocks are only ~11% of + # the corpus and software-systems alone is 25%. + per_group = routing.get("max_alerts_per_group_per_day", 1) + group_used: dict[str, int] = {} + + def group_of(s: ClusterScore, kind: str) -> str: + block = s.interest.block if kind == "interest" else s.importance.block + return groups.get(block, block) + + def group_full(s: ClusterScore, kind: str) -> bool: + g = group_of(s, kind) + return group_used.get(g, 0) >= per_group + + def take_group(s: ClusterScore, kind: str) -> None: + g = group_of(s, kind) + group_used[g] = group_used.get(g, 0) + 1 + decisions: dict[str, Decision] = {} # Deferred items are digest CANDIDATES, not digest members. Everything that # misses an alert lane competes for the same capped digest, so a quiet-hours @@ -132,12 +152,16 @@ def route( if important_budget <= 0: defer(s, "important-budget-exhausted") continue + if group_full(s, "importance"): + defer(s, f"group-quota:{group_of(s, 'importance')}") + continue if quiet_now: if not may_wake or night_budget <= 0: defer(s, "quiet-hours") continue night_budget -= 1 important_budget -= 1 + take_group(s, "importance") settle(s, IMPORTANT, f"importance:{s.importance.block}", s.importance_score) # --- interest lane: never overrides quiet hours --- @@ -155,7 +179,11 @@ def route( if interest_budget <= 0: defer(s, "interest-budget-exhausted") continue + if group_full(s, "interest"): + defer(s, f"group-quota:{group_of(s, 'interest')}") + continue interest_budget -= 1 + take_group(s, "interest") settle(s, INTEREST, f"interest:{s.interest.block}", s.interest_score) # --- one capped, ranked digest --- diff --git a/news/score.py b/news/score.py index a5e9d33..fb9972d 100644 --- a/news/score.py +++ b/news/score.py @@ -110,6 +110,29 @@ def _z(values: np.ndarray) -> np.ndarray: return (values - float(values.mean())) / std +def _top_anchored(values: np.ndarray) -> np.ndarray: + """Rescale so each block's own 99th percentile lands at 1.0. + + Z-scoring subtracts the block's MEAN, which silently encodes how much of the + corpus that block matches. Measured: `software-systems` matches ~25% of this + reader's corpus (Hacker News, Lobsters, Phoronix, LWN, GitHub releases, + reddit) and has the highest mean raw similarity of any block, 0.537 -- yet + the lowest maximum z, 2.51. `conflict-mideast-israel` matches 5% of the + corpus, has a lower mean of 0.460, and reaches z 3.47. + + The effect is perverse: a topic is penalised for being well covered in your + own feed list. Anchoring on the upper tail instead makes "the best item this + block has seen" comparable across blocks, leaving `weight` to express actual + preference rather than corpus composition. + """ + med = float(np.percentile(values, 50)) + hi = float(np.percentile(values, 99)) + return (values - med) / max(hi - med, 1e-9) + + +NORMALIZERS = {"zscore-per-block": _z, "top-anchored": _top_anchored} + + def score_clusters( con: sqlite3.Connection, profile_vectors: ProfileVectors, @@ -142,11 +165,16 @@ def score_clusters( veto_raw = profile_vectors.raw_matrix("veto", mat) demote_raw = profile_vectors.raw_matrix("demote", mat) - interest_z = {b: _z(v) * profile_vectors.weight("interest", b) for b, v in interest_raw.items()} - importance_z = { - b: _z(v) * profile_vectors.weight("importance", b) for b, v in importance_raw.items() + norm = NORMALIZERS.get(scoring_cfg.get("normalize", "zscore-per-block"), _z) + interest_z = { + b: norm(v) * profile_vectors.weight("interest", b) for b, v in interest_raw.items() } - veto_z = {b: _z(v) for b, v in veto_raw.items()} + importance_z = { + b: norm(v) * profile_vectors.weight("importance", b) for b, v in importance_raw.items() + } + # Veto stays on z-scores: it is a comparison against the interest signal, + # and both sides must be on the same scale. + veto_z = {b: norm(v) for b, v in veto_raw.items()} def argbest(per_block: dict[str, np.ndarray], i: int) -> tuple[str, float]: return max(((b, float(v[i])) for b, v in per_block.items()), key=lambda kv: kv[1]) diff --git a/profile/config.toml b/profile/config.toml index d04c79c..1394800 100644 --- a/profile/config.toml +++ b/profile/config.toml @@ -72,6 +72,9 @@ corroboration_depth_weight = 0.2 # Daily budgets. Deliberately small: the point of the system is a handful of # well-chosen interruptions, not a feed. interest_per_day = 3 +# Max alerts per quota group per day (see `group` in profile.toml). Stops one +# subject taking every slot. +max_alerts_per_group_per_day = 1 important_per_day = 2 # Hard cap on waking the reader during quiet hours, inside important_per_day. important_per_night = 1 diff --git a/profile/profile.toml b/profile/profile.toml index 783d5b9..b391ba9 100644 --- a/profile/profile.toml +++ b/profile/profile.toml @@ -10,6 +10,10 @@ schema_version = 2 # =========================================================== interests ======= +# `group` shares one notification quota across related blocks. Without it the +# five conflict-* blocks could each claim an alert slot and fill a day with the +# same subject -- which is what happened: 5 of 5 alerts were Middle East. +# # --- foreign affairs, split from armed conflict ------------------------------ # Connor: "my actual war interest is low, but unfortunately all wars are # important." The click history looked war-heavy only because world-news feeds @@ -30,7 +34,8 @@ probes = [ # The A-E ordering Connor gave, encoded as weights. Even the lowest band still # reaches him -- but through importance, on major developments only. [[interest]] -id = "conflict-anti-imperial" # (A) top of his ordering +id = "conflict-anti-imperial" +group = "conflict" # (A) top of his ordering weight = 0.9 probes = [ "An armed liberation movement fights a colonial or occupying power for self-determination.", @@ -39,7 +44,8 @@ probes = [ ] [[interest]] -id = "conflict-us-involved" # (B) +id = "conflict-us-involved" +group = "conflict" # (B) weight = 0.85 probes = [ "US forces carry out strikes and the administration defends the legal basis for the war.", @@ -47,7 +53,8 @@ probes = [ ] [[interest]] -id = "conflict-mideast-israel" # (C) +id = "conflict-mideast-israel" +group = "conflict" # (C) weight = 0.8 probes = [ "Israeli forces strike targets in Gaza, Lebanon or Syria amid continuing escalation.", @@ -56,7 +63,8 @@ probes = [ ] [[interest]] -id = "conflict-undercovered" # (D) +id = "conflict-undercovered" +group = "conflict" # (D) weight = 0.75 probes = [ "Fighting intensifies in Sudan as famine spreads and aid convoys are blocked.", @@ -65,7 +73,8 @@ probes = [ ] [[interest]] -id = "conflict-general" # (E) -- low interest, still important +id = "conflict-general" +group = "conflict" # (E) -- low interest, still important weight = 0.45 probes = [ "Front-line fighting continues as both sides exchange drone and artillery strikes.", @@ -137,6 +146,7 @@ probes = [ [[interest]] id = "labor-inequality" +group = "left-politics" weight = 0.85 probes = [ "Workers vote to unionize after a long organizing campaign; the employer contests the result.", @@ -148,6 +158,7 @@ probes = [ [[interest]] id = "left-movements" +group = "left-politics" weight = 0.85 probes = [ "A socialist party wins a decisive election, shifting the governing coalition leftward.", @@ -157,6 +168,7 @@ probes = [ [[interest]] id = "anticolonial-sovereignty" +group = "left-politics" weight = 0.8 probes = [ "Indigenous communities win recognition of land rights against extractive industry.", @@ -176,6 +188,7 @@ probes = [ [[interest]] id = "political-theory" +group = "left-politics" weight = 0.7 probes = [ "An essay argues about socialism, imperialism and political economy, drawing on history rather than reporting a news event.", @@ -204,6 +217,7 @@ probes = [ [[interest]] id = "cincinnati" +group = "local" weight = 0.9 local = true probes = [ @@ -216,6 +230,7 @@ probes = [ [[interest]] id = "kentucky-louisville" +group = "local" weight = 0.6 local = true probes = [ @@ -226,6 +241,7 @@ probes = [ # coverage alongside Cincinnati city government. [[interest]] id = "ohio-state-politics" +group = "local" weight = 0.75 local = true probes = [