Cap how many already-seen stories return to the digest

A cluster stays eligible as long as fresh reports keep joining it, which
refreshes last_seen_at. That is right -- an ongoing story is today's news -- but
it is not bounded by the window, so the digest drifts into being mostly re-runs.
Measured against the live 4,372-cluster corpus, replaying today's recorded
digest as a prior day: 21 of 28 slots would go to stories already delivered,
leaving 7 new.

So cap them at three rather than excluding them. A digest gives each story about
a sentence, so an update that does not move the headline is barely expressed
here anyway -- but a genuine escalation on a story being followed must not be
silently dropped, and the survivors are the strongest by max(interest,
importance), the same "best match" measure the triage feed's default sort uses.

Same corpus with the cap: 3 repeats, 25 new, digest still full at 28. The freed
slots had good material waiting below the line the whole time.

Applied BEFORE the admission passes, not inside them. The importance allowance
would otherwise smuggle repeats past the cap, which is exactly how an earlier
attempt at the group cap failed, and a test now pins that. Serendipity is
covered too: the candidate list is rebuilt after capping, so a capped repeat
cannot return through a reserved slot.

`day < today` in the lookback so re-running the digest does not read its own
rows as prior appearances and cap the whole thing to three.

NOT DEPLOYED -- image unbuilt, running container still on d78eec0.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_015xPLgmcXCc9JjVwmQF63eM

news/config.py

@@ -129,6 +129,10 @@ class Config:
             "digest_max_per_group": r.get("digest_max_per_group", 4),
             "digest_importance_slots": r.get("digest_importance_slots", 6),
             "serendipity_max_interest": r.get("serendipity_max_interest", 0.5),
+            # How many stories the reader has already had in a digest may come
+            # back, and how far back "already had" looks.
+            "digest_max_repeats": r.get("digest_max_repeats", 3),
+            "digest_repeat_lookback_days": r.get("digest_repeat_lookback_days", 3),
         }
 
     # -- reader ------------------------------------------------------------

news/route.py

@@ -259,6 +259,48 @@ def route(
     imp_slots = routing.get("digest_importance_slots", 6)
 
     candidates = [s for s in live if s.cluster_id not in decisions]
+
+    # Repeats: stories the reader has already had in a digest, back for another
+    # day because a fresh report joined the cluster and refreshed its
+    # last_seen_at. That freshness rule is right -- an ongoing story IS today's
+    # news -- but it is not bounded by the window, so on a measured day 9 of 28
+    # digest stories were ones already seen.
+    #
+    # Cap them rather than exclude them. A digest gives each story about a
+    # sentence, so an update that does not change the headline is barely
+    # expressed here anyway; but a genuine escalation on a story you are
+    # following must not be silently dropped. So the strongest few come back and
+    # the rest give up their slots to something new.
+    #
+    # Ranked by max(interest, importance) -- "best match", the same measure the
+    # triage feed's default sort uses -- because a repeat earns its slot on
+    # either axis.
+    #
+    # Applied BEFORE the admission passes below, not inside them. The importance
+    # allowance would otherwise smuggle repeats past this cap, which is exactly
+    # how an earlier attempt at the group cap failed.
+    max_repeats = routing.get("digest_max_repeats", 3)
+    lookback = routing.get("digest_repeat_lookback_days", 3)
+    today = now.date()
+    seen_before = {
+        r["cluster_id"]
+        for r in con.execute(
+            # `day < today` on purpose: re-running the digest must not count
+            # this morning's own rows as prior appearances and cap everything.
+            "SELECT DISTINCT cluster_id FROM digest_items WHERE day >= ? AND day < ?",
+            ((today - timedelta(days=lookback)).isoformat(), today.isoformat()),
+        )
+    }
+    if seen_before:
+        returning = sorted(
+            (s for s in candidates if s.cluster_id in seen_before),
+            key=lambda x: -max(x.interest_score, x.importance_score))
+        for s in returning[max_repeats:]:
+            settle(s, DROP, "repeat-capped", max(s.interest_score, s.importance_score))
+        # Rebuilt so the capped ones are invisible to every pass below,
+        # serendipity included -- otherwise a repeat could return through it.
+        candidates = [s for s in candidates if s.cluster_id not in decisions]
+
     taken = 0
     group_count: dict[str, int] = {}
 

profile/config.toml

@@ -115,6 +115,22 @@ digest_importance_slots = 6
 # genuinely outside the profile rather than near-misses.
 serendipity_max_interest = 0.5
 
+# How many stories you have ALREADY had in a digest may come back for another
+# day. A story stays eligible as long as fresh reports keep joining its cluster
+# -- correct, an ongoing story is today's news -- but unbounded that made the
+# digest mostly re-runs: 9 of 28 on a measured day.
+#
+# Capped rather than excluded, because a real escalation on a story you are
+# following must still reach you. The survivors are the best matches by
+# max(interest, importance), the same measure the triage feed's default sort
+# uses. Three is deliberately small: a digest gives each story about a sentence,
+# so an update that does not change the headline is barely expressed here
+# anyway, and the freed slots go to stories you have not seen.
+digest_max_repeats = 3
+# How far back "already had it" looks. Matches the notification suppression
+# horizon; a story returning after longer than this is treated as new again.
+digest_repeat_lookback_days = 3
+
 # Aggregator maturation: an HN post at 3 points is noise, at 300 it is news.
 # 100, not 150: the HN front page has a median of ~117 points, so a higher bar
 # would reject most of what the crowd has actually promoted.

tests/test_route.py

@@ -206,3 +206,90 @@ def test_a_second_send_waits_for_the_gap(con):
     later = NOW + timedelta(minutes=80)
     out2 = route(con, [mk("a", interest=9.0)], ROUTING, DELIVERY_TZ, later)
     assert lanes(out2)["a"] == INTEREST
+
+
+def _seen_on(con, day, *cluster_ids):
+    """Record a prior digest appearance."""
+    for rank, cid in enumerate(cluster_ids):
+        con.execute("""INSERT INTO digest_items (day,cluster_id,rank,score,reason)
+                       VALUES (?,?,?,1.0,'digest')""", (day.isoformat(), cid, rank))
+
+
+def test_only_the_strongest_repeats_come_back(con):
+    """A story stays eligible while fresh reports keep joining its cluster, so
+    without a cap the digest becomes mostly re-runs -- 9 of 28 on a real day."""
+    yesterday = (NOW - timedelta(days=1)).date()
+    _seen_on(con, yesterday, *[f"old{i}" for i in range(6)])
+
+    # Six returning stories at descending strength, plus fresh material. All
+    # below interest_threshold, or they would be notified rather than digested.
+    scores = [mk(f"old{i}", interest=1.9 - i * 0.1) for i in range(6)]
+    scores += [mk(f"new{i}", interest=1.0) for i in range(10)]
+    out = route(con, scores, {**ROUTING, "digest_max_repeats": 3}, DELIVERY, NOW)
+
+    digested = {d.cluster_id for d in out if d.lane == DIGEST}
+    assert {"old0", "old1", "old2"} <= digested, "the strongest repeats must return"
+    for cid in ("old3", "old4", "old5"):
+        assert cid not in digested, f"{cid} is a fourth-or-later repeat"
+    # Freed slots go to stories not yet seen, not to nothing.
+    assert len([c for c in digested if c.startswith("new")]) == 10
+
+
+def test_a_capped_repeat_says_why(con):
+    _seen_on(con, (NOW - timedelta(days=1)).date(), "a", "b")
+    out = route(con, [mk("a", interest=1.9), mk("b", interest=1.8)],
+                {**ROUTING, "digest_max_repeats": 1}, DELIVERY, NOW)
+    reasons = {d.cluster_id: d.reason for d in out}
+    assert reasons["b"] == "repeat-capped"
+
+
+def test_repeats_are_ranked_by_best_match_not_interest_alone(con):
+    """A repeat earns its slot on either axis: a story you are only mildly
+    interested in but which has escalated must still get through."""
+    _seen_on(con, (NOW - timedelta(days=1)).date(), "quiet", "escalating")
+    scores = [mk("quiet", interest=1.5, importance=0.0),
+              mk("escalating", interest=0.6, importance=1.95)]
+    out = route(con, scores, {**ROUTING, "digest_max_repeats": 1}, DELIVERY, NOW)
+    kept = {d.cluster_id for d in out if d.lane == DIGEST}
+    assert kept == {"escalating"}, (
+        "ranked by interest alone, the quiet story would have taken the slot")
+
+
+def test_the_importance_allowance_cannot_smuggle_repeats_past_the_cap(con):
+    """The group cap failed this exact way once: a bounded allowance applied
+    after the fact swallowed the rule it was meant to respect."""
+    _seen_on(con, (NOW - timedelta(days=1)).date(), *[f"r{i}" for i in range(6)])
+    scores = [mk(f"r{i}", interest=0.6, importance=9.0 - i * 0.1) for i in range(6)]
+    out = route(con, scores, {**ROUTING, "digest_max_repeats": 2,
+                              "digest_importance_slots": 6}, DELIVERY, NOW)
+    assert sum(1 for d in out if d.lane == DIGEST) == 2
+
+
+def test_a_repeat_cannot_return_through_a_serendipity_slot(con):
+    _seen_on(con, (NOW - timedelta(days=1)).date(), "off1", "off2", "off3")
+    delivery = {**DELIVERY, "serendipity_slots": 2}
+    scores = [mk(f"on{i}", interest=1.9, importance=1.9) for i in range(30)]
+    scores += [mk("off1", interest=-1.0, importance=1.0),
+               mk("off2", interest=-2.0, importance=0.9),
+               mk("off3", interest=-3.0, importance=0.8)]
+    out = route(con, scores, {**ROUTING, "digest_max_items": 5, "digest_max_repeats": 1},
+                delivery, NOW)
+    picked = {d.cluster_id for d in out if d.reason == "serendipity"}
+    assert picked <= {"off1"}, f"a capped repeat came back as serendipity: {picked}"
+
+
+def test_rerunning_the_digest_does_not_treat_its_own_rows_as_repeats(con):
+    """`day < today`: without it, a second run on the same day would see this
+    morning's rows and cap the whole digest to three stories."""
+    _seen_on(con, NOW.date(), *[f"t{i}" for i in range(6)])
+    scores = [mk(f"t{i}", interest=1.0) for i in range(6)]
+    out = route(con, scores, {**ROUTING, "digest_max_repeats": 3}, DELIVERY, NOW)
+    assert sum(1 for d in out if d.lane == DIGEST) == 6
+
+
+def test_a_story_returning_after_the_lookback_counts_as_new(con):
+    _seen_on(con, (NOW - timedelta(days=10)).date(), "a", "b", "c", "d")
+    scores = [mk(c, interest=1.0) for c in ("a", "b", "c", "d")]
+    out = route(con, scores, {**ROUTING, "digest_max_repeats": 1,
+                              "digest_repeat_lookback_days": 3}, DELIVERY, NOW)
+    assert sum(1 for d in out if d.lane == DIGEST) == 4