diff --git a/news/config.py b/news/config.py index 0bc760d..f2c357b 100644 --- a/news/config.py +++ b/news/config.py @@ -119,6 +119,8 @@ class Config: "hn_min_score": r.get("hn_min_score", 150), "max_age_hours": r.get("max_age_hours", 48), "digest_max_items": r.get("digest_max_items", 25), + "digest_max_per_group": r.get("digest_max_per_group", 4), + "digest_importance_slots": r.get("digest_importance_slots", 6), "serendipity_max_interest": r.get("serendipity_max_interest", 0.5), } diff --git a/news/route.py b/news/route.py index 456b73a..bc1e06e 100644 --- a/news/route.py +++ b/news/route.py @@ -193,24 +193,61 @@ def route( take_group(s, "interest") settle(s, INTEREST, f"interest:{s.interest.block}", s.interest_score) - # --- one capped, ranked digest --- + # --- one capped digest --- + # The digest is a READING surface, so most of it is ranked by INTEREST. + # Ranking by max(interest, importance) let importance -- which is largely a + # "big world news" detector -- take 22 of 28 slots, while software-systems + # took 0 despite being 141 of the eligible clusters. + # + # Importance keeps a bounded, explicit allowance rather than an exemption. + # An earlier attempt exempted every DEFERRED item from the group cap and + # changed nothing: `important_threshold` is low enough that nearly every + # cluster clears it, so once the daily budget is spent essentially the whole + # pool is deferred and the exemption swallowed the cap. floor = routing.get("digest_floor", 0.0) cap = routing.get("digest_max_items", 25) - pool = sorted( - (s for s in live if s.cluster_id not in decisions), - key=lambda x: -max(x.interest_score, x.importance_score), - ) + per_group = routing.get("digest_max_per_group", 4) + imp_slots = routing.get("digest_importance_slots", 6) + + candidates = [s for s in live if s.cluster_id not in decisions] taken = 0 - for s in pool: - value = max(s.interest_score, s.importance_score) - reason = deferred.get(s.cluster_id, "digest") - if value < floor and s.cluster_id not in deferred: - settle(s, DROP, "below-floor", value) - elif taken < cap: - settle(s, DIGEST, reason, value) + group_count: dict[str, int] = {} + + def admit(s: ClusterScore, reason: str, value: float) -> None: + nonlocal taken + settle(s, DIGEST, reason, value) + g = group_of(s, "interest") + group_count[g] = group_count.get(g, 0) + 1 + taken += 1 + + # 1. A fixed allowance for the day's most important stories, whatever the + # topic, so a major event stays in the record even on a day when the + # reader's own subjects are busy. + for s in sorted(candidates, key=lambda x: -x.importance_score)[:imp_slots]: + if s.importance_score >= routing.get("important_threshold", 2.0): + admit(s, deferred.get(s.cluster_id, "digest-important"), s.importance_score) + + # 2. Everything else by interest, capped per group. + overflow: list[ClusterScore] = [] + for s in sorted(candidates, key=lambda x: -x.interest_score): + if s.cluster_id in decisions: + continue + if s.interest_score < floor: + settle(s, DROP, "below-floor", s.interest_score) + continue + if taken >= cap or group_count.get(group_of(s, "interest"), 0) >= per_group: + overflow.append(s) + continue + admit(s, deferred.get(s.cluster_id, "digest"), s.interest_score) + + # 3. Capacity the group caps left unused goes back to the best remaining + # items, so a quiet day still fills the digest. + for s in overflow: + if taken < cap: + settle(s, DIGEST, "digest-overflow", s.interest_score) taken += 1 else: - settle(s, DROP, "digest-full", value) + settle(s, DROP, "digest-full", s.interest_score) # Serendipity: reserve slots for items that scored well on importance but # POORLY on the interest profile, so the digest does not become a mirror of @@ -219,7 +256,7 @@ def route( if slots: ceiling = routing.get("serendipity_max_interest", 0.5) outside = sorted( - (s for s in pool + (s for s in candidates if decisions[s.cluster_id].lane == DROP and s.interest_score < ceiling), key=lambda x: -x.importance_score, ) diff --git a/news/sources/hn.py b/news/sources/hn.py index a859744..365ddd0 100644 --- a/news/sources/hn.py +++ b/news/sources/hn.py @@ -13,15 +13,22 @@ from datetime import UTC, datetime from ..embed import embed_text from ..util import canonical_url, content_hash, stable_id -API = "https://hn.algolia.com/api/v1/search_by_date" +# `search_by_date` returns the newest posts, which arrive at 2-3 points. That +# alone can never satisfy the maturation gate: once a post ages out of the newest +# window its score is never refetched, so it can never climb past the bar. The +# `search` endpoint with tags=front_page returns the crowd's verdict instead +# (measured: median 117 points, max 912) and re-polling it is what actually +# supplies maturation. Both are configured; see sources.toml. +API_BY_DATE = "https://hn.algolia.com/api/v1/search_by_date" +API_SEARCH = "https://hn.algolia.com/api/v1/search" async def fetch(client, source) -> tuple[list[dict], dict]: - params = { - "tags": source.get("query", "story"), - "hitsPerPage": str(source.get("page_size", 100)), - } - resp = await client.get(API, params=params) + tags = source.get("query", "story") + # front_page is a ranking, not a time slice, so it lives on /search. + endpoint = API_SEARCH if tags == "front_page" else API_BY_DATE + params = {"tags": tags, "hitsPerPage": str(source.get("page_size", 100))} + resp = await client.get(endpoint, params=params) resp.raise_for_status() now = datetime.now(UTC).isoformat() diff --git a/profile/config.toml b/profile/config.toml index 7e8f52d..1fe033e 100644 --- a/profile/config.toml +++ b/profile/config.toml @@ -97,12 +97,22 @@ digest_floor = 0.5 # A floor alone yielded 198 items a day. The digest is capped by count and fed # strongest-first, so the cap is always spent on the best candidates. digest_max_items = 25 +# Max items per topic group in the digest, so no single subject crowds it out. +# Measured before this existed: conflict and diplomacy took 18 of 28 slots while +# software-systems took 0, despite being 141 of the eligible clusters. +digest_max_per_group = 4 +# Slots reserved for the day's most important stories regardless of topic, so a +# major event stays in the record even when your own interests are busy. The +# remaining slots are ranked by interest. +digest_importance_slots = 6 # Serendipity slots are filled from items BELOW this interest score, so they are # genuinely outside the profile rather than near-misses. serendipity_max_interest = 0.5 # Aggregator maturation: an HN post at 3 points is noise, at 300 it is news. -hn_min_score = 150 +# 100, not 150: the HN front page has a median of ~117 points, so a higher bar +# would reject most of what the crowd has actually promoted. +hn_min_score = 100 # Nothing older than this may generate a notification. Feeds return large # archives on first fetch -- one run pulled 88 items going back five months -- diff --git a/profile/sources.toml b/profile/sources.toml index c57feb8..f5c8616 100644 --- a/profile/sources.toml +++ b/profile/sources.toml @@ -118,6 +118,20 @@ kind = "hn" bucket = "hacker-news" poll_interval_secs = 1800 +# The front page, which is the only way HN scores ever mature. Items fetched by +# `search_by_date` arrive at 2-3 points and, once they age out of that window, +# are never refetched -- so on their own they can never clear the maturation +# bar and HN's best content stays permanently invisible. Re-polling the front +# page refreshes the score of whatever the crowd has actually promoted. +[[source]] +name = "Hacker News front page" +url = "https://hn.algolia.com/api/v1/search" +kind = "hn" +query = "front_page" +bucket = "hacker-news" +page_size = 50 +poll_interval_secs = 1800 + [[source]] name = "Lobsters" url = "https://lobste.rs/rss"