connor
7f402f22fe
Connor: too much world news, very little software/aerospace/urbanism/Cincinnati. Measuring found three separate causes. 1. The digest ranked by max(interest, importance), and importance is largely a "big world news" detector -- 22 of 28 slots were entered on the importance axis. A personal reading list ordered by global importance is the wrong objective. The bulk of the digest is now ranked by INTEREST, with a bounded allowance (digest_importance_slots = 6) so a major event still makes the record on a busy day. 2. No per-group cap, so conflict and diplomacy took 18 of 28 slots. digest_max_per_group = 4, with unused capacity flowing back to the best remaining items so a quiet day still fills. 3. HN maturation could never happen. Items came only from `search_by_date`, which returns posts at 2-3 points; once one ages out of the newest window its score is never refetched, so it can never clear the bar and HN's best content was structurally invisible. 45 software clusters were sitting in "aggregator-immature". Added the front page via the `search` endpoint, which returns the crowd's verdict (measured: median 117 points, max 912) and whose re-polling is what actually supplies maturation. hn_min_score 150 -> 100, since a higher bar rejects most of the front page. Digest went from 8 groups (conflict 10, diplomacy 8, software 0) to 13 groups with a far flatter distribution, and the software slot is now "Benchmarking Qwen3.8 27B quantizations" -- the local-inference material that was asked for. Process note: the first two attempts at this silently did nothing. I used str.replace() without asserting the anchor matched, so a stale anchor wrote the file back unchanged while reporting success, and I measured "no change" twice before checking the file. Edits here now assert before writing.
145 lines
6.5 KiB
TOML
145 lines
6.5 KiB
TOML
# Runtime configuration and tuning knobs.
|
|
#
|
|
# Three-file split, so a new user knows what to touch:
|
|
# config.toml this file -- machinery. Sane defaults; rarely edited.
|
|
# profile.toml who you are: interests, vetoes, importance, delivery.
|
|
# sources.toml what to read.
|
|
#
|
|
# Nothing here is specific to any person. Everything personal lives in the
|
|
# other two files.
|
|
|
|
[http]
|
|
# Sent on every outbound request. Feed operators use it to identify traffic and
|
|
# some will block a generic or absent agent, so put a real contact here.
|
|
# CHANGE THIS if you are not Connor -- it is the contact address feed operators
|
|
# see, and pointing them at someone else's domain is rude and unhelpful.
|
|
user_agent = "news/0.1 (+https://news.rcjohnstone.com; personal feed reader; one user)"
|
|
timeout_secs = 30
|
|
connect_timeout_secs = 10
|
|
concurrency = 8
|
|
max_body_bytes = 10485760
|
|
|
|
[embedding]
|
|
# Any model fastembed can serve. `dim` must match the model.
|
|
#
|
|
# The default is English-only. For native-language sources use a multilingual
|
|
# model -- "intfloat/multilingual-e5-small" (384 dim) is the usual choice.
|
|
# Be aware that changing the model INVALIDATES the calibrated constants below
|
|
# and in [scoring]: they are cosine thresholds, and a different model puts
|
|
# similarity on a different scale. Re-run the calibration before trusting them.
|
|
model = "BAAI/bge-small-en-v1.5"
|
|
dim = 384
|
|
|
|
[clustering]
|
|
# Calibrated by inspection on a 3,885-item corpus; see CALIBRATION.md.
|
|
# 0.70 merged "Solheim Cup capsules" with "FIBA Women's World Cup Glance";
|
|
# 0.86 under-merged. Over-merging is the worse failure because it inflates the
|
|
# corroboration signal that feeds importance.
|
|
#
|
|
# Re-tune if your sources differ substantially in style or language.
|
|
threshold = 0.82
|
|
window_hours = 48
|
|
|
|
# Above this cosine, two items from DIFFERENT newsrooms are treated as the same
|
|
# syndicated copy rather than independent corroboration. Deliberately high:
|
|
# eight outlets each writing their own Renoir-heist headline stayed separate,
|
|
# while a local station reprinting a wire dispatch collapsed correctly.
|
|
syndication_threshold = 0.97
|
|
|
|
[scoring]
|
|
# Compare blocks by z-score WITHIN each block, never by raw cosine: cosine
|
|
# magnitude is not comparable across blocks, and a plain max let one block win
|
|
# 78% of all items. See CALIBRATION.md.
|
|
# "zscore-per-block" or "top-anchored". Both normalise per GROUP (see `group`
|
|
# in profile.toml), not per block: taking the max across blocks gives a subject
|
|
# one draw per block, so a topic split into five blocks beats one covered by a
|
|
# single block purely by having more chances.
|
|
#
|
|
# NOTE the thresholds in [routing] are in z-score units. "top-anchored" puts
|
|
# scores on a different scale (1.0 = that group's 99th percentile), so switching
|
|
# normalizer REQUIRES re-tuning them -- verified: leaving them at 2.0 silences
|
|
# the interest lane completely.
|
|
normalize = "zscore-per-block"
|
|
|
|
# Veto is not a plain similarity threshold. A story can be genuinely adjacent to
|
|
# a vetoed topic and still be wanted, so an item is vetoed on unambiguous
|
|
# similarity OR when the veto signal beats the best interest signal by a margin.
|
|
veto_absolute = 0.60
|
|
veto_margin_over_interest = 0.3
|
|
|
|
# Corroboration = origins + pickup_weight*log1p(pickups) + depth_weight*log1p(depth).
|
|
# "Several outlets carried this" covers four different situations; see
|
|
# news/cluster.py:Corroboration. Origins (independent newsrooms doing their own
|
|
# reporting) dominate; the other two break ties among the many two- and
|
|
# three-origin clusters. Measured on a real corpus: 163 origins, 90 depth,
|
|
# 11 pickups -- depth matters far more here because these sources mostly write
|
|
# their own copy. Raise pickup_weight if you add local papers running wire.
|
|
corroboration_pickup_weight = 0.3
|
|
corroboration_depth_weight = 0.2
|
|
|
|
[routing]
|
|
# Daily budgets. Deliberately small: the point of the system is a handful of
|
|
# well-chosen interruptions, not a feed.
|
|
interest_per_day = 3
|
|
# Max alerts per quota group per day (see `group` in profile.toml). Stops one
|
|
# subject taking every slot.
|
|
max_alerts_per_group_per_day = 1
|
|
important_per_day = 2
|
|
# Hard cap on waking the reader during quiet hours, inside important_per_day.
|
|
important_per_night = 1
|
|
|
|
# Thresholds are in z-score units -- "how many standard deviations above normal
|
|
# for this block", not raw cosine. Calibrate with `news simulate`, which reports
|
|
# the notifications-per-day these values would actually have produced.
|
|
interest_threshold = 2.0
|
|
important_threshold = 2.0
|
|
digest_floor = 0.5
|
|
# A floor alone yielded 198 items a day. The digest is capped by count and fed
|
|
# strongest-first, so the cap is always spent on the best candidates.
|
|
digest_max_items = 25
|
|
# Max items per topic group in the digest, so no single subject crowds it out.
|
|
# Measured before this existed: conflict and diplomacy took 18 of 28 slots while
|
|
# software-systems took 0, despite being 141 of the eligible clusters.
|
|
digest_max_per_group = 4
|
|
# Slots reserved for the day's most important stories regardless of topic, so a
|
|
# major event stays in the record even when your own interests are busy. The
|
|
# remaining slots are ranked by interest.
|
|
digest_importance_slots = 6
|
|
# Serendipity slots are filled from items BELOW this interest score, so they are
|
|
# genuinely outside the profile rather than near-misses.
|
|
serendipity_max_interest = 0.5
|
|
|
|
# Aggregator maturation: an HN post at 3 points is noise, at 300 it is news.
|
|
# 100, not 150: the HN front page has a median of ~117 points, so a higher bar
|
|
# would reject most of what the crowd has actually promoted.
|
|
hn_min_score = 100
|
|
|
|
# Nothing older than this may generate a notification. Feeds return large
|
|
# archives on first fetch -- one run pulled 88 items going back five months --
|
|
# and those belong in the scoring corpus but not in your pocket.
|
|
max_age_hours = 48
|
|
|
|
[scoring.importance_weights]
|
|
# Importance is NOT corroboration count -- that was the old system's error.
|
|
# Velocity is weighted as heavily as corroboration: 0 to 6 independent origins
|
|
# in ninety minutes is what breaking news looks like, and a story sitting at 6
|
|
# origins for three days is not.
|
|
block = 1.0
|
|
corroboration = 0.4
|
|
velocity = 0.5
|
|
prominence = 0.3
|
|
aggregator = 0.3
|
|
|
|
[llm]
|
|
# The ENTIRE generative-model surface of this system: one call per day to write
|
|
# the digest prose. Clustering, relevance, importance and vetoes are all
|
|
# embeddings and arithmetic -- no model in the scoring path.
|
|
#
|
|
# Cost on this alias is pinned to 0.0 in the LiteLLM config, so a NONZERO spend
|
|
# row is the deliberate quota-overflow alarm, not a billing surprise.
|
|
enabled = true
|
|
base_url = "http://litellm:4000"
|
|
model = "news/glm-5.3-flash"
|
|
api_key_env = "NEWS_LITELLM_KEY"
|
|
timeout_secs = 60
|