connor
98bba62976
Three bugs from the first day of real use. 1. Quiet hours were evaluated in UTC. The router is handed a UTC `now` and compared its hour against "22:00-07:00" directly, so notifications were silenced from 18:00 local -- prime reading time -- and the whole day's budget was released at 03:00 local. The reported avalanche landed at 03:39 and 03:55. Quiet hours are now converted to the reader's wall clock first. 2. The timezone never resolved anyway. TZ was "America/Louisville", which exists on this Arch host but NOT in a Debian container, which ships only the canonical "America/Kentucky/Louisville". The container fell back to UTC and `date` printed "America" as the zone abbreviation. The zone is now read from config rather than the environment, so a bad TZ cannot silently change behaviour, and the compose entry uses the canonical name. NOTE: ~20 other services in compose.yml carry the same bad string; they were left alone since it only affects their log timestamps. 3. Budgets are not a rate. Six notifications fired in a single minute because the daily allowance could all be spent in one cycle. Each lane now has a minimum gap (interest 75min, importance 30min) and sends at most one per cycle, which turns the budget into a rate spread across the day. Separately, the morning digest had silently fallen back to the templated list. The real 28-story payload takes 70.3s to generate against a 60s timeout. Raised to 180s -- and write_prose now LOGS why it failed instead of returning None in silence, which is what made a one-line timeout cost a debugging session.
153 lines
6.9 KiB
TOML
153 lines
6.9 KiB
TOML
# Runtime configuration and tuning knobs.
|
|
#
|
|
# Three-file split, so a new user knows what to touch:
|
|
# config.toml this file -- machinery. Sane defaults; rarely edited.
|
|
# profile.toml who you are: interests, vetoes, importance, delivery.
|
|
# sources.toml what to read.
|
|
#
|
|
# Nothing here is specific to any person. Everything personal lives in the
|
|
# other two files.
|
|
|
|
[http]
|
|
# Sent on every outbound request. Feed operators use it to identify traffic and
|
|
# some will block a generic or absent agent, so put a real contact here.
|
|
# CHANGE THIS if you are not Connor -- it is the contact address feed operators
|
|
# see, and pointing them at someone else's domain is rude and unhelpful.
|
|
user_agent = "news/0.1 (+https://news.rcjohnstone.com; personal feed reader; one user)"
|
|
timeout_secs = 30
|
|
connect_timeout_secs = 10
|
|
concurrency = 8
|
|
max_body_bytes = 10485760
|
|
|
|
[embedding]
|
|
# Any model fastembed can serve. `dim` must match the model.
|
|
#
|
|
# The default is English-only. For native-language sources use a multilingual
|
|
# model -- "intfloat/multilingual-e5-small" (384 dim) is the usual choice.
|
|
# Be aware that changing the model INVALIDATES the calibrated constants below
|
|
# and in [scoring]: they are cosine thresholds, and a different model puts
|
|
# similarity on a different scale. Re-run the calibration before trusting them.
|
|
model = "BAAI/bge-small-en-v1.5"
|
|
dim = 384
|
|
|
|
[clustering]
|
|
# Calibrated by inspection on a 3,885-item corpus; see CALIBRATION.md.
|
|
# 0.70 merged "Solheim Cup capsules" with "FIBA Women's World Cup Glance";
|
|
# 0.86 under-merged. Over-merging is the worse failure because it inflates the
|
|
# corroboration signal that feeds importance.
|
|
#
|
|
# Re-tune if your sources differ substantially in style or language.
|
|
threshold = 0.82
|
|
window_hours = 48
|
|
|
|
# Above this cosine, two items from DIFFERENT newsrooms are treated as the same
|
|
# syndicated copy rather than independent corroboration. Deliberately high:
|
|
# eight outlets each writing their own Renoir-heist headline stayed separate,
|
|
# while a local station reprinting a wire dispatch collapsed correctly.
|
|
syndication_threshold = 0.97
|
|
|
|
[scoring]
|
|
# Compare blocks by z-score WITHIN each block, never by raw cosine: cosine
|
|
# magnitude is not comparable across blocks, and a plain max let one block win
|
|
# 78% of all items. See CALIBRATION.md.
|
|
# "zscore-per-block" or "top-anchored". Both normalise per GROUP (see `group`
|
|
# in profile.toml), not per block: taking the max across blocks gives a subject
|
|
# one draw per block, so a topic split into five blocks beats one covered by a
|
|
# single block purely by having more chances.
|
|
#
|
|
# NOTE the thresholds in [routing] are in z-score units. "top-anchored" puts
|
|
# scores on a different scale (1.0 = that group's 99th percentile), so switching
|
|
# normalizer REQUIRES re-tuning them -- verified: leaving them at 2.0 silences
|
|
# the interest lane completely.
|
|
normalize = "zscore-per-block"
|
|
|
|
# Veto is not a plain similarity threshold. A story can be genuinely adjacent to
|
|
# a vetoed topic and still be wanted, so an item is vetoed on unambiguous
|
|
# similarity OR when the veto signal beats the best interest signal by a margin.
|
|
veto_absolute = 0.60
|
|
veto_margin_over_interest = 0.3
|
|
|
|
# Corroboration = origins + pickup_weight*log1p(pickups) + depth_weight*log1p(depth).
|
|
# "Several outlets carried this" covers four different situations; see
|
|
# news/cluster.py:Corroboration. Origins (independent newsrooms doing their own
|
|
# reporting) dominate; the other two break ties among the many two- and
|
|
# three-origin clusters. Measured on a real corpus: 163 origins, 90 depth,
|
|
# 11 pickups -- depth matters far more here because these sources mostly write
|
|
# their own copy. Raise pickup_weight if you add local papers running wire.
|
|
corroboration_pickup_weight = 0.3
|
|
corroboration_depth_weight = 0.2
|
|
|
|
[routing]
|
|
# Daily budgets. Deliberately small: the point of the system is a handful of
|
|
# well-chosen interruptions, not a feed.
|
|
interest_per_day = 3
|
|
# Max alerts per quota group per day (see `group` in profile.toml). Stops one
|
|
# subject taking every slot.
|
|
max_alerts_per_group_per_day = 1
|
|
important_per_day = 2
|
|
# Hard cap on waking the reader during quiet hours, inside important_per_day.
|
|
important_per_night = 1
|
|
|
|
# Minimum spacing between sends on a lane. Budgets alone are not a rate: without
|
|
# these the whole day's allowance fires in one cycle, and six notifications
|
|
# arrived in the same minute.
|
|
interest_min_gap_minutes = 75
|
|
important_min_gap_minutes = 30
|
|
|
|
# Thresholds are in z-score units -- "how many standard deviations above normal
|
|
# for this block", not raw cosine. Calibrate with `news simulate`, which reports
|
|
# the notifications-per-day these values would actually have produced.
|
|
interest_threshold = 2.0
|
|
important_threshold = 2.0
|
|
digest_floor = 0.5
|
|
# A floor alone yielded 198 items a day. The digest is capped by count and fed
|
|
# strongest-first, so the cap is always spent on the best candidates.
|
|
digest_max_items = 25
|
|
# Max items per topic group in the digest, so no single subject crowds it out.
|
|
# Measured before this existed: conflict and diplomacy took 18 of 28 slots while
|
|
# software-systems took 0, despite being 141 of the eligible clusters.
|
|
digest_max_per_group = 4
|
|
# Slots reserved for the day's most important stories regardless of topic, so a
|
|
# major event stays in the record even when your own interests are busy. The
|
|
# remaining slots are ranked by interest.
|
|
digest_importance_slots = 6
|
|
# Serendipity slots are filled from items BELOW this interest score, so they are
|
|
# genuinely outside the profile rather than near-misses.
|
|
serendipity_max_interest = 0.5
|
|
|
|
# Aggregator maturation: an HN post at 3 points is noise, at 300 it is news.
|
|
# 100, not 150: the HN front page has a median of ~117 points, so a higher bar
|
|
# would reject most of what the crowd has actually promoted.
|
|
hn_min_score = 100
|
|
|
|
# Nothing older than this may generate a notification. Feeds return large
|
|
# archives on first fetch -- one run pulled 88 items going back five months --
|
|
# and those belong in the scoring corpus but not in your pocket.
|
|
max_age_hours = 48
|
|
|
|
[scoring.importance_weights]
|
|
# Importance is NOT corroboration count -- that was the old system's error.
|
|
# Velocity is weighted as heavily as corroboration: 0 to 6 independent origins
|
|
# in ninety minutes is what breaking news looks like, and a story sitting at 6
|
|
# origins for three days is not.
|
|
block = 1.0
|
|
corroboration = 0.4
|
|
velocity = 0.5
|
|
prominence = 0.3
|
|
aggregator = 0.3
|
|
|
|
[llm]
|
|
# The ENTIRE generative-model surface of this system: one call per day to write
|
|
# the digest prose. Clustering, relevance, importance and vetoes are all
|
|
# embeddings and arithmetic -- no model in the scoring path.
|
|
#
|
|
# Cost on this alias is pinned to 0.0 in the LiteLLM config, so a NONZERO spend
|
|
# row is the deliberate quota-overflow alarm, not a billing surprise.
|
|
enabled = true
|
|
base_url = "http://litellm:4000"
|
|
model = "news/glm-5.3-flash"
|
|
api_key_env = "NEWS_LITELLM_KEY"
|
|
# 180, not 60: the real digest payload (28 stories) took 70.3s to generate, so a
|
|
# 60s timeout failed every morning and silently fell back to the templated list.
|
|
timeout_secs = 180
|