Make the system portable: split config from profile, remove hard-coding

Audit found three things compiled in that shouldn't have been: the User-Agent
carried the author's domain, the cluster threshold and window were module
constants the pipeline never overrode, and the embedding model was pinned --
which blocked the one change most likely to be wanted, swapping in a
multilingual model for native-language sources.

Configuration is now three files with distinct roles: config.toml (machinery),
profile.toml (who the reader is), sources.toml (what to read). All are
relocatable via NEWS_CONFIG_DIR. news/*.py now contains no interest, place name,
feed URL, or personal identifier.

tests/test_config.py keeps it that way: it builds a configuration for an
invented reader and asserts none of the author's identity or preferences reaches
the running code. Writing it immediately found a real bug -- CONFIG_DIR was
resolved at import time, so NEWS_CONFIG_DIR silently did nothing if set after
the module was first imported. Now resolved per instantiation.

Documented the coupling that portability creates: [clustering].threshold and
[scoring].veto_absolute are cosine constants calibrated against one corpus with
one model, so changing [embedding].model invalidates both.
This commit is contained in:
connor committed 2026-09-08 15:31:43 -04:00
1 parent 6dc3694260
commit 863f5e20ac
8 files changed
+317 -36

No files matched your search

+53
View File
@@ -0,0 +1,53 @@
# Runtime configuration and tuning knobs.
#
# Three-file split, so a new user knows what to touch:
# config.toml this file -- machinery. Sane defaults; rarely edited.
# profile.toml who you are: interests, vetoes, importance, delivery.
# sources.toml what to read.
#
# Nothing here is specific to any person. Everything personal lives in the
# other two files.
[http]
# Sent on every outbound request. Feed operators use it to identify traffic and
# some will block a generic or absent agent, so put a real contact here.
# CHANGE THIS if you are not Connor -- it is the contact address feed operators
# see, and pointing them at someone else's domain is rude and unhelpful.
user_agent = "news/0.1 (+https://news.rcjohnstone.com; personal feed reader; one user)"
timeout_secs = 30
connect_timeout_secs = 10
concurrency = 8
max_body_bytes = 10485760
[embedding]
# Any model fastembed can serve. `dim` must match the model.
#
# The default is English-only. For native-language sources use a multilingual
# model -- "intfloat/multilingual-e5-small" (384 dim) is the usual choice.
# Be aware that changing the model INVALIDATES the calibrated constants below
# and in [scoring]: they are cosine thresholds, and a different model puts
# similarity on a different scale. Re-run the calibration before trusting them.
model = "BAAI/bge-small-en-v1.5"
dim = 384
[clustering]
# Calibrated by inspection on a 3,885-item corpus; see CALIBRATION.md.
# 0.70 merged "Solheim Cup capsules" with "FIBA Women's World Cup Glance";
# 0.86 under-merged. Over-merging is the worse failure because it inflates the
# corroboration signal that feeds importance.
#
# Re-tune if your sources differ substantially in style or language.
threshold = 0.82
window_hours = 48
[scoring]
# Compare blocks by z-score WITHIN each block, never by raw cosine: cosine
# magnitude is not comparable across blocks, and a plain max let one block win
# 78% of all items. See CALIBRATION.md.
normalize = "zscore-per-block"
# Veto is not a plain similarity threshold. A story can be genuinely adjacent to
# a vetoed topic and still be wanted, so an item is vetoed on unambiguous
# similarity OR when the veto signal beats the best interest signal by a margin.
veto_absolute = 0.60
veto_margin_over_interest = 0.3
+5 -17
View File
@@ -419,25 +419,13 @@ hn_min_score = 150
# failure was collecting 32 of them, not collecting shallow ones.
feedback_actions = ["interested", "not_interested"]
# Embedding stays English-only (bge-small-en-v1.5). Measured 2026-09-08: the
# same story in Spanish and English scored 0.758 and 0.600 cosine, against a
# random-pair p99 of 0.634 -- so one of two translation pairs was
# indistinguishable from an unrelated article. Native-language Italian and
# Spanish feeds would rank erratically, so Italy and Mexico are covered by
# English-language sources (ANSA English, Mexico News Daily) instead. Revisit
# with multilingual-e5-small only if native sources become worth re-calibrating
# every threshold for.
embedding_language = "en"
# Embedding is English-only by default (see config.toml [embedding]). Measured
# 2026-09-08: the same story in Spanish and English scored 0.758 and 0.600
# cosine against a random-pair p99 of 0.634, so one of two translation pairs
# was indistinguishable from an unrelated article. Italy and Mexico are
# therefore covered by English-language sources.
# Conflict outside categories A-D notifies only on a major development -- a new
# offensive, a ceasefire, mass casualties, a state entering. Routine front-line
# reporting goes to the digest.
conflict_general_requires_importance = true
# =========================================================== scoring =========
# Fitted 2026-09-08 against 3,885 items and 40 real feedback labels.
# See profile/CALIBRATION.md.
[scoring]
normalize = "zscore-per-block"
veto_absolute = 0.60
veto_margin_over_interest = 0.3