connor
d57b9e092c
Connor's observation: two outlets running the same story can be caught by comparing the text, without anyone having to know who owns whom. Corroboration is now origin_count(), a union-find where two items share an origin if EITHER they are in the same bucket, OR their text is near-identical (normalised title match, or cosine >= 0.97). The rules are complementary: buckets are same-owner/different-text (NYT filing five distinct AfD pieces), text matching is different-owner/same-text (a local station reprinting wire). On real data it fires exactly where it should: the vote-by-mail cluster collapses 2 buckets -> 1 origin, catching the States Newsroom syndication automatically -- the case previously fixed by hand in config -- and a Smithsonian story collapses 3 -> 2, catching WCPO reprinting wire, which no static bucket can express since WCPO is genuinely independent on local stories. The eight Renoir-heist reports, each independently written, correctly stay at 8. Across 91 multi-item clusters: 174 buckets -> 163 true origins. Note this is the one place a lexical comparison is right. Clustering fails on lexical methods because paraphrases share no wording, which is why the SimHash stage was measured and deleted -- but syndicated copy is literally the same bytes. First formulation was wrong and the data caught it: summing syndication groups and solo buckets produced 3 origins for a 2-bucket cluster. Syndication may only ever reduce corroboration, which the union-find guarantees and a test asserts.
60 lines
2.6 KiB
TOML
60 lines
2.6 KiB
TOML
# Runtime configuration and tuning knobs.
|
|
#
|
|
# Three-file split, so a new user knows what to touch:
|
|
# config.toml this file -- machinery. Sane defaults; rarely edited.
|
|
# profile.toml who you are: interests, vetoes, importance, delivery.
|
|
# sources.toml what to read.
|
|
#
|
|
# Nothing here is specific to any person. Everything personal lives in the
|
|
# other two files.
|
|
|
|
[http]
|
|
# Sent on every outbound request. Feed operators use it to identify traffic and
|
|
# some will block a generic or absent agent, so put a real contact here.
|
|
# CHANGE THIS if you are not Connor -- it is the contact address feed operators
|
|
# see, and pointing them at someone else's domain is rude and unhelpful.
|
|
user_agent = "news/0.1 (+https://news.rcjohnstone.com; personal feed reader; one user)"
|
|
timeout_secs = 30
|
|
connect_timeout_secs = 10
|
|
concurrency = 8
|
|
max_body_bytes = 10485760
|
|
|
|
[embedding]
|
|
# Any model fastembed can serve. `dim` must match the model.
|
|
#
|
|
# The default is English-only. For native-language sources use a multilingual
|
|
# model -- "intfloat/multilingual-e5-small" (384 dim) is the usual choice.
|
|
# Be aware that changing the model INVALIDATES the calibrated constants below
|
|
# and in [scoring]: they are cosine thresholds, and a different model puts
|
|
# similarity on a different scale. Re-run the calibration before trusting them.
|
|
model = "BAAI/bge-small-en-v1.5"
|
|
dim = 384
|
|
|
|
[clustering]
|
|
# Calibrated by inspection on a 3,885-item corpus; see CALIBRATION.md.
|
|
# 0.70 merged "Solheim Cup capsules" with "FIBA Women's World Cup Glance";
|
|
# 0.86 under-merged. Over-merging is the worse failure because it inflates the
|
|
# corroboration signal that feeds importance.
|
|
#
|
|
# Re-tune if your sources differ substantially in style or language.
|
|
threshold = 0.82
|
|
window_hours = 48
|
|
|
|
# Above this cosine, two items from DIFFERENT newsrooms are treated as the same
|
|
# syndicated copy rather than independent corroboration. Deliberately high:
|
|
# eight outlets each writing their own Renoir-heist headline stayed separate,
|
|
# while a local station reprinting a wire dispatch collapsed correctly.
|
|
syndication_threshold = 0.97
|
|
|
|
[scoring]
|
|
# Compare blocks by z-score WITHIN each block, never by raw cosine: cosine
|
|
# magnitude is not comparable across blocks, and a plain max let one block win
|
|
# 78% of all items. See CALIBRATION.md.
|
|
normalize = "zscore-per-block"
|
|
|
|
# Veto is not a plain similarity threshold. A story can be genuinely adjacent to
|
|
# a vetoed topic and still be wanted, so an item is vetoed on unambiguous
|
|
# similarity OR when the veto signal beats the best interest signal by a margin.
|
|
veto_absolute = 0.60
|
|
veto_margin_over_interest = 0.3
|