From d047c58d8a18684ac2cf49322dae905d463f933f Mon Sep 17 00:00:00 2001 From: Connor Johnstone Date: Tue, 8 Sep 2026 15:44:37 -0400 Subject: [PATCH] Merge States Newsroom affiliates into one independence bucket Reading real clustered output caught a config error the tests could not: Ohio Capital Journal and Kentucky Lantern are both States Newsroom outlets syndicating the same national desk, and carried 10 byte-identical headlines in a single poll -- more than any other source pair. Configured as separate buckets they would have doubled the corroboration signal on every syndicated story, which is exactly the failure the bucket concept exists to prevent. Found by checking identical headlines across all source pairs rather than spot-fixing the one noticed by eye. That sweep also confirmed the buckets already set deliberately are right: BBC Top/World, Ars Technica/Science and NPR News/Politics each share a bucket and each showed shared headlines. Regression test added. --- profile/sources.toml | 10 ++++++++-- tests/test_cluster.py | 17 +++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/profile/sources.toml b/profile/sources.toml index 78821fe..c57feb8 100644 --- a/profile/sources.toml +++ b/profile/sources.toml @@ -237,10 +237,16 @@ name = "NPR Politics" url = "https://feeds.npr.org/1014/rss.xml" bucket = "npr" +# Ohio Capital Journal and Kentucky Lantern are both States Newsroom affiliates +# and syndicate the same national desk -- they carried 10 byte-identical +# headlines in a single poll, more than any other source pair in the config. +# One bucket, so their shared copy reads as one confirmation rather than two. +# They still contribute genuinely independent state-level reporting; the cost of +# merging is slight under-counting there, which is the safe direction. [[source]] name = "Ohio Capital Journal" url = "https://ohiocapitaljournal.com/feed/" -bucket = "ohio-capital-journal" +bucket = "states-newsroom" poll_interval_secs = 3600 [[source]] @@ -258,7 +264,7 @@ poll_interval_secs = 3600 [[source]] name = "Kentucky Lantern" url = "https://kentuckylantern.com/feed/" -bucket = "kentucky-lantern" +bucket = "states-newsroom" poll_interval_secs = 3600 # ------------------------------------------------------------ italy / mexico -- diff --git a/tests/test_cluster.py b/tests/test_cluster.py index 5d369ff..5b59831 100644 --- a/tests/test_cluster.py +++ b/tests/test_cluster.py @@ -79,3 +79,20 @@ def test_assignment_is_idempotent(con): assert a[0].cluster_id == b[0].cluster_id n = con.execute("SELECT COUNT(*) c FROM cluster_members").fetchone()["c"] assert n == 1, "re-assigning an item must not duplicate membership" + + +def test_syndicated_partners_in_one_bucket_count_once(con): + """Regression: Ohio Capital Journal and Kentucky Lantern are both States + Newsroom and carried 10 identical headlines in one poll. Configured as + separate buckets they would have doubled the corroboration signal.""" + v = unit(1, 0, 0) + for sid, name in (("sn1", "Ohio Capital Journal"), ("sn2", "Kentucky Lantern")): + store.upsert_source(con, id=sid, name=name, url=f"http://{sid}", kind="rss", + bucket="states-newsroom", weight=1.0, prominence=0, + enabled=1, poll_interval_secs=900) + rows = [add(con, "s1i", "sn1", "2026-01-01T00:00:00+00:00", "hs1"), + add(con, "s2i", "sn2", "2026-01-01T00:30:00+00:00", "hs2")] + out = cluster.assign(con, rows, {"hs1": v, "hs2": v}) + assert out[0].cluster_id == out[1].cluster_id + assert cluster.bucket_count(con, out[0].cluster_id) == 1, ( + "syndicated partners must not read as independent confirmation")