connor
d047c58d8a
Reading real clustered output caught a config error the tests could not: Ohio Capital Journal and Kentucky Lantern are both States Newsroom outlets syndicating the same national desk, and carried 10 byte-identical headlines in a single poll -- more than any other source pair. Configured as separate buckets they would have doubled the corroboration signal on every syndicated story, which is exactly the failure the bucket concept exists to prevent. Found by checking identical headlines across all source pairs rather than spot-fixing the one noticed by eye. That sweep also confirmed the buckets already set deliberately are right: BBC Top/World, Ars Technica/Science and NPR News/Politics each share a bucket and each showed shared headlines. Regression test added.
99 lines
4.3 KiB
Python
99 lines
4.3 KiB
Python
"""Clustering behaviour, using synthetic vectors so no model download is needed."""
|
|
|
|
import numpy as np
|
|
import pytest
|
|
|
|
from news import cluster, store
|
|
from news.util import stable_id
|
|
|
|
|
|
@pytest.fixture
|
|
def con(tmp_path):
|
|
c = store.connect(tmp_path / "t.db")
|
|
store.upsert_source(
|
|
c, id="s1", name="A", url="http://a", kind="rss", bucket="a",
|
|
weight=1.0, prominence=0, enabled=1, poll_interval_secs=900)
|
|
store.upsert_source(
|
|
c, id="s2", name="B", url="http://b", kind="rss", bucket="b",
|
|
weight=1.0, prominence=0, enabled=1, poll_interval_secs=900)
|
|
return c
|
|
|
|
|
|
def add(con, item_id, source_id, when, chash):
|
|
con.execute(
|
|
"""INSERT INTO items (id,source_id,title,summary,url,canonical_url,
|
|
published_at,first_seen_at,content_hash)
|
|
VALUES (?,?,?,'',?,?,?,?,?)""",
|
|
(item_id, source_id, f"t-{item_id}", f"http://x/{item_id}",
|
|
f"http://x/{item_id}", when, when, chash))
|
|
return con.execute("SELECT * FROM items WHERE id=?", (item_id,)).fetchone()
|
|
|
|
|
|
def unit(*v):
|
|
a = np.array(v, dtype=np.float32)
|
|
return a / np.linalg.norm(a)
|
|
|
|
|
|
def test_similar_items_merge_and_distinct_ones_do_not(con):
|
|
near_a, near_b = unit(1, 0.02, 0), unit(1, 0.06, 0) # cosine ~0.999
|
|
far = unit(0, 1, 0) # orthogonal
|
|
rows = [add(con, "i1", "s1", "2026-01-01T00:00:00+00:00", "h1"),
|
|
add(con, "i2", "s2", "2026-01-01T01:00:00+00:00", "h2"),
|
|
add(con, "i3", "s1", "2026-01-01T02:00:00+00:00", "h3")]
|
|
out = cluster.assign(con, rows, {"h1": near_a, "h2": near_b, "h3": far})
|
|
by_item = {a.item_id: a.cluster_id for a in out}
|
|
assert by_item["i1"] == by_item["i2"], "paraphrase-near items must merge"
|
|
assert by_item["i3"] != by_item["i1"], "unrelated item must not merge"
|
|
|
|
|
|
def test_items_outside_the_window_do_not_merge(con):
|
|
v = unit(1, 0, 0)
|
|
rows = [add(con, "i1", "s1", "2026-01-01T00:00:00+00:00", "h1"),
|
|
add(con, "i2", "s2", "2026-01-05T00:00:00+00:00", "h2")] # 4 days later
|
|
out = cluster.assign(con, rows, {"h1": v, "h2": v})
|
|
assert out[0].cluster_id != out[1].cluster_id
|
|
|
|
|
|
def test_bucket_count_counts_newsrooms_not_feeds(con):
|
|
"""Two feeds from one newsroom are one independent confirmation, not two."""
|
|
v = unit(1, 0, 0)
|
|
store.upsert_source(
|
|
con, id="s3", name="A2", url="http://a2", kind="rss", bucket="a",
|
|
weight=1.0, prominence=0, enabled=1, poll_interval_secs=900)
|
|
rows = [add(con, "i1", "s1", "2026-01-01T00:00:00+00:00", "h1"),
|
|
add(con, "i2", "s3", "2026-01-01T01:00:00+00:00", "h2")]
|
|
out = cluster.assign(con, rows, {"h1": v, "h2": v})
|
|
assert out[0].cluster_id == out[1].cluster_id
|
|
assert cluster.bucket_count(con, out[0].cluster_id) == 1
|
|
|
|
rows2 = [add(con, "i3", "s2", "2026-01-01T02:00:00+00:00", "h3")]
|
|
cluster.assign(con, rows2, {"h3": v})
|
|
assert cluster.bucket_count(con, out[0].cluster_id) == 2
|
|
|
|
|
|
def test_assignment_is_idempotent(con):
|
|
v = unit(1, 0, 0)
|
|
rows = [add(con, "i1", "s1", "2026-01-01T00:00:00+00:00", "h1")]
|
|
a = cluster.assign(con, rows, {"h1": v})
|
|
b = cluster.assign(con, rows, {"h1": v})
|
|
assert a[0].cluster_id == b[0].cluster_id
|
|
n = con.execute("SELECT COUNT(*) c FROM cluster_members").fetchone()["c"]
|
|
assert n == 1, "re-assigning an item must not duplicate membership"
|
|
|
|
|
|
def test_syndicated_partners_in_one_bucket_count_once(con):
|
|
"""Regression: Ohio Capital Journal and Kentucky Lantern are both States
|
|
Newsroom and carried 10 identical headlines in one poll. Configured as
|
|
separate buckets they would have doubled the corroboration signal."""
|
|
v = unit(1, 0, 0)
|
|
for sid, name in (("sn1", "Ohio Capital Journal"), ("sn2", "Kentucky Lantern")):
|
|
store.upsert_source(con, id=sid, name=name, url=f"http://{sid}", kind="rss",
|
|
bucket="states-newsroom", weight=1.0, prominence=0,
|
|
enabled=1, poll_interval_secs=900)
|
|
rows = [add(con, "s1i", "sn1", "2026-01-01T00:00:00+00:00", "hs1"),
|
|
add(con, "s2i", "sn2", "2026-01-01T00:30:00+00:00", "hs2")]
|
|
out = cluster.assign(con, rows, {"hs1": v, "hs2": v})
|
|
assert out[0].cluster_id == out[1].cluster_id
|
|
assert cluster.bucket_count(con, out[0].cluster_id) == 1, (
|
|
"syndicated partners must not read as independent confirmation")
|