"""Clustering behaviour, using synthetic vectors so no model download is needed.""" import numpy as np import pytest from news import cluster, store from news.util import stable_id @pytest.fixture def con(tmp_path): c = store.connect(tmp_path / "t.db") store.upsert_source( c, id="s1", name="A", url="http://a", kind="rss", bucket="a", weight=1.0, prominence=0, enabled=1, poll_interval_secs=900) store.upsert_source( c, id="s2", name="B", url="http://b", kind="rss", bucket="b", weight=1.0, prominence=0, enabled=1, poll_interval_secs=900) return c def add(con, item_id, source_id, when, chash): con.execute( """INSERT INTO items (id,source_id,title,summary,url,canonical_url, published_at,first_seen_at,content_hash) VALUES (?,?,?,'',?,?,?,?,?)""", (item_id, source_id, f"t-{item_id}", f"http://x/{item_id}", f"http://x/{item_id}", when, when, chash)) return con.execute("SELECT * FROM items WHERE id=?", (item_id,)).fetchone() def unit(*v): a = np.array(v, dtype=np.float32) return a / np.linalg.norm(a) def test_similar_items_merge_and_distinct_ones_do_not(con): near_a, near_b = unit(1, 0.02, 0), unit(1, 0.06, 0) # cosine ~0.999 far = unit(0, 1, 0) # orthogonal rows = [add(con, "i1", "s1", "2026-01-01T00:00:00+00:00", "h1"), add(con, "i2", "s2", "2026-01-01T01:00:00+00:00", "h2"), add(con, "i3", "s1", "2026-01-01T02:00:00+00:00", "h3")] out = cluster.assign(con, rows, {"h1": near_a, "h2": near_b, "h3": far}) by_item = {a.item_id: a.cluster_id for a in out} assert by_item["i1"] == by_item["i2"], "paraphrase-near items must merge" assert by_item["i3"] != by_item["i1"], "unrelated item must not merge" def test_items_outside_the_window_do_not_merge(con): v = unit(1, 0, 0) rows = [add(con, "i1", "s1", "2026-01-01T00:00:00+00:00", "h1"), add(con, "i2", "s2", "2026-01-05T00:00:00+00:00", "h2")] # 4 days later out = cluster.assign(con, rows, {"h1": v, "h2": v}) assert out[0].cluster_id != out[1].cluster_id def test_bucket_count_counts_newsrooms_not_feeds(con): """Two feeds from one newsroom are one independent confirmation, not two.""" v = unit(1, 0, 0) store.upsert_source( con, id="s3", name="A2", url="http://a2", kind="rss", bucket="a", weight=1.0, prominence=0, enabled=1, poll_interval_secs=900) rows = [add(con, "i1", "s1", "2026-01-01T00:00:00+00:00", "h1"), add(con, "i2", "s3", "2026-01-01T01:00:00+00:00", "h2")] out = cluster.assign(con, rows, {"h1": v, "h2": v}) assert out[0].cluster_id == out[1].cluster_id assert cluster.bucket_count(con, out[0].cluster_id) == 1 rows2 = [add(con, "i3", "s2", "2026-01-01T02:00:00+00:00", "h3")] cluster.assign(con, rows2, {"h3": v}) assert cluster.bucket_count(con, out[0].cluster_id) == 2 def test_assignment_is_idempotent(con): v = unit(1, 0, 0) rows = [add(con, "i1", "s1", "2026-01-01T00:00:00+00:00", "h1")] a = cluster.assign(con, rows, {"h1": v}) b = cluster.assign(con, rows, {"h1": v}) assert a[0].cluster_id == b[0].cluster_id n = con.execute("SELECT COUNT(*) c FROM cluster_members").fetchone()["c"] assert n == 1, "re-assigning an item must not duplicate membership" def test_syndicated_partners_in_one_bucket_count_once(con): """Regression: Ohio Capital Journal and Kentucky Lantern are both States Newsroom and carried 10 identical headlines in one poll. Configured as separate buckets they would have doubled the corroboration signal.""" v = unit(1, 0, 0) for sid, name in (("sn1", "Ohio Capital Journal"), ("sn2", "Kentucky Lantern")): store.upsert_source(con, id=sid, name=name, url=f"http://{sid}", kind="rss", bucket="states-newsroom", weight=1.0, prominence=0, enabled=1, poll_interval_secs=900) rows = [add(con, "s1i", "sn1", "2026-01-01T00:00:00+00:00", "hs1"), add(con, "s2i", "sn2", "2026-01-01T00:30:00+00:00", "hs2")] out = cluster.assign(con, rows, {"hs1": v, "hs2": v}) assert out[0].cluster_id == out[1].cluster_id assert cluster.bucket_count(con, out[0].cluster_id) == 1, ( "syndicated partners must not read as independent confirmation")