Tiered corroboration: origins, pickups and depth scored separately

Connor's point: "several outlets carried this" covers situations that carry
different amounts of information, and origin_count() flattened them all to one
number. Splitting on same/different newsroom and same/different text gives four
cases:

  same newsroom, same text  -> pure duplicate (BBC World + BBC top). Zero
                               signal; the front-page placement it does carry is
                               already captured by sources.prominence.
  same newsroom, diff text  -> DEPTH. A newsroom filing four pieces on one story
                               is a real editorial judgement, previously
                               discarded entirely.
  diff newsroom, same text  -> PICKUPS. Syndicated copy; each carrier chose to
                               run it. Weak but non-zero.
  diff newsroom, diff text  -> ORIGINS. Independent judgement and independent
                               newsgathering. Strongest.

score = origins + 0.3*log1p(pickups) + 0.2*log1p(depth), weights in config.
Origins stay dominant; the others break ties among the many two- and
three-origin clusters, and log damping stops a high-volume feed inflating depth.

Measuring first changed the design. Depth outnumbers pickups 78 to 11 across 91
real clusters, because these sources are mostly national outlets writing their
own copy rather than local papers running wire -- so the more valuable half was
the one not originally proposed. UK settlements scores 10.55 from 10 origins
plus 15 same-newsroom pieces; AfD scores 5.55 from 5 origins plus 15.

Tests assert the four-case ordering directly, and that twenty pieces from one
newsroom still rank below two independent ones. Both needed a helper generating
vectors with exact pairwise cosine: the band between the 0.82 clustering
threshold and the 0.97 syndication threshold is narrow, and two earlier drafts
of these tests used ad-hoc vectors that fell outside it and asserted nothing.

tests/test_cluster.py

@@ -153,3 +153,114 @@ def test_origin_count_never_exceeds_bucket_count(con):
              ("x3", "s1", "2026-01-01T00:20:00+00:00", "hx")]
     cid = _cluster_of(con, specs, v)
     assert cluster.origin_count(con, cid) <= cluster.bucket_count(con, cid)
+
+
+def equicos(n, c=0.90, dim=None):
+    """n unit vectors with pairwise cosine exactly `c`.
+
+    Needed because these tests must sit in a narrow band: above the 0.82
+    clustering threshold so the items form ONE cluster, below the 0.97
+    syndication threshold so they read as distinct pieces of writing. Earlier
+    drafts of this test used ad-hoc vectors that fell outside it in both
+    directions and tested nothing.
+    """
+    dim = dim or n + 1
+    out = []
+    for i in range(n):
+        v = np.zeros(dim, dtype=np.float32)
+        v[0] = np.sqrt(c)
+        v[1 + i] = np.sqrt(1 - c)
+        out.append(v)
+    return out
+
+
+def _mk(con, spec, vec_for):
+    """spec: list of (item_id, source_id, title, content_hash). One cluster."""
+    when = "2026-01-01T00:{:02d}:00+00:00"
+    rows = []
+    for n, (iid, sid, title, chash) in enumerate(spec):
+        con.execute(
+            """INSERT INTO items (id,source_id,title,summary,url,canonical_url,
+                                  published_at,first_seen_at,content_hash)
+               VALUES (?,?,?,'',?,?,?,?,?)""",
+            (iid, sid, title, f"http://x/{iid}", f"http://x/{iid}",
+             when.format(n), when.format(n), chash))
+        v = vec_for(chash)
+        con.execute("INSERT OR REPLACE INTO embeddings (content_hash,dim,vec) VALUES (?,?,?)",
+                    (chash, len(v), store.to_blob(v)))
+    rows = con.execute("SELECT * FROM items ORDER BY published_at").fetchall()
+    v = {r["content_hash"]: vec_for(r["content_hash"]) for r in rows}
+    return cluster.assign(con, list(rows), v)[0].cluster_id
+
+
+def _sources(con, *pairs):
+    for sid, bucket in pairs:
+        store.upsert_source(con, id=sid, name=sid, url=f"http://{sid}", kind="rss",
+                            bucket=bucket, weight=1.0, prominence=0, enabled=1,
+                            poll_interval_secs=900)
+
+
+def test_corroboration_ranks_the_four_cases_in_the_right_order(con):
+    """The 2x2 that motivates the split, each case scored in isolation.
+
+    Ordering must be: four independent newsrooms > four outlets running one
+    wire story > one newsroom filing four pieces > one story on two feeds of
+    the same newsroom.
+    """
+    # "wire" is one shared vector (cosine 1.0 -> syndicated); t0..t3 and u0..u3
+    # are pairwise 0.90 -> one cluster, four distinct pieces of writing.
+    distinct = equicos(4, 0.90, dim=6)
+    table = {"wire": np.eye(6, dtype=np.float32)[0]}
+    for i in range(4):
+        table[f"t{i}"] = distinct[i]
+        table[f"u{i}"] = distinct[i]
+
+    def vec(h):
+        return table[h]
+
+    # (a) one newsroom, one story, two feeds -> pure duplicate
+    c1 = store.connect(":memory:")
+    _sources(c1, ("f1", "bbc"), ("f2", "bbc"))
+    cid = _mk(c1, [("a1", "f1", "Same headline", "wire"),
+                   ("a2", "f2", "Same headline", "wire")], vec)
+    dup = cluster.corroboration(c1, cid)
+
+    # (b) one newsroom, four distinct pieces -> depth
+    c2 = store.connect(":memory:")
+    _sources(c2, ("g1", "nyt"))
+    cid = _mk(c2, [(f"b{i}", "g1", f"Distinct angle {i}", f"t{i}") for i in range(4)], vec)
+    deep = cluster.corroboration(c2, cid)
+
+    # (c) four newsrooms, one syndicated wire story -> pickups
+    c3 = store.connect(":memory:")
+    _sources(c3, *[(f"h{i}", f"local{i}") for i in range(4)])
+    cid = _mk(c3, [(f"c{i}", f"h{i}", "Wire dispatch text", "wire") for i in range(4)], vec)
+    wire = cluster.corroboration(c3, cid)
+
+    # (d) four newsrooms each writing their own -> origins
+    c4 = store.connect(":memory:")
+    _sources(c4, *[(f"k{i}", f"nat{i}") for i in range(4)])
+    cid = _mk(c4, [(f"d{i}", f"k{i}", f"Own reporting {i}", f"u{i}") for i in range(4)], vec)
+    orig = cluster.corroboration(c4, cid)
+
+    assert dup.origins == 1 and dup.pickups == 0 and dup.depth == 0
+    assert deep.origins == 1 and deep.depth == 3
+    assert wire.origins == 1 and wire.pickups == 3
+    assert orig.origins == 4
+
+    assert orig.score > wire.score > deep.score > dup.score, (
+        f"expected origins > pickups > depth > duplicate, got "
+        f"{orig.score:.2f} / {wire.score:.2f} / {deep.score:.2f} / {dup.score:.2f}")
+
+
+def test_origins_dominate_the_tiebreakers(con):
+    """No amount of depth or pickups may outweigh one extra real newsroom."""
+    c = store.connect(":memory:")
+    _sources(c, ("m1", "one"))
+    vs = equicos(20, 0.90)
+    table = {f"v{i}": vs[i] for i in range(20)}
+    cid = _mk(c, [(f"e{i}", "m1", f"Piece {i}", f"v{i}") for i in range(20)],
+              lambda h: table[h])
+    twenty_deep = cluster.corroboration(c, cid)
+    assert twenty_deep.score < 2.0, (
+        "20 pieces from one newsroom must still rank below two independent ones")