Handle paywalled sources; add free New York coverage

The NYT is unreadable without a subscription, but dropping it would throw away
real signal -- knowing the NYT covered a story is corroboration, and it feeds
importance. So sources carry a `paywalled` flag: they still cluster and
corroborate, but are never chosen as the link for a notification or a headline
when a readable report of the same story exists, and are marked in the UI.

Deliberately NOT doing paywall circumvention. The fix is to prefer a source you
can actually read, and to add outlets that do not have the problem.

Five free New York outlets added, all verified live: The City, Gothamist,
New York Focus, Documented NY and amNY -- independent and largely nonprofit,
covering the NYC politics the NYT was being relied on for. Hell Gate was probed
too and has no discoverable feed at any of the usual paths.

Includes an additive migration for the new column, since CREATE TABLE IF NOT
EXISTS will not add a column to a table that already exists.
This commit is contained in:
connor committed 2026-09-09 10:01:43 -04:00
1 parent 98bba62976
commit 9cba8db7bb
8 files changed
+127 -14

No files matched your search

+40
View File
@@ -354,3 +354,43 @@ def test_sort_control_offers_every_order_and_marks_the_active_one(tmp_path, monk
def test_unknown_sort_falls_back_rather_than_erroring(tmp_path, monkeypatch):
html = _triage_db(tmp_path, monkeypatch).get("/?sort=../../etc/passwd").text
assert 'value="best" selected' in html
def test_a_paywalled_source_is_never_the_link_when_a_readable_one_exists(tmp_path, monkeypatch):
"""Knowing the NYT covered a story is real corroboration; being handed its
unreadable link is not useful. Readable reports lead."""
import numpy as np
from datetime import UTC, datetime
monkeypatch.setenv("NEWS_DB", str(tmp_path / "p.db"))
monkeypatch.setenv("NEWS_FEEDBACK_SECRET", SECRET)
con = store.connect(tmp_path / "p.db")
now = datetime.now(UTC)
store.upsert_source(con, id="nyt", name="NYT", url="http://nyt", kind="rss",
bucket="nyt", weight=1.0, prominence=0, enabled=1,
poll_interval_secs=900, paywalled=1)
store.upsert_source(con, id="free", name="Gothamist", url="http://g", kind="rss",
bucket="g", weight=1.0, prominence=0, enabled=1,
poll_interval_secs=900, paywalled=0)
# The paywalled item is NEWER, so recency alone would put it first.
for iid, sid, mins in (("free_i", "free", 30), ("nyt_i", "nyt", 1)):
when = now.replace(microsecond=0).isoformat()
con.execute("""INSERT INTO items (id,source_id,title,summary,url,canonical_url,
published_at,first_seen_at,content_hash)
VALUES (?,?,?,'',?,?,?,?,?)""",
(iid, sid, f"Report from {sid}", f"http://x/{iid}", f"http://x/{iid}",
when if sid == "nyt" else "2026-01-01T00:00:00+00:00",
when, iid))
con.execute("""INSERT INTO clusters (id,centroid,size,first_seen_at,last_seen_at,
lead_item_id) VALUES ('c',?,2,?,?,'free_i')""",
(store.to_blob(np.zeros(384, dtype="float32")),
now.isoformat(), now.isoformat()))
for iid in ("free_i", "nyt_i"):
con.execute("INSERT INTO cluster_members VALUES ('c',?,?)", (iid, now.isoformat()))
from news import web
web._VECTORS = None
html = TestClient(web.app).get("/?show=all").text
assert html.index("Report from free") < html.index("Report from nyt"), (
"the readable report must be listed first even though it is older")
assert "🔒" in html, "paywalled outlets should be marked"