Handle paywalled sources; add free New York coverage
The NYT is unreadable without a subscription, but dropping it would throw away real signal -- knowing the NYT covered a story is corroboration, and it feeds importance. So sources carry a `paywalled` flag: they still cluster and corroborate, but are never chosen as the link for a notification or a headline when a readable report of the same story exists, and are marked in the UI. Deliberately NOT doing paywall circumvention. The fix is to prefer a source you can actually read, and to add outlets that do not have the problem. Five free New York outlets added, all verified live: The City, Gothamist, New York Focus, Documented NY and amNY -- independent and largely nonprofit, covering the NYC politics the NYT was being relied on for. Hell Gate was probed too and has no discoverable feed at any of the usual paths. Includes an additive migration for the new column, since CREATE TABLE IF NOT EXISTS will not add a column to a table that already exists.
This commit is contained in:
1 parent
98bba62976
commit
9cba8db7bb
8 files changed
+127
-14
No files matched your search
@@ -354,3 +354,43 @@ def test_sort_control_offers_every_order_and_marks_the_active_one(tmp_path, monk
|
||||
def test_unknown_sort_falls_back_rather_than_erroring(tmp_path, monkeypatch):
|
||||
html = _triage_db(tmp_path, monkeypatch).get("/?sort=../../etc/passwd").text
|
||||
assert 'value="best" selected' in html
|
||||
|
||||
|
||||
def test_a_paywalled_source_is_never_the_link_when_a_readable_one_exists(tmp_path, monkeypatch):
|
||||
"""Knowing the NYT covered a story is real corroboration; being handed its
|
||||
unreadable link is not useful. Readable reports lead."""
|
||||
import numpy as np
|
||||
from datetime import UTC, datetime
|
||||
|
||||
monkeypatch.setenv("NEWS_DB", str(tmp_path / "p.db"))
|
||||
monkeypatch.setenv("NEWS_FEEDBACK_SECRET", SECRET)
|
||||
con = store.connect(tmp_path / "p.db")
|
||||
now = datetime.now(UTC)
|
||||
store.upsert_source(con, id="nyt", name="NYT", url="http://nyt", kind="rss",
|
||||
bucket="nyt", weight=1.0, prominence=0, enabled=1,
|
||||
poll_interval_secs=900, paywalled=1)
|
||||
store.upsert_source(con, id="free", name="Gothamist", url="http://g", kind="rss",
|
||||
bucket="g", weight=1.0, prominence=0, enabled=1,
|
||||
poll_interval_secs=900, paywalled=0)
|
||||
# The paywalled item is NEWER, so recency alone would put it first.
|
||||
for iid, sid, mins in (("free_i", "free", 30), ("nyt_i", "nyt", 1)):
|
||||
when = now.replace(microsecond=0).isoformat()
|
||||
con.execute("""INSERT INTO items (id,source_id,title,summary,url,canonical_url,
|
||||
published_at,first_seen_at,content_hash)
|
||||
VALUES (?,?,?,'',?,?,?,?,?)""",
|
||||
(iid, sid, f"Report from {sid}", f"http://x/{iid}", f"http://x/{iid}",
|
||||
when if sid == "nyt" else "2026-01-01T00:00:00+00:00",
|
||||
when, iid))
|
||||
con.execute("""INSERT INTO clusters (id,centroid,size,first_seen_at,last_seen_at,
|
||||
lead_item_id) VALUES ('c',?,2,?,?,'free_i')""",
|
||||
(store.to_blob(np.zeros(384, dtype="float32")),
|
||||
now.isoformat(), now.isoformat()))
|
||||
for iid in ("free_i", "nyt_i"):
|
||||
con.execute("INSERT INTO cluster_members VALUES ('c',?,?)", (iid, now.isoformat()))
|
||||
|
||||
from news import web
|
||||
web._VECTORS = None
|
||||
html = TestClient(web.app).get("/?show=all").text
|
||||
assert html.index("Report from free") < html.index("Report from nyt"), (
|
||||
"the readable report must be listed first even though it is older")
|
||||
assert "🔒" in html, "paywalled outlets should be marked"
|
||||
Reference in new issue
Block a user