diff --git a/news/cli.py b/news/cli.py
index 1319502..3aa2586 100644
--- a/news/cli.py
+++ b/news/cli.py
@@ -177,8 +177,14 @@ def cmd_notify(args) -> int:
for d in sending:
s = by[d.cluster_id]
link = con.execute(
- """SELECT i.url FROM cluster_members cm JOIN items i ON i.id=cm.item_id
- WHERE cm.cluster_id=? LIMIT 1""", (d.cluster_id,)).fetchone()
+ """SELECT i.url FROM cluster_members cm
+ JOIN items i ON i.id = cm.item_id
+ JOIN sources so ON so.id = i.source_id
+ WHERE cm.cluster_id = ?
+ -- Never hand the phone a paywalled link when a readable
+ -- report of the same story exists.
+ ORDER BY so.paywalled ASC, i.published_at DESC
+ LIMIT 1""", (d.cluster_id,)).fetchone()
url = link["url"] if link else ""
body = f"{s.corroboration.origins} source(s) ยท {d.reason}"
if args.dry_run:
diff --git a/news/config.py b/news/config.py
index c9626ad..2708d42 100644
--- a/news/config.py
+++ b/news/config.py
@@ -137,6 +137,7 @@ class Config:
merged.setdefault("bucket", merged["name"].lower().replace(" ", "-"))
merged.setdefault("weight", 1.0)
merged.setdefault("prominence", 0)
+ merged["paywalled"] = 1 if merged.get("paywalled") else 0
merged.setdefault("enabled", True)
merged.setdefault("poll_interval_secs", 900)
out.append(merged)
diff --git a/news/pipeline.py b/news/pipeline.py
index b046d75..8b12f79 100644
--- a/news/pipeline.py
+++ b/news/pipeline.py
@@ -50,7 +50,7 @@ def sync_sources(con: sqlite3.Connection, cfg: Config) -> list[dict]:
src["enabled"] = 1 if src.get("enabled", True) else 0
store.upsert_source(con, **{k: src[k] for k in (
"id", "name", "url", "kind", "bucket", "weight",
- "prominence", "enabled", "poll_interval_secs")})
+ "prominence", "enabled", "poll_interval_secs", "paywalled")})
if src["enabled"]:
row = con.execute(
"SELECT etag, last_modified FROM sources WHERE id=?", (src["id"],)
diff --git a/news/store.py b/news/store.py
index b06e34e..6105167 100644
--- a/news/store.py
+++ b/news/store.py
@@ -31,6 +31,11 @@ CREATE TABLE IF NOT EXISTS sources (
prominence INTEGER NOT NULL DEFAULT 0,
enabled INTEGER NOT NULL DEFAULT 1,
poll_interval_secs INTEGER NOT NULL DEFAULT 900,
+ -- 1 when articles are behind a hard paywall. Such a source still counts for
+ -- clustering and corroboration -- knowing the NYT covered something is real
+ -- signal -- but it must not be the link you are handed, because the link does
+ -- not work.
+ paywalled INTEGER NOT NULL DEFAULT 0,
etag TEXT,
last_modified TEXT,
last_polled_at TEXT,
@@ -136,6 +141,11 @@ def connect(path: str | Path) -> sqlite3.Connection:
# that must not drop writes.
con.execute(f"PRAGMA busy_timeout={int(BUSY_TIMEOUT_SECS * 1000)}")
con.executescript(SCHEMA)
+ # Additive migration: CREATE TABLE IF NOT EXISTS will not add a column to a
+ # table that already exists.
+ cols = {r["name"] for r in con.execute("PRAGMA table_info(sources)")}
+ if "paywalled" not in cols:
+ con.execute("ALTER TABLE sources ADD COLUMN paywalled INTEGER NOT NULL DEFAULT 0")
return con
@@ -149,15 +159,16 @@ def from_blob(blob: bytes) -> np.ndarray:
def upsert_source(con: sqlite3.Connection, **kw) -> None:
"""Config is authoritative for everything except the polling state columns."""
+ kw.setdefault("paywalled", 0)
con.execute(
"""INSERT INTO sources (id,name,url,kind,bucket,weight,prominence,
- enabled,poll_interval_secs)
+ enabled,poll_interval_secs,paywalled)
VALUES (:id,:name,:url,:kind,:bucket,:weight,:prominence,
- :enabled,:poll_interval_secs)
+ :enabled,:poll_interval_secs,:paywalled)
ON CONFLICT(url) DO UPDATE SET
name=excluded.name, kind=excluded.kind, bucket=excluded.bucket,
weight=excluded.weight, prominence=excluded.prominence,
- enabled=excluded.enabled,
+ enabled=excluded.enabled, paywalled=excluded.paywalled,
poll_interval_secs=excluded.poll_interval_secs""",
kw,
)
diff --git a/news/templates/triage.html b/news/templates/triage.html
index 84bdada..6a745f8 100644
--- a/news/templates/triage.html
+++ b/news/templates/triage.html
@@ -113,7 +113,7 @@
{% for s in r.sources[:3] %}
{{ s.when }}
{{ s.title }}
- {{ s.name }}
+ {{ s.name }}{% if s.paywalled %} ๐{% endif %}
{% endfor %}
{% if r.sources|length > 3 %}
@@ -124,7 +124,7 @@
{% for s in r.sources[3:] %}
{{ s.when }}
{{ s.title }}
- {{ s.name }}
+ {{ s.name }}{% if s.paywalled %} ๐{% endif %}
{% endfor %}
diff --git a/news/web.py b/news/web.py
index 4108c8e..a590043 100644
--- a/news/web.py
+++ b/news/web.py
@@ -125,11 +125,13 @@ async def triage(request):
# items and multi-item clusters hold 249 between them, so rendering the
# lot inline is cheaper than a lazy-loading endpoint would be to build.
members = con.execute(
- """SELECT i.title, i.url, i.published_at, so.name FROM cluster_members cm
+ """SELECT i.title, i.url, i.published_at, so.name, so.paywalled
+ FROM cluster_members cm
JOIN items i ON i.id = cm.item_id
JOIN sources so ON so.id = i.source_id
WHERE cm.cluster_id = ?
- ORDER BY i.published_at DESC""", (s.cluster_id,)).fetchall()
+ -- Readable sources first, so the headline link actually opens.
+ ORDER BY so.paywalled ASC, i.published_at DESC""", (s.cluster_id,)).fetchall()
rows.append({
"id": s.cluster_id, "headline": s.lead_title,
"topic": s.interest.block, "importance_topic": s.importance.block,
@@ -145,7 +147,8 @@ async def triage(request):
"tok_yes": sign(secret, s.cluster_id, "interested") if secret else "",
"tok_no": sign(secret, s.cluster_id, "not_interested") if secret else "",
"sources": [{"name": m["name"], "title": m["title"], "url": m["url"],
- "when": (m["published_at"] or "")[11:16]}
+ "when": (m["published_at"] or "")[11:16],
+ "paywalled": bool(m["paywalled"])}
for m in members],
})
if len(rows) >= limit:
@@ -394,16 +397,19 @@ async def digest_page(request):
stories = []
for r in rows:
members = con.execute(
- """SELECT i.title, i.url, i.published_at, s.name FROM cluster_members cm
+ """SELECT i.title, i.url, i.published_at, s.name, s.paywalled
+ FROM cluster_members cm
JOIN items i ON i.id = cm.item_id
JOIN sources s ON s.id = i.source_id
- WHERE cm.cluster_id = ? ORDER BY i.published_at DESC LIMIT 6""",
+ WHERE cm.cluster_id = ?
+ ORDER BY s.paywalled ASC, i.published_at DESC LIMIT 6""",
(r["cluster_id"],),
).fetchall()
stories.append({
"id": r["cluster_id"], "headline": r["headline"], "lane": r["lane"],
"sources": [{"name": m["name"], "title": m["title"], "url": m["url"],
- "when": (m["published_at"] or "")[11:16]}
+ "when": (m["published_at"] or "")[11:16],
+ "paywalled": bool(m["paywalled"])}
for m in members],
})
diff --git a/profile/sources.toml b/profile/sources.toml
index f5c8616..089c97a 100644
--- a/profile/sources.toml
+++ b/profile/sources.toml
@@ -6,6 +6,11 @@
# confirmations. Getting this wrong is what made the old system fire priority-5
# pushes on anything two feeds happened to carry.
#
+# `paywalled = true` marks a source whose articles cannot be read without a
+# subscription. Such a source still contributes to clustering and corroboration,
+# but is never chosen as the link for a notification or a headline when a
+# readable report of the same story exists.
+#
# `weight` is editorial trust, carried over from Connor's old config where he
# had deliberately set Al Jazeera highest (1.2) and NYT lowest (0.8). It affects
# importance and tie-breaking, never whether an item is visible at all.
@@ -42,11 +47,16 @@ name = "Guardian World"
url = "https://www.theguardian.com/world/rss"
bucket = "guardian"
+# Kept, but marked paywalled. Its articles cannot actually be read without a
+# subscription, so it must never be the link you are handed -- but knowing the
+# NYT covered a story is still real corroboration, and dropping it would throw
+# that signal away. See `paywalled` above.
[[source]]
name = "NYT World"
url = "https://rss.nytimes.com/services/xml/rss/nyt/World.xml"
bucket = "nyt"
weight = 0.8
+paywalled = true
[[source]]
name = "NPR News"
@@ -411,3 +421,42 @@ weight = 1.1
url = "https://www.dropsitenews.com/feed"
bucket = "drop-site"
poll_interval_secs = 7200
+
+
+# ==================================================== New York / local left ====
+# Added 2026-09-09. NYC politics matters to Connor and the NYT is unreadable
+# without a subscription, so these are free, independent outlets covering the
+# same ground. All verified live.
+
+[[source]]
+name = "The City"
+url = "https://www.thecity.nyc/rss/"
+bucket = "the-city-nyc"
+weight = 1.1
+poll_interval_secs = 3600
+
+[[source]]
+name = "Gothamist"
+url = "https://gothamist.com/feed"
+bucket = "gothamist"
+poll_interval_secs = 3600
+
+[[source]]
+name = "New York Focus"
+url = "https://www.nysfocus.com/feed"
+bucket = "ny-focus"
+weight = 1.1
+poll_interval_secs = 7200
+
+[[source]]
+name = "Documented NY"
+url = "https://documentedny.com/feed/"
+bucket = "documented-ny"
+weight = 1.1
+poll_interval_secs = 7200
+
+[[source]]
+name = "amNY"
+url = "https://www.amny.com/feed/"
+bucket = "amny"
+poll_interval_secs = 7200
diff --git a/tests/test_delivery.py b/tests/test_delivery.py
index dbd5409..b6920a0 100644
--- a/tests/test_delivery.py
+++ b/tests/test_delivery.py
@@ -354,3 +354,43 @@ def test_sort_control_offers_every_order_and_marks_the_active_one(tmp_path, monk
def test_unknown_sort_falls_back_rather_than_erroring(tmp_path, monkeypatch):
html = _triage_db(tmp_path, monkeypatch).get("/?sort=../../etc/passwd").text
assert 'value="best" selected' in html
+
+
+def test_a_paywalled_source_is_never_the_link_when_a_readable_one_exists(tmp_path, monkeypatch):
+ """Knowing the NYT covered a story is real corroboration; being handed its
+ unreadable link is not useful. Readable reports lead."""
+ import numpy as np
+ from datetime import UTC, datetime
+
+ monkeypatch.setenv("NEWS_DB", str(tmp_path / "p.db"))
+ monkeypatch.setenv("NEWS_FEEDBACK_SECRET", SECRET)
+ con = store.connect(tmp_path / "p.db")
+ now = datetime.now(UTC)
+ store.upsert_source(con, id="nyt", name="NYT", url="http://nyt", kind="rss",
+ bucket="nyt", weight=1.0, prominence=0, enabled=1,
+ poll_interval_secs=900, paywalled=1)
+ store.upsert_source(con, id="free", name="Gothamist", url="http://g", kind="rss",
+ bucket="g", weight=1.0, prominence=0, enabled=1,
+ poll_interval_secs=900, paywalled=0)
+ # The paywalled item is NEWER, so recency alone would put it first.
+ for iid, sid, mins in (("free_i", "free", 30), ("nyt_i", "nyt", 1)):
+ when = now.replace(microsecond=0).isoformat()
+ con.execute("""INSERT INTO items (id,source_id,title,summary,url,canonical_url,
+ published_at,first_seen_at,content_hash)
+ VALUES (?,?,?,'',?,?,?,?,?)""",
+ (iid, sid, f"Report from {sid}", f"http://x/{iid}", f"http://x/{iid}",
+ when if sid == "nyt" else "2026-01-01T00:00:00+00:00",
+ when, iid))
+ con.execute("""INSERT INTO clusters (id,centroid,size,first_seen_at,last_seen_at,
+ lead_item_id) VALUES ('c',?,2,?,?,'free_i')""",
+ (store.to_blob(np.zeros(384, dtype="float32")),
+ now.isoformat(), now.isoformat()))
+ for iid in ("free_i", "nyt_i"):
+ con.execute("INSERT INTO cluster_members VALUES ('c',?,?)", (iid, now.isoformat()))
+
+ from news import web
+ web._VECTORS = None
+ html = TestClient(web.app).get("/?show=all").text
+ assert html.index("Report from free") < html.index("Report from nyt"), (
+ "the readable report must be listed first even though it is older")
+ assert "๐" in html, "paywalled outlets should be marked"