connor/news
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Handle paywalled sources; add free New York coverage

Browse Source 
The NYT is unreadable without a subscription, but dropping it would throw away
real signal -- knowing the NYT covered a story is corroboration, and it feeds
importance. So sources carry a `paywalled` flag: they still cluster and
corroborate, but are never chosen as the link for a notification or a headline
when a readable report of the same story exists, and are marked in the UI.

Deliberately NOT doing paywall circumvention. The fix is to prefer a source you
can actually read, and to add outlets that do not have the problem.

Five free New York outlets added, all verified live: The City, Gothamist,
New York Focus, Documented NY and amNY -- independent and largely nonprofit,
covering the NYC politics the NYT was being relied on for. Hell Gate was probed
too and has no discoverable feed at any of the usual paths.

Includes an additive migration for the new column, since CREATE TABLE IF NOT
EXISTS will not add a column to a table that already exists.
This commit is contained in:
connor committed 2026-09-09 10:01:43 -04:00
1 parent 98bba62976
commit 9cba8db7bb
8 files changed
+127 -14
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

news/cli.py
+8 -2
View File
@@ -177,8 +177,14 @@ def cmd_notify(args) -> int:
for d in sending:
s = by[d.cluster_id]
link = con.execute(
"""SELECT i.url FROM cluster_members cm JOIN items i ON i.id=cm.item_id
WHERE cm.cluster_id=? LIMIT 1""", (d.cluster_id,)).fetchone()
"""SELECT i.url FROM cluster_members cm
JOIN items i ON i.id = cm.item_id
JOIN sources so ON so.id = i.source_id
WHERE cm.cluster_id = ?
-- Never hand the phone a paywalled link when a readable
-- report of the same story exists.
ORDER BY so.paywalled ASC, i.published_at DESC
LIMIT 1""", (d.cluster_id,)).fetchone()
url = link["url"] if link else ""
body = f"{s.corroboration.origins} source(s) · {d.reason}"
if args.dry_run:
news/config.py
+1
View File
@@ -137,6 +137,7 @@ class Config:
merged.setdefault("bucket", merged["name"].lower().replace(" ", "-"))
merged.setdefault("weight", 1.0)
merged.setdefault("prominence", 0)
merged["paywalled"] = 1 if merged.get("paywalled") else 0
merged.setdefault("enabled", True)
merged.setdefault("poll_interval_secs", 900)
out.append(merged)
news/pipeline.py
+1 -1
View File
@@ -50,7 +50,7 @@ def sync_sources(con: sqlite3.Connection, cfg: Config) -> list[dict]:
src["enabled"] = 1 if src.get("enabled", True) else 0
store.upsert_source(con, **{k: src[k] for k in (
"id", "name", "url", "kind", "bucket", "weight",
"prominence", "enabled", "poll_interval_secs")})
"prominence", "enabled", "poll_interval_secs", "paywalled")})
if src["enabled"]:
row = con.execute(
"SELECT etag, last_modified FROM sources WHERE id=?", (src["id"],)
news/store.py
+14 -3
View File
@@ -31,6 +31,11 @@ CREATE TABLE IF NOT EXISTS sources (
prominence INTEGER NOT NULL DEFAULT 0,
enabled INTEGER NOT NULL DEFAULT 1,
poll_interval_secs INTEGER NOT NULL DEFAULT 900,
-- 1 when articles are behind a hard paywall. Such a source still counts for
-- clustering and corroboration -- knowing the NYT covered something is real
-- signal -- but it must not be the link you are handed, because the link does
-- not work.
paywalled INTEGER NOT NULL DEFAULT 0,
etag TEXT,
last_modified TEXT,
last_polled_at TEXT,
@@ -136,6 +141,11 @@ def connect(path: str | Path) -> sqlite3.Connection:
# that must not drop writes.
con.execute(f"PRAGMA busy_timeout={int(BUSY_TIMEOUT_SECS * 1000)}")
con.executescript(SCHEMA)
# Additive migration: CREATE TABLE IF NOT EXISTS will not add a column to a
# table that already exists.
cols = {r["name"] for r in con.execute("PRAGMA table_info(sources)")}
if "paywalled" not in cols:
con.execute("ALTER TABLE sources ADD COLUMN paywalled INTEGER NOT NULL DEFAULT 0")
return con
@@ -149,15 +159,16 @@ def from_blob(blob: bytes) -> np.ndarray:
def upsert_source(con: sqlite3.Connection, **kw) -> None:
"""Config is authoritative for everything except the polling state columns."""
kw.setdefault("paywalled", 0)
con.execute(
"""INSERT INTO sources (id,name,url,kind,bucket,weight,prominence,
enabled,poll_interval_secs)
enabled,poll_interval_secs,paywalled)
VALUES (:id,:name,:url,:kind,:bucket,:weight,:prominence,
:enabled,:poll_interval_secs)
:enabled,:poll_interval_secs,:paywalled)
ON CONFLICT(url) DO UPDATE SET
name=excluded.name, kind=excluded.kind, bucket=excluded.bucket,
weight=excluded.weight, prominence=excluded.prominence,
enabled=excluded.enabled,
enabled=excluded.enabled, paywalled=excluded.paywalled,
poll_interval_secs=excluded.poll_interval_secs""",
kw,
)
news/templates/triage.html
+2 -2
View File
@@ -113,7 +113,7 @@
{% for s in r.sources[:3] %}
<li><span class="when">{{ s.when }}</span>
<a href="{{ s.url }}" target="_blank" rel="noopener">{{ s.title }}</a>
<span class="outlet">{{ s.name }}</span></li>
<span class="outlet">{{ s.name }}{% if s.paywalled %} 🔒{% endif %}</span></li>
{% endfor %}
</ul>
{% if r.sources|length > 3 %}
@@ -124,7 +124,7 @@
{% for s in r.sources[3:] %}
<li><span class="when">{{ s.when }}</span>
<a href="{{ s.url }}" target="_blank" rel="noopener">{{ s.title }}</a>
<span class="outlet">{{ s.name }}</span></li>
<span class="outlet">{{ s.name }}{% if s.paywalled %} 🔒{% endif %}</span></li>
{% endfor %}
</ul>
</details>
news/web.py
+12 -6
View File
@@ -125,11 +125,13 @@ async def triage(request):
# items and multi-item clusters hold 249 between them, so rendering the
# lot inline is cheaper than a lazy-loading endpoint would be to build.
members = con.execute(
"""SELECT i.title, i.url, i.published_at, so.name FROM cluster_members cm
"""SELECT i.title, i.url, i.published_at, so.name, so.paywalled
FROM cluster_members cm
JOIN items i ON i.id = cm.item_id
JOIN sources so ON so.id = i.source_id
WHERE cm.cluster_id = ?
ORDER BY i.published_at DESC""", (s.cluster_id,)).fetchall()
-- Readable sources first, so the headline link actually opens.
ORDER BY so.paywalled ASC, i.published_at DESC""", (s.cluster_id,)).fetchall()
rows.append({
"id": s.cluster_id, "headline": s.lead_title,
"topic": s.interest.block, "importance_topic": s.importance.block,
@@ -145,7 +147,8 @@ async def triage(request):
"tok_yes": sign(secret, s.cluster_id, "interested") if secret else "",
"tok_no": sign(secret, s.cluster_id, "not_interested") if secret else "",
"sources": [{"name": m["name"], "title": m["title"], "url": m["url"],
"when": (m["published_at"] or "")[11:16]}
"when": (m["published_at"] or "")[11:16],
"paywalled": bool(m["paywalled"])}
for m in members],
})
if len(rows) >= limit:
@@ -394,16 +397,19 @@ async def digest_page(request):
stories = []
for r in rows:
members = con.execute(
"""SELECT i.title, i.url, i.published_at, s.name FROM cluster_members cm
"""SELECT i.title, i.url, i.published_at, s.name, s.paywalled
FROM cluster_members cm
JOIN items i ON i.id = cm.item_id
JOIN sources s ON s.id = i.source_id
WHERE cm.cluster_id = ? ORDER BY i.published_at DESC LIMIT 6""",
WHERE cm.cluster_id = ?
ORDER BY s.paywalled ASC, i.published_at DESC LIMIT 6""",
(r["cluster_id"],),
).fetchall()
stories.append({
"id": r["cluster_id"], "headline": r["headline"], "lane": r["lane"],
"sources": [{"name": m["name"], "title": m["title"], "url": m["url"],
"when": (m["published_at"] or "")[11:16]}
"when": (m["published_at"] or "")[11:16],
"paywalled": bool(m["paywalled"])}
for m in members],
})
profile/sources.toml
+49
View File
@@ -6,6 +6,11 @@
# confirmations. Getting this wrong is what made the old system fire priority-5
# pushes on anything two feeds happened to carry.
#
# `paywalled = true` marks a source whose articles cannot be read without a
# subscription. Such a source still contributes to clustering and corroboration,
# but is never chosen as the link for a notification or a headline when a
# readable report of the same story exists.
#
# `weight` is editorial trust, carried over from Connor's old config where he
# had deliberately set Al Jazeera highest (1.2) and NYT lowest (0.8). It affects
# importance and tie-breaking, never whether an item is visible at all.
@@ -42,11 +47,16 @@ name = "Guardian World"
url = "https://www.theguardian.com/world/rss"
bucket = "guardian"
# Kept, but marked paywalled. Its articles cannot actually be read without a
# subscription, so it must never be the link you are handed -- but knowing the
# NYT covered a story is still real corroboration, and dropping it would throw
# that signal away. See `paywalled` above.
[[source]]
name = "NYT World"
url = "https://rss.nytimes.com/services/xml/rss/nyt/World.xml"
bucket = "nyt"
weight = 0.8
paywalled = true
[[source]]
name = "NPR News"
@@ -411,3 +421,42 @@ weight = 1.1
url = "https://www.dropsitenews.com/feed"
bucket = "drop-site"
poll_interval_secs = 7200
# ==================================================== New York / local left ====
# Added 2026-09-09. NYC politics matters to Connor and the NYT is unreadable
# without a subscription, so these are free, independent outlets covering the
# same ground. All verified live.
[[source]]
name = "The City"
url = "https://www.thecity.nyc/rss/"
bucket = "the-city-nyc"
weight = 1.1
poll_interval_secs = 3600
[[source]]
name = "Gothamist"
url = "https://gothamist.com/feed"
bucket = "gothamist"
poll_interval_secs = 3600
[[source]]
name = "New York Focus"
url = "https://www.nysfocus.com/feed"
bucket = "ny-focus"
weight = 1.1
poll_interval_secs = 7200
[[source]]
name = "Documented NY"
url = "https://documentedny.com/feed/"
bucket = "documented-ny"
weight = 1.1
poll_interval_secs = 7200
[[source]]
name = "amNY"
url = "https://www.amny.com/feed/"
bucket = "amny"
poll_interval_secs = 7200
tests/test_delivery.py
+40
View File
@@ -354,3 +354,43 @@ def test_sort_control_offers_every_order_and_marks_the_active_one(tmp_path, monk
def test_unknown_sort_falls_back_rather_than_erroring(tmp_path, monkeypatch):
html = _triage_db(tmp_path, monkeypatch).get("/?sort=../../etc/passwd").text
assert 'value="best" selected' in html
def test_a_paywalled_source_is_never_the_link_when_a_readable_one_exists(tmp_path, monkeypatch):
"""Knowing the NYT covered a story is real corroboration; being handed its
unreadable link is not useful. Readable reports lead."""
import numpy as np
from datetime import UTC, datetime
monkeypatch.setenv("NEWS_DB", str(tmp_path / "p.db"))
monkeypatch.setenv("NEWS_FEEDBACK_SECRET", SECRET)
con = store.connect(tmp_path / "p.db")
now = datetime.now(UTC)
store.upsert_source(con, id="nyt", name="NYT", url="http://nyt", kind="rss",
bucket="nyt", weight=1.0, prominence=0, enabled=1,
poll_interval_secs=900, paywalled=1)
store.upsert_source(con, id="free", name="Gothamist", url="http://g", kind="rss",
bucket="g", weight=1.0, prominence=0, enabled=1,
poll_interval_secs=900, paywalled=0)
# The paywalled item is NEWER, so recency alone would put it first.
for iid, sid, mins in (("free_i", "free", 30), ("nyt_i", "nyt", 1)):
when = now.replace(microsecond=0).isoformat()
con.execute("""INSERT INTO items (id,source_id,title,summary,url,canonical_url,
published_at,first_seen_at,content_hash)
VALUES (?,?,?,'',?,?,?,?,?)""",
(iid, sid, f"Report from {sid}", f"http://x/{iid}", f"http://x/{iid}",
when if sid == "nyt" else "2026-01-01T00:00:00+00:00",
when, iid))
con.execute("""INSERT INTO clusters (id,centroid,size,first_seen_at,last_seen_at,
lead_item_id) VALUES ('c',?,2,?,?,'free_i')""",
(store.to_blob(np.zeros(384, dtype="float32")),
now.isoformat(), now.isoformat()))
for iid in ("free_i", "nyt_i"):
con.execute("INSERT INTO cluster_members VALUES ('c',?,?)", (iid, now.isoformat()))
from news import web
web._VECTORS = None
html = TestClient(web.app).get("/?show=all").text
assert html.index("Report from free") < html.index("Report from nyt"), (
"the readable report must be listed first even though it is older")
assert "🔒" in html, "paywalled outlets should be marked"