connor
64690ecfe0
Pipeline: fetch -> normalise -> embed (CPU) -> cluster. `news poll` runs a cycle; `news sources` and `news clusters` inspect the result. Replay and the live service share run_cycle so tuning and production cannot diverge -- the old system's replay harness hardcoded default params and could never sweep the thresholds it existed to tune. Calibrated, not guessed: - Cluster threshold 0.82, from reading real output. 0.70 merged "Solheim Cup capsules" with "FIBA Women's World Cup Glance"; 0.82 gives clean groups (76 items on the AfD Saxony-Anhalt election, 27 on the Nepal tunnel rescue). The old system produced 515 clusters of which 514 were singletons. - Corroboration counts DISTINCT buckets, not feeds, so ten outlets reprinting one wire story is one confirmation rather than ten. Removed after measurement: a SimHash near-duplicate stage. It gave +1.2 bits of separation on titles and -0.2 with summaries -- news duplicates are paraphrases, which is the same lexical trap that sank the old Jaccard clustering. Sources: all 39 fetched against the live web, not assumed. Dropped CBC (every feed URL times out from this host) and Tech Policy Press (serves HTML at every documented feed path). Fixed WVXU (index.rss is an empty stub; local-news.rss works). Reddit moved to .rss because .json now 403s without OAuth, and its intervals raised because it 429s after three quick requests. Deutsche Welle is back: it ships RSS 1.0/RDF, which feedparser reads and the old hand-rolled quick-xml parser could not.
82 lines
2.6 KiB
Python
82 lines
2.6 KiB
Python
"""RSS/Atom parsing, including the RDF format the old system could not read."""
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
from news.sources import rss
|
|
|
|
RDF = """<?xml version="1.0"?>
|
|
<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
|
|
xmlns="http://purl.org/rss/1.0/">
|
|
<channel rdf:about="http://x"><title>T</title><link>http://x</link>
|
|
<description>d</description></channel>
|
|
<item rdf:about="http://x/1">
|
|
<title>German economy frets about election result</title>
|
|
<link>http://x/1?utm_source=rss</link>
|
|
<description>Some <b>markup</b> here</description>
|
|
</item>
|
|
</rdf:RDF>"""
|
|
|
|
ATOM = """<?xml version="1.0"?>
|
|
<feed xmlns="http://www.w3.org/2005/Atom">
|
|
<title>F</title>
|
|
<entry>
|
|
<title>Release v1.2.3</title>
|
|
<link rel="alternate" href="http://x/rel"/>
|
|
<link rel="self" href="http://x/self"/>
|
|
<updated>2026-09-08T12:00:00Z</updated>
|
|
</entry>
|
|
</feed>"""
|
|
|
|
|
|
def _source():
|
|
return {"id": "s1", "url": "http://feed", "kind": "rss"}
|
|
|
|
|
|
async def _fetch(body: str, status: int = 200, headers: dict | None = None):
|
|
def handler(request):
|
|
return httpx.Response(status, content=body.encode(), headers=headers or {})
|
|
|
|
transport = httpx.MockTransport(handler)
|
|
async with httpx.AsyncClient(transport=transport) as client:
|
|
return await rss.fetch(client, _source())
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_parses_rss_1_rdf():
|
|
"""DW ships RDF; the old hand-rolled parser yielded zero items silently."""
|
|
items, _ = await _fetch(RDF)
|
|
assert len(items) == 1
|
|
assert items[0]["title"].startswith("German economy")
|
|
# tracking parameter stripped, markup removed from the summary
|
|
assert items[0]["canonical_url"] == "http://x/1"
|
|
assert "<b>" not in items[0]["summary"] and "markup" in items[0]["summary"]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_atom_prefers_alternate_link_over_self():
|
|
items, _ = await _fetch(ATOM)
|
|
assert len(items) == 1
|
|
assert items[0]["canonical_url"] == "http://x/rel"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_304_returns_no_items_and_is_not_an_error():
|
|
items, state = await _fetch("", status=304)
|
|
assert items == [] and state["status"] == "304"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_429_is_reported_as_rate_limited_not_raised():
|
|
"""Reddit 429s readily; that must back off, not raise every cycle."""
|
|
items, state = await _fetch("", status=429, headers={"retry-after": "120"})
|
|
assert items == []
|
|
assert state["rate_limited"] is True
|
|
assert state["retry_after"] == "120"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_server_error_still_raises():
|
|
with pytest.raises(httpx.HTTPStatusError):
|
|
await _fetch("", status=503)
|