connor
c03df8c1b5
Reddit returns 403 or 429 depending on the moment -- measured across four subreddits in one pass: two 403, two 429, while three of the four had ingested 44-55 items earlier, so the source works intermittently. Only the 429 was handled softly, so the 403 raised and logged an error every fifteen minutes for a source that was fine. Neither status is actionable per cycle. Both now back off quietly and record the status on the source row, where it is visible via `news sources`.
90 lines
2.9 KiB
Python
90 lines
2.9 KiB
Python
"""RSS/Atom parsing, including the RDF format the old system could not read."""
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
from news.sources import rss
|
|
|
|
RDF = """<?xml version="1.0"?>
|
|
<rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
|
|
xmlns="http://purl.org/rss/1.0/">
|
|
<channel rdf:about="http://x"><title>T</title><link>http://x</link>
|
|
<description>d</description></channel>
|
|
<item rdf:about="http://x/1">
|
|
<title>German economy frets about election result</title>
|
|
<link>http://x/1?utm_source=rss</link>
|
|
<description>Some <b>markup</b> here</description>
|
|
</item>
|
|
</rdf:RDF>"""
|
|
|
|
ATOM = """<?xml version="1.0"?>
|
|
<feed xmlns="http://www.w3.org/2005/Atom">
|
|
<title>F</title>
|
|
<entry>
|
|
<title>Release v1.2.3</title>
|
|
<link rel="alternate" href="http://x/rel"/>
|
|
<link rel="self" href="http://x/self"/>
|
|
<updated>2026-09-08T12:00:00Z</updated>
|
|
</entry>
|
|
</feed>"""
|
|
|
|
|
|
def _source():
|
|
return {"id": "s1", "url": "http://feed", "kind": "rss"}
|
|
|
|
|
|
async def _fetch(body: str, status: int = 200, headers: dict | None = None):
|
|
def handler(request):
|
|
return httpx.Response(status, content=body.encode(), headers=headers or {})
|
|
|
|
transport = httpx.MockTransport(handler)
|
|
async with httpx.AsyncClient(transport=transport) as client:
|
|
return await rss.fetch(client, _source())
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_parses_rss_1_rdf():
|
|
"""DW ships RDF; the old hand-rolled parser yielded zero items silently."""
|
|
items, _ = await _fetch(RDF)
|
|
assert len(items) == 1
|
|
assert items[0]["title"].startswith("German economy")
|
|
# tracking parameter stripped, markup removed from the summary
|
|
assert items[0]["canonical_url"] == "http://x/1"
|
|
assert "<b>" not in items[0]["summary"] and "markup" in items[0]["summary"]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_atom_prefers_alternate_link_over_self():
|
|
items, _ = await _fetch(ATOM)
|
|
assert len(items) == 1
|
|
assert items[0]["canonical_url"] == "http://x/rel"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_304_returns_no_items_and_is_not_an_error():
|
|
items, state = await _fetch("", status=304)
|
|
assert items == [] and state["status"] == "304"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_429_is_reported_as_rate_limited_not_raised():
|
|
"""Reddit 429s readily; that must back off, not raise every cycle."""
|
|
items, state = await _fetch("", status=429, headers={"retry-after": "120"})
|
|
assert items == []
|
|
assert state["rate_limited"] is True
|
|
assert state["retry_after"] == "120"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_403_is_also_treated_as_blocked_not_an_error():
|
|
"""Reddit returns 403 or 429 depending on the moment; neither is actionable
|
|
per cycle, and raising on one logged an error every fifteen minutes."""
|
|
items, state = await _fetch("", status=403)
|
|
assert items == [] and state["rate_limited"] is True
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_server_error_still_raises():
|
|
with pytest.raises(httpx.HTTPStatusError):
|
|
await _fetch("", status=503)
|