connor
b0e509ea92
Found in production, not in tests. SQLite's default busy timeout is ZERO, so when a poll held the write lock while storing ~1,500 embeddings, a concurrent feedback POST failed instantly with "database is locked" rather than waiting. Polls take minutes and run every fifteen, so this was a wide failure window on the single endpoint that must never drop a write -- feedback is the only way the taste model learns anything, and the previous system's entire failure was collecting 32 labels. Connections now set busy_timeout=15s. Regression test included, and its first draft was wrong in an instructive way: the competing thread held the write lock continuously, which no finite timeout can survive. It now writes in bursts with gaps, like a real poll cycle, which is the contention actually being tested. Also cleans up the static-site plumbing left over from the Rust service: the caddy /srv/news bind mount, the built SPA under data/news/web, and the news-update systemd units -- which turned out to be DANGLING symlinks into docs/projects/news/deploy/, the path the old repo used before being renamed to news_triage, and had been failing every ten minutes unnoticed ever since.
131 lines
4.9 KiB
Python
131 lines
4.9 KiB
Python
"""Configuration must actually drive behaviour.
|
|
|
|
The point of these tests is portability: someone who is not the author should be
|
|
able to run this by replacing the TOML files, without editing Python. Each test
|
|
below asserts that a value in config reaches the code that uses it.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import textwrap
|
|
|
|
import numpy as np
|
|
import pytest
|
|
|
|
from news import cluster, store
|
|
from news.config import Config
|
|
from news.embed import Embedder
|
|
|
|
|
|
@pytest.fixture
|
|
def cfgdir(tmp_path, monkeypatch):
|
|
(tmp_path / "config.toml").write_text(textwrap.dedent("""
|
|
[http]
|
|
user_agent = "someone-else/9.9 (+https://elsewhere.example)"
|
|
timeout_secs = 7
|
|
concurrency = 3
|
|
[embedding]
|
|
model = "intfloat/multilingual-e5-small"
|
|
dim = 512
|
|
[clustering]
|
|
threshold = 0.5
|
|
window_hours = 6
|
|
[scoring]
|
|
veto_absolute = 0.99
|
|
"""))
|
|
(tmp_path / "sources.toml").write_text(textwrap.dedent("""
|
|
[defaults]
|
|
poll_interval_secs = 111
|
|
[[source]]
|
|
name = "Someone Else's Feed"
|
|
url = "https://elsewhere.example/rss"
|
|
bucket = "elsewhere"
|
|
"""))
|
|
(tmp_path / "profile.toml").write_text(textwrap.dedent("""
|
|
[[interest]]
|
|
id = "trains"
|
|
weight = 0.4
|
|
probes = ["Rolling stock procurement and timetable changes."]
|
|
[delivery]
|
|
digest_hour = "19:00"
|
|
"""))
|
|
monkeypatch.setenv("NEWS_CONFIG_DIR", str(tmp_path))
|
|
return tmp_path
|
|
|
|
|
|
def test_no_authors_identity_leaks_into_a_fresh_config(cfgdir):
|
|
cfg = Config()
|
|
assert "rcjohnstone" not in cfg.user_agent
|
|
assert cfg.user_agent.startswith("someone-else/9.9")
|
|
|
|
|
|
def test_machinery_values_are_read_from_config(cfgdir):
|
|
cfg = Config()
|
|
assert cfg.http["timeout_secs"] == 7
|
|
assert cfg.http["concurrency"] == 3
|
|
assert cfg.embedding["model"] == "intfloat/multilingual-e5-small"
|
|
assert cfg.embedding["dim"] == 512
|
|
assert cfg.clustering["threshold"] == 0.5
|
|
assert cfg.clustering["window_hours"] == 6
|
|
assert cfg.scoring["veto_absolute"] == 0.99
|
|
|
|
|
|
def test_embedder_takes_its_model_from_config_without_loading_it(cfgdir):
|
|
"""A different reader can swap in a multilingual model from TOML alone."""
|
|
emb = Embedder.from_config(Config())
|
|
assert emb._model_name == "intfloat/multilingual-e5-small"
|
|
assert emb.encode([]).shape == (0, 512) # honours the configured dim
|
|
|
|
|
|
def test_reader_profile_is_entirely_data(cfgdir):
|
|
cfg = Config()
|
|
profile = cfg.profile()
|
|
ids = [b["id"] for b in profile["interest"]]
|
|
assert ids == ["trains"], "profile content must come only from TOML"
|
|
assert cfg.delivery["digest_hour"] == "19:00"
|
|
|
|
|
|
def test_sources_come_from_config_with_defaults_applied(cfgdir):
|
|
srcs = Config().sources()
|
|
assert len(srcs) == 1
|
|
assert srcs[0]["name"] == "Someone Else's Feed"
|
|
assert srcs[0]["poll_interval_secs"] == 111 # from [defaults]
|
|
assert srcs[0]["kind"] == "rss" and srcs[0]["weight"] == 1.0
|
|
|
|
|
|
def test_configured_cluster_threshold_changes_the_outcome(tmp_path):
|
|
"""The calibrated 0.82 is a default, not a law: config must override it."""
|
|
con = store.connect(tmp_path / "t.db")
|
|
store.upsert_source(con, id="s1", name="A", url="http://a", kind="rss",
|
|
bucket="a", weight=1.0, prominence=0, enabled=1,
|
|
poll_interval_secs=900)
|
|
|
|
def add(i, when):
|
|
con.execute(
|
|
"""INSERT INTO items (id,source_id,title,summary,url,canonical_url,
|
|
published_at,first_seen_at,content_hash)
|
|
VALUES (?,?,?,'',?,?,?,?,?)""",
|
|
(i, "s1", i, f"http://x/{i}", f"http://x/{i}", when, when, i))
|
|
return con.execute("SELECT * FROM items WHERE id=?", (i,)).fetchone()
|
|
|
|
rows = [add("i1", "2026-01-01T00:00:00+00:00"), add("i2", "2026-01-01T01:00:00+00:00")]
|
|
a, b = np.array([1, 0], np.float32), np.array([0.6, 0.8], np.float32) # cosine 0.6
|
|
vecs = {"i1": a, "i2": b}
|
|
|
|
strict = cluster.assign(con, rows, vecs, threshold=0.9)
|
|
assert strict[0].cluster_id != strict[1].cluster_id
|
|
|
|
con2 = store.connect(tmp_path / "t2.db")
|
|
store.upsert_source(con2, id="s1", name="A", url="http://a", kind="rss",
|
|
bucket="a", weight=1.0, prominence=0, enabled=1,
|
|
poll_interval_secs=900)
|
|
for i, when in (("i1", "2026-01-01T00:00:00+00:00"), ("i2", "2026-01-01T01:00:00+00:00")):
|
|
con2.execute(
|
|
"""INSERT INTO items (id,source_id,title,summary,url,canonical_url,
|
|
published_at,first_seen_at,content_hash)
|
|
VALUES (?,?,?,'',?,?,?,?,?)""",
|
|
(i, "s1", i, f"http://x/{i}", f"http://x/{i}", when, when, i))
|
|
rows2 = con2.execute("SELECT * FROM items ORDER BY published_at").fetchall()
|
|
loose = cluster.assign(con2, list(rows2), vecs, threshold=0.5)
|
|
assert loose[0].cluster_id == loose[1].cluster_id
|