v0.2 wave 1: merge-rule-v2 clustering with LLM-candidate gray zone, 0.2.0 version plumbing, deploy verification docs
Check / guardrails (push) Successful in 1m43s
Check / bundle (push) Successful in 1m15s
Check / check (push) Successful in 2m42s
Image / image (push) Failing after 20s

This commit is contained in:
2026-09-03 16:46:12 -04:00
parent c7da2704da
commit 43b94489f5
26 changed files with 1034 additions and 247 deletions
+18 -2
View File
@@ -11,6 +11,8 @@ on:
push:
branches:
- main
tags:
- 'v*'
jobs:
image:
@@ -18,6 +20,10 @@ jobs:
steps:
- uses: actions/checkout@v4
- name: Compute short sha
id: short
run: echo "sha=$(git rev-parse --short HEAD)" >> "$GITHUB_OUTPUT"
- uses: docker/setup-buildx-action@v3
- uses: docker/login-action@v3
@@ -28,14 +34,24 @@ jobs:
# Tagged with the commit as well as `latest`, so that "what is actually
# running" has an answer, and so a rollback is a tag rather than a revert
# and a rebuild.
# and a rebuild. On a `v*` tag build the tag name is added too, so an
# exact version can be pinned; on branch pushes that expression is empty
# and the line is dropped. The version build-arg/label follow the same
# rule and feed /api/version through the Dockerfile's ARGs.
- uses: docker/build-push-action@v5
with:
context: .
file: ./Dockerfile
push: true
build-args: |
NEWS_BUILD_VERSION=${{ github.ref_type == 'tag' && github.ref_name || 'dev' }}
NEWS_BUILD_SHA=${{ github.sha }}
labels: |
org.opencontainers.image.version=${{ github.ref_type == 'tag' && github.ref_name || 'dev' }}
org.opencontainers.image.revision=${{ github.sha }}
tags: |
${{ vars.REGISTRY }}/connor/news:latest
${{ vars.REGISTRY }}/connor/news:${{ github.sha }}
${{ vars.REGISTRY }}/connor/news:${{ steps.short.outputs.sha }}
${{ vars.REGISTRY }}/connor/news:${{ github.ref_type == 'tag' && github.ref_name || '' }}
cache-from: type=gha
cache-to: type=gha,mode=max
Generated
+6 -6
View File
@@ -1630,7 +1630,7 @@ dependencies = [
[[package]]
name = "news-cli"
version = "0.1.0"
version = "0.2.0"
dependencies = [
"chrono",
"clap",
@@ -1646,7 +1646,7 @@ dependencies = [
[[package]]
name = "news-core"
version = "0.1.0"
version = "0.2.0"
dependencies = [
"chrono",
"pretty_assertions",
@@ -1657,7 +1657,7 @@ dependencies = [
[[package]]
name = "news-ingest"
version = "0.1.0"
version = "0.2.0"
dependencies = [
"chrono",
"news-core",
@@ -1672,7 +1672,7 @@ dependencies = [
[[package]]
name = "news-server"
version = "0.1.0"
version = "0.2.0"
dependencies = [
"axum",
"chrono",
@@ -1702,7 +1702,7 @@ dependencies = [
[[package]]
name = "news-store"
version = "0.1.0"
version = "0.2.0"
dependencies = [
"chrono",
"news-core",
@@ -1715,7 +1715,7 @@ dependencies = [
[[package]]
name = "news-web"
version = "0.1.0"
version = "0.2.0"
dependencies = [
"console_error_panic_hook",
"gloo-net",
+1 -1
View File
@@ -10,7 +10,7 @@ members = [
]
[workspace.package]
version = "0.1.0"
version = "0.2.0"
edition = "2024"
license = "MIT"
+9
View File
@@ -62,9 +62,18 @@ RUN --mount=type=cache,id=web-registry,target=/usr/local/cargo/registry \
# ----------------------------------------------------------------- backend --
FROM rust:1.89-slim-bookworm AS server
# Build metadata for /api/version, baked in by build.rs (see that file for
# the fallbacks). release.yml passes the real values; a bare `docker build`
# gets `dev`/`unknown`, matching a plain `cargo build` on a dev machine.
ARG NEWS_BUILD_VERSION=dev
ARG NEWS_BUILD_SHA=unknown
WORKDIR /app
COPY . .
ENV NEWS_BUILD_VERSION=${NEWS_BUILD_VERSION} \
NEWS_BUILD_SHA=${NEWS_BUILD_SHA}
# The binary is copied out of the cache mount because a cache mount is not part
# of the layer: whatever is written there is gone by the time the next stage
# looks. This is also why there is no dummy-source dance -- the cache does what
+1
View File
@@ -7,6 +7,7 @@ mod replay;
#[derive(Parser)]
#[command(name = "news-cli")]
#[command(about = "Offline replay harness for news-triage decisions")]
#[command(version)]
struct Cli {
#[command(subcommand)]
command: Command,
+15 -4
View File
@@ -11,13 +11,15 @@ use news_server::{
DecisionRow, ImportanceScorer, NotificationGate, NotifyConfig, RelevanceScorer, TokenBucket,
format_decision_header, format_decision_row,
};
use news_store::{ClusterRepo, ItemRepo, MetricKind, PercentileTracker, SourceRepo, init_db};
use news_store::{
AssignOutcome, ClusterParams, ClusterRepo, ItemRepo, MetricKind, PercentileTracker, SourceRepo,
init_db,
};
use serde::Deserialize;
use uuid::Uuid;
const DEFAULT_NOW: &str = "2026-01-15T12:00:00Z";
const DEFAULT_WEIGHT: f64 = 0.8;
const CLUSTER_THRESHOLD: f64 = 0.5;
#[derive(Debug, Default, Deserialize)]
struct Manifest {
@@ -69,9 +71,18 @@ pub async fn run(
let items = parse_fixture(&body, &source)?;
for item in items {
items_repo.insert_if_new(&item).await?;
let cluster = clusters_repo
.assign_or_create(&item, CLUSTER_THRESHOLD)
let outcome = clusters_repo
.assign_or_create(&item, &ClusterParams::default())
.await?;
// Gray-zone pairs stay split in replay (no LLM layer).
let cluster_id = match outcome {
AssignOutcome::Assigned(id) | AssignOutcome::Created(id) => id,
AssignOutcome::LlmCandidates(_) => continue,
};
let cluster = clusters_repo
.get(cluster_id)
.await?
.ok_or("cluster missing after assignment")?;
item_by_id.insert(item.id, item);
seen_clusters.insert(cluster.id, cluster);
}
@@ -28,5 +28,13 @@
<description>A winter storm warning is in effect for higher elevations.</description>
<category>World</category>
</item>
<item>
<title>Summit postponed amid election dispute</title>
<link>https://example.test/source-c/summit-dispute</link>
<guid isPermaLink="true">https://example.test/source-c/summit-dispute</guid>
<pubDate>Wed, 14 Jan 2026 11:25:00 GMT</pubDate>
<description>Organizers delayed the gathering while officials review the dispute.</description>
<category>World</category>
</item>
</channel>
</rss>
+236 -62
View File
@@ -1,47 +1,151 @@
//! Title-similarity clustering for grouping raw feed items that describe
//! the same breaking story.
//! Lexical clustering for grouping raw feed items that describe the same
//! breaking story: weighted token-set containment between title/summary
//! pairs, gated by a minimum count of shared common (non-proper-noun)
//! tokens. Pure functions, no I/O.
use std::collections::HashSet;
use crate::RawItem;
/// Summary tokens folded into the scoring pool. The cap keeps short-title
/// pairs decisive when one outlet attaches a long summary.
const SUMMARY_TOKEN_LIMIT: usize = 40;
/// Tiny built-in stopword list. These words are dropped before token-set
/// comparison because they carry almost no story-discriminating signal and
/// dominate syndication-style title variants ("the coast" vs "coast").
const STOPWORDS: &[&str] = &["a", "the", "of", "to", "in", "and", "for", "on", "at"];
/// Tiny built-in stopword list. These words are dropped before comparison
/// because they carry almost no story-discriminating signal and dominate
/// syndication-style title variants ("the coast" vs "coast"). `new`,
/// `first`, and `top` are deliberately NOT stopwords — they are content.
const STOPWORDS: &[&str] = &[
"a", "the", "of", "to", "in", "and", "for", "on", "at", //
"is", "are", "was", "were", "be", "been", "as", "by", "from", //
"has", "have", "had", "that", "this", "with", "after", "over", //
"into", "amid", "during", "says", "said", "say", "tells", //
"will", "would", "could", "can",
];
/// Returns the Jaccard similarity of two titles, in the range `0.0..=1.0`.
///
/// Tokenisation rules:
/// * split on non-alphanumeric characters;
/// * lower-case each token;
/// * drop empty tokens;
/// * drop the small built-in stopword set documented above.
///
/// If both token sets are empty the similarity is defined as `0.0` so the
/// function always returns a valid, bounded value.
pub fn similarity(a: &RawItem, b: &RawItem) -> f64 {
let a_tokens: HashSet<String> = tokenize(&a.title).into_iter().collect();
let b_tokens: HashSet<String> = tokenize(&b.title).into_iter().collect();
/// Text of one item taking part in a comparison.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ItemText {
pub title: String,
pub summary: String,
}
if a_tokens.is_empty() && b_tokens.is_empty() {
/// Tunables for the similarity engine.
#[derive(Debug, Clone, PartialEq)]
pub struct SimParams {
/// Score at/above which a pair can merge (still subject to the
/// corroboration gate).
pub threshold: f64,
/// Width of the below-threshold band that still routes to LLM
/// adjudication instead of an outright split.
pub gray_zone_width: f64,
/// Minimum count of shared non-proper-noun content tokens for a merge.
pub corroboration_min: usize,
/// Weight of a proper-noun token relative to a common token.
pub proper_noun_weight: f64,
}
impl Default for SimParams {
fn default() -> Self {
Self {
threshold: 0.45,
gray_zone_width: 0.15,
corroboration_min: 2,
proper_noun_weight: 2.0,
}
}
}
/// Verdict on whether two items describe the same story.
#[derive(Debug, Clone, PartialEq)]
pub struct MatchVerdict {
/// Weighted containment score, `0.0..=1.0`.
pub score: f64,
/// Merge into one cluster without further adjudication.
pub merge: bool,
/// Not a merge, but close enough that LLM consolidation (WS2) should
/// adjudicate; when no LLM runs, the pair stays split.
pub llm_candidate: bool,
}
/// One side's token pool: title tokens plus the first
/// [`SUMMARY_TOKEN_LIMIT`] summary tokens, with the proper-noun subset.
#[derive(Debug)]
struct Side {
pool: HashSet<String>,
proper: HashSet<String>,
}
impl Side {
fn new(title: &str, summary: &str) -> Self {
let mut pool: HashSet<String> = tokenize(title).into_iter().collect();
pool.extend(tokenize(summary).into_iter().take(SUMMARY_TOKEN_LIMIT));
Self {
pool,
proper: proper_tokens_from_title(title),
}
}
}
/// Compares two items under `p` and returns the match verdict.
pub fn compare(a: &ItemText, b: &ItemText, p: &SimParams) -> MatchVerdict {
let a_side = Side::new(&a.title, &a.summary);
let b_side = Side::new(&b.title, &b.summary);
let score = weighted_containment(&a_side, &b_side, p.proper_noun_weight);
let shared_non_proper = a_side
.pool
.intersection(&b_side.pool)
.filter(|token| !a_side.proper.contains(*token) && !b_side.proper.contains(*token))
.count();
// Merge rule v2 (user-directed, two correction rounds): corroboration is
// the gate, proper nouns are optional weighted evidence. "Donald Trump
// does X" vs "Donald Trump does Y" shares only {does} -> never merges;
// "scientists clone first human" variants share {scientists, clone,
// first, human} -> merge with zero proper nouns.
let merge = score >= p.threshold && shared_non_proper >= p.corroboration_min;
let llm_candidate = !merge
&& (score >= p.threshold - p.gray_zone_width
|| (score >= p.threshold && shared_non_proper >= 1));
MatchVerdict {
score,
merge,
llm_candidate,
}
}
/// Weighted containment of the smaller pool in the larger pool:
/// `sum(w(shared)) / sum(w(smaller pool))`, capped at `1.0`. Proper-noun
/// weights come from the smaller side, which keeps the ratio bounded.
fn weighted_containment(a: &Side, b: &Side, proper_weight: f64) -> f64 {
let (small, large) = if a.pool.len() <= b.pool.len() {
(a, b)
} else {
(b, a)
};
let weight = |token: &String| {
if small.proper.contains(token) {
proper_weight
} else {
1.0
}
};
let denominator: f64 = small.pool.iter().map(&weight).sum();
if denominator <= 0.0 {
return 0.0;
}
let intersection = a_tokens.intersection(&b_tokens).count();
let union = a_tokens.union(&b_tokens).count();
intersection as f64 / union as f64
let shared: f64 = small
.pool
.iter()
.filter(|token| large.pool.contains(*token))
.map(&weight)
.sum();
(shared / denominator).min(1.0)
}
/// Returns `true` when `sim` meets or exceeds the caller-supplied threshold.
pub fn should_cluster(sim: f64, threshold: f64) -> bool {
sim >= threshold
}
/// Tokenises `text` using the same rules as title-similarity clustering:
/// split on non-alphanumeric characters, lower-case each token, drop empty
/// tokens, and drop the small built-in stopword set.
/// Tokenises `text`: split on non-alphanumeric characters, lower-case each
/// token, drop empty tokens and the built-in stopword set. Numerals are
/// kept as tokens (`91`, `24`).
pub fn tokenize(text: &str) -> Vec<String> {
text.split(|c: char| !c.is_alphanumeric())
.map(str::to_lowercase)
@@ -49,6 +153,44 @@ pub fn tokenize(text: &str) -> Vec<String> {
.collect()
}
/// Detects proper nouns from the original title casing: mid-title
/// capitalized tokens that are not stopwords. The first word is ambiguous
/// between headline-case and sentence-case, so it only counts as proper
/// when the title opens with a capitalized multi-token sequence ("Donald
/// Trump ..."); a lone capitalized first word is treated as grammar, not a
/// name ("Scientists clone ...").
fn proper_tokens_from_title(title: &str) -> HashSet<String> {
let raw: Vec<&str> = title
.split(|c: char| !c.is_alphanumeric())
.filter(|token| !token.is_empty())
.collect();
let mut proper = HashSet::new();
for (index, token) in raw.iter().enumerate() {
let Some(first_char) = token.chars().next() else {
continue;
};
if !first_char.is_uppercase() {
continue;
}
let lower = token.to_lowercase();
if is_stopword(&lower) {
continue;
}
if index == 0 {
let opens_name = raw
.get(1)
.and_then(|next| next.chars().next())
.is_some_and(|c| c.is_uppercase());
if !opens_name {
continue;
}
}
proper.insert(lower);
}
proper
}
fn is_stopword(token: &str) -> bool {
STOPWORDS.contains(&token)
}
@@ -56,54 +198,86 @@ fn is_stopword(token: &str) -> bool {
#[cfg(test)]
mod tests {
use super::*;
use chrono::Utc;
use uuid::Uuid;
fn item(title: &str) -> RawItem {
RawItem {
id: Uuid::new_v4(),
source_id: Uuid::new_v4(),
fn text(title: &str) -> ItemText {
ItemText {
title: title.to_string(),
summary: "summary".to_string(),
link: "https://example.test/story".to_string(),
published_at: Utc::now(),
kind: crate::ItemKind::News,
summary: String::new(),
}
}
#[test]
fn identical_titles_have_similarity_one() {
let a = item("Storm system moves toward the coast");
let b = item("Storm system moves toward the coast");
fn steinem_paraphrases_merge() -> Result<(), Box<dyn std::error::Error>> {
// Shared common tokens: dies, 91. "Steinem" is proper in the first
// title but optional weighted evidence, never required.
let a = text("Gloria Steinem dies aged 91");
let b = text("Steinem, feminist icon, dies at 91");
assert_eq!(similarity(&a, &b), 1.0);
let verdict = compare(&a, &b, &SimParams::default());
assert!(verdict.merge, "expected merge, got {verdict:?}");
Ok(())
}
#[test]
fn disjoint_titles_have_similarity_zero() {
let a = item("red blue green");
let b = item("one two three");
fn trump_different_stories_split() -> Result<(), Box<dyn std::error::Error>> {
// Shared tokens are only the entity {donald, trump}: zero common
// corroboration, so the pair must split even though the score is
// at the threshold.
let a = text("Donald Trump announces new tariffs on imports");
let b = text("Donald Trump endorses House candidate in Ohio");
assert_eq!(similarity(&a, &b), 0.0);
let verdict = compare(&a, &b, &SimParams::default());
assert!(!verdict.merge, "expected split, got {verdict:?}");
Ok(())
}
#[test]
fn stopword_difference_keeps_syndication_titles_high() {
let a = item("Storm system moves toward the coast");
let b = item("Storm system moves toward coast");
fn no_proper_noun_merge() -> Result<(), Box<dyn std::error::Error>> {
// Both titles are sentence-case, so no token is a proper noun; the
// pair merges on common-word strength alone.
let a = text("Scientists clone first human embryo");
let b = text("First human clone created by scientists");
let sim = similarity(&a, &b);
let verdict = compare(&a, &b, &SimParams::default());
assert!(verdict.merge, "expected merge, got {verdict:?}");
Ok(())
}
#[test]
fn entity_only_high_score_blocked() -> Result<(), Box<dyn std::error::Error>> {
// High score but shared common tokens = {does} only (1 < 2):
// entity-only overlap is never sufficient evidence, so the pair is
// blocked from merging and routed to LLM adjudication.
let a = text("Donald Trump does X");
let b = text("Donald Trump does Y");
let params = SimParams::default();
let verdict = compare(&a, &b, &params);
assert!(
sim > 0.8,
"expected similarity > 0.8 for syndication titles, got {sim}"
verdict.score >= params.threshold,
"expected score at/above threshold, got {verdict:?}"
);
assert!(!verdict.merge, "expected split, got {verdict:?}");
assert!(
verdict.llm_candidate,
"expected LLM candidate, got {verdict:?}"
);
Ok(())
}
#[test]
fn should_cluster_respects_threshold() {
assert!(should_cluster(0.75, 0.5));
assert!(should_cluster(0.5, 0.5));
assert!(!should_cluster(0.49, 0.5));
fn numerals_corroborate() -> Result<(), Box<dyn std::error::Error>> {
// Numerals are content tokens: {24} counts toward corroboration.
let a = text("At least 24 killed in crash");
let b = text("24 killed in highway crash");
let verdict = compare(&a, &b, &SimParams::default());
assert!(verdict.merge, "expected merge, got {verdict:?}");
Ok(())
}
}
+57
View File
@@ -0,0 +1,57 @@
//! Build script: bakes build metadata into the binary as compile-time
//! environment variables, which the `/api/version` handler reads via `env!`.
//!
//! Values come from build-time env (`NEWS_BUILD_VERSION` / `NEWS_BUILD_SHA` /
//! `NEWS_BUILD_TIME`) when present — the Dockerfile server stage sets the
//! first two from CI — and fall back to `dev` / `unknown` / the current UTC
//! timestamp, so a plain `cargo build` on a dev machine still yields a
//! truthful (if unglamorous) `/api/version`.
use std::time::{SystemTime, UNIX_EPOCH};
fn main() {
println!("cargo:rerun-if-env-changed=NEWS_BUILD_VERSION");
println!("cargo:rerun-if-env-changed=NEWS_BUILD_SHA");
println!("cargo:rerun-if-env-changed=NEWS_BUILD_TIME");
let version = std::env::var("NEWS_BUILD_VERSION").unwrap_or_else(|_| "dev".into());
let sha = std::env::var("NEWS_BUILD_SHA").unwrap_or_else(|_| "unknown".into());
let time = std::env::var("NEWS_BUILD_TIME").unwrap_or_else(|_| rfc3339_now());
println!("cargo:rustc-env=NEWS_BUILD_VERSION={version}");
println!("cargo:rustc-env=NEWS_BUILD_SHA={sha}");
println!("cargo:rustc-env=NEWS_BUILD_TIME={time}");
}
/// Current UTC time as an RFC3339 timestamp, std-only (chrono stays out of
/// the build graph). Clock before 1970 degrades to the epoch rather than
/// failing the build.
fn rfc3339_now() -> String {
let secs = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_secs())
.unwrap_or(0);
let (year, month, day) = civil_from_days((secs / 86_400) as i64);
let rem = secs % 86_400;
format!(
"{year:04}-{month:02}-{day:02}T{:02}:{:02}:{:02}Z",
rem / 3_600,
(rem % 3_600) / 60,
rem % 60
)
}
/// Inverse of days-from-civil (Hinnant 2017): days since the Unix epoch to a
/// proleptic-Gregorian (year, month, day). Inputs are non-negative here.
fn civil_from_days(days: i64) -> (i64, u32, u32) {
let z = days + 719_468;
let era = if z >= 0 { z } else { z - 146_096 } / 146_097;
let doe = (z - era * 146_097) as u64;
let yoe = (doe - doe / 1_460 + doe / 36_524 - doe / 146_096) / 365;
let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
let mp = (5 * doy + 2) / 153;
let day = (doy - (153 * mp + 2) / 5 + 1) as u32;
let month = if mp < 10 { mp + 3 } else { mp - 9 } as u32;
let year = yoe as i64 + era * 400 + i64::from(month <= 2);
(year, month, day)
}
+2 -1
View File
@@ -33,7 +33,7 @@ mod tests {
use tower::ServiceExt;
use crate::ServerError;
use crate::config::{NotifySection, SourceConfig, TopicsSection};
use crate::config::{ClusteringSection, NotifySection, SourceConfig, TopicsSection};
fn test_config() -> NewsConfig {
NewsConfig {
@@ -61,6 +61,7 @@ mod tests {
interests: vec!["science".into()],
blocklist: vec!["celebrity".into()],
},
clustering: ClusteringSection::default(),
}
}
+12 -4
View File
@@ -122,7 +122,7 @@ mod tests {
use axum::http::Request;
use chrono::Utc;
use news_core::{ItemKind, RawItem, Source, SourceKind};
use news_store::{ClusterRepo, ItemRepo, SourceRepo, init_db};
use news_store::{AssignOutcome, ClusterParams, ClusterRepo, ItemRepo, SourceRepo, init_db};
use serde_json::json;
use tower::ServiceExt;
@@ -158,10 +158,18 @@ mod tests {
SourceRepo::new(pool.clone()).upsert(&source).await?;
let item = test_item(source.id);
ItemRepo::new(pool.clone()).insert_if_new(&item).await?;
let cluster = ClusterRepo::new(pool.clone())
.assign_or_create(&item, 0.5)
let outcome = ClusterRepo::new(pool.clone())
.assign_or_create(&item, &ClusterParams::default())
.await?;
Ok((pool, cluster.id))
let cluster_id = match outcome {
AssignOutcome::Assigned(id) | AssignOutcome::Created(id) => id,
AssignOutcome::LlmCandidates(_) => {
return Err(ServerError::DateTime(
"seed item unexpectedly landed in the gray zone".into(),
));
}
};
Ok((pool, cluster_id))
}
fn build_request(json_body: &Value) -> Result<Request<Body>, ServerError> {
+65
View File
@@ -1,5 +1,70 @@
//! HTTP API routes.
//!
//! The stateless `/api/version` endpoint lives here; the endpoints that need
//! state get their own module (`config`, `feedback`, `stories`).
pub mod config;
pub mod feedback;
pub mod stories;
use axum::{Json, Router, response::IntoResponse, routing::get};
/// Router for the endpoints that carry no state. Merged into the app beside
/// the per-module routers in `main.rs`.
pub fn router() -> Router {
Router::new().route("/api/version", get(get_version))
}
/// GET /api/version: build metadata for the web footer (`v{version} · {sha}`).
/// Values are baked in at compile time by `build.rs` — env vars set at
/// runtime have no effect.
async fn get_version() -> impl IntoResponse {
Json(serde_json::json!({
"version": env!("NEWS_BUILD_VERSION"),
"git_sha": env!("NEWS_BUILD_SHA"),
"build_time": env!("NEWS_BUILD_TIME"),
}))
}
#[cfg(test)]
mod tests {
use super::*;
use axum::body::Body;
use axum::http::{Request, StatusCode};
use tower::ServiceExt;
use crate::ServerError;
/// Compile-time proof that `build.rs` populated the three `env!`s and
/// that the route serves them: if any `env!` were missing, this crate
/// would not compile; if the route were miswired, the request would fail.
#[tokio::test]
async fn version_reports_all_build_fields() -> Result<(), ServerError> {
let request = Request::builder()
.method("GET")
.uri("/api/version")
.body(Body::empty())
.map_err(|e| ServerError::DateTime(e.to_string()))?;
let response = match router().oneshot(request).await {
Ok(res) => res,
Err(infallible) => match infallible {},
};
assert_eq!(response.status(), StatusCode::OK);
let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)
.await
.map_err(|e| ServerError::DateTime(e.to_string()))?;
let body: serde_json::Value = serde_json::from_slice(&bytes)?;
for field in ["version", "git_sha", "build_time"] {
let value = body
.get(field)
.and_then(|v| v.as_str())
.ok_or_else(|| ServerError::DateTime(format!("missing field: {field}")))?;
assert!(!value.is_empty(), "{field} must be non-empty");
}
Ok(())
}
}
+19 -5
View File
@@ -4,7 +4,7 @@ use axum::body::Body;
use axum::http::{Request, StatusCode};
use chrono::Utc;
use news_core::{ItemKind, RawItem, Source, SourceKind, StoryCluster};
use news_store::{ClusterRepo, ItemRepo, SourceRepo, init_db};
use news_store::{AssignOutcome, ClusterParams, ClusterRepo, ItemRepo, SourceRepo, init_db};
use serde_json::Value;
use tokio::sync::{Mutex, RwLock};
use tower::ServiceExt;
@@ -12,7 +12,7 @@ use uuid::Uuid;
use crate::ServerError;
use crate::api::stories::router;
use crate::config::{NewsConfig, NotifySection, SourceConfig, TopicsSection};
use crate::config::{ClusteringSection, NewsConfig, NotifySection, SourceConfig, TopicsSection};
use crate::gate::{NotificationGate, NotifyConfig, TokenBucket};
fn test_config() -> NewsConfig {
@@ -41,6 +41,7 @@ fn test_config() -> NewsConfig {
interests: vec!["science".into()],
blocklist: vec!["celebrity".into()],
},
clustering: ClusteringSection::default(),
}
}
@@ -86,9 +87,22 @@ async fn seeded_pool() -> Result<(sqlx::SqlitePool, StoryCluster), ServerError>
SourceRepo::new(pool.clone()).upsert(&source).await?;
let item = test_item(source.id);
ItemRepo::new(pool.clone()).insert_if_new(&item).await?;
let cluster = ClusterRepo::new(pool.clone())
.assign_or_create(&item, 0.95)
let clusters = ClusterRepo::new(pool.clone());
let outcome = clusters
.assign_or_create(&item, &ClusterParams::default())
.await?;
let cluster_id = match outcome {
AssignOutcome::Assigned(id) | AssignOutcome::Created(id) => id,
AssignOutcome::LlmCandidates(_) => {
return Err(ServerError::DateTime(
"seed item unexpectedly landed in the gray zone".into(),
));
}
};
let cluster = clusters
.get(cluster_id)
.await?
.ok_or_else(|| ServerError::DateTime("cluster missing after assignment".into()))?;
Ok((pool, cluster))
}
@@ -264,7 +278,7 @@ async fn blocklisted_story_serializes_relevance_as_null() -> Result<(), ServerEr
let item = blocklisted_item(source.id);
ItemRepo::new(pool.clone()).insert_if_new(&item).await?;
ClusterRepo::new(pool.clone())
.assign_or_create(&item, 0.95)
.assign_or_create(&item, &ClusterParams::default())
.await?;
let config = Arc::new(RwLock::new(test_config()));
+101
View File
@@ -20,6 +20,10 @@ pub struct NewsConfig {
pub notify: NotifySection,
/// Topic filters.
pub topics: TopicsSection,
/// Lexical clustering tunables; absent sections fall back to defaults
/// so pre-v0.2 config files keep parsing unchanged.
#[serde(default)]
pub clustering: ClusteringSection,
}
/// A single feed source.
@@ -93,6 +97,75 @@ fn default_timezone() -> String {
"America/Louisville".to_string()
}
/// Lexical-clustering tunables used by the scheduler's assign step.
#[derive(Debug, Clone, serde::Deserialize, serde::Serialize, PartialEq)]
pub struct ClusteringSection {
/// Score at/above which an item can merge into a cluster (still subject
/// to the shared-token corroboration gate).
#[serde(default = "default_similarity_threshold")]
pub similarity_threshold: f64,
/// How far back a cluster stays warm for new items, in hours.
#[serde(default = "default_window_hours")]
pub window_hours: i64,
/// Width of the below-threshold band that routes to LLM adjudication
/// instead of an outright split.
#[serde(default = "default_gray_zone_width")]
pub gray_zone_width: f64,
/// Minimum count of shared non-proper-noun content tokens for a merge.
#[serde(default = "default_corroboration_min")]
pub corroboration_min: usize,
/// Weight of a mid-title capitalized token relative to a common token.
#[serde(default = "default_proper_noun_weight")]
pub proper_noun_weight: f64,
}
impl Default for ClusteringSection {
fn default() -> Self {
Self {
similarity_threshold: default_similarity_threshold(),
window_hours: default_window_hours(),
gray_zone_width: default_gray_zone_width(),
corroboration_min: default_corroboration_min(),
proper_noun_weight: default_proper_noun_weight(),
}
}
}
fn default_similarity_threshold() -> f64 {
0.45
}
fn default_window_hours() -> i64 {
48
}
fn default_gray_zone_width() -> f64 {
0.15
}
fn default_corroboration_min() -> usize {
2
}
fn default_proper_noun_weight() -> f64 {
2.0
}
impl ClusteringSection {
/// Builds the store-layer clustering parameters from this section.
pub fn cluster_params(&self) -> news_store::ClusterParams {
news_store::ClusterParams {
sim: news_core::clustering::SimParams {
threshold: self.similarity_threshold,
gray_zone_width: self.gray_zone_width,
corroboration_min: self.corroboration_min,
proper_noun_weight: self.proper_noun_weight,
},
window_hours: self.window_hours,
}
}
}
/// Operator-controlled topic filters.
#[derive(Debug, Clone, serde::Deserialize, serde::Serialize, PartialEq)]
pub struct TopicsSection {
@@ -251,6 +324,34 @@ blocklist = ["royal", "celebrity"]
Ok(())
}
#[tokio::test]
async fn clustering_fields_default_and_parse() -> Result<(), ServerError> {
let dir = tempfile::tempdir()?;
let path = dir.path().join("config.toml");
std::fs::write(&path, valid_config_toml())?;
let config = load(&path)?;
assert_eq!(config.clustering.similarity_threshold, 0.45);
assert_eq!(config.clustering.window_hours, 48);
assert_eq!(config.clustering.gray_zone_width, 0.15);
assert_eq!(config.clustering.corroboration_min, 2);
assert_eq!(config.clustering.proper_noun_weight, 2.0);
let with_explicit = format!(
"{}\n[clustering]\nsimilarity_threshold = 0.5\nwindow_hours = 24\n\
gray_zone_width = 0.1\ncorroboration_min = 3\nproper_noun_weight = 3.0",
valid_config_toml()
);
std::fs::write(&path, with_explicit)?;
let config = load(&path)?;
assert_eq!(config.clustering.similarity_threshold, 0.5);
assert_eq!(config.clustering.window_hours, 24);
assert_eq!(config.clustering.gray_zone_width, 0.1);
assert_eq!(config.clustering.corroboration_min, 3);
assert_eq!(config.clustering.proper_noun_weight, 3.0);
Ok(())
}
#[traced_test]
#[tokio::test]
async fn bad_reload_keeps_last_good_config() -> Result<(), ServerError> {
+2 -1
View File
@@ -711,7 +711,7 @@ mod tests {
#[test]
fn from_config_maps_notify_section_and_rejects_bad_inputs() -> Result<(), ServerError> {
use crate::config::{NewsConfig, NotifySection, TopicsSection};
use crate::config::{ClusteringSection, NewsConfig, NotifySection, TopicsSection};
let config = NewsConfig {
sources: Vec::new(),
@@ -732,6 +732,7 @@ mod tests {
interests: Vec::new(),
blocklist: Vec::new(),
},
clustering: ClusteringSection::default(),
};
let mapped = NotifyConfig::from_config(&config)?;
+3
View File
@@ -89,6 +89,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
Arc::clone(&config),
))
.merge(news_server::api::config::router(Arc::clone(&config)))
.merge(news_server::api::router())
.merge(news_server::health::router(rt.pool, rt.metrics));
let port: u16 = std::env::var("NEWS_PORT")
@@ -109,6 +110,7 @@ async fn run_once(
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
use news_server::scheduler::poll_source_once;
let clustering = config.clustering.cluster_params();
let mut title_width = 40_usize;
for source in &rt.sources {
if let Err(err) = poll_source_once(
@@ -121,6 +123,7 @@ async fn run_once(
&rt.gate,
ntfy,
&rt.metrics,
&clustering,
dry_run,
&mut title_width,
)
+14 -6
View File
@@ -1,7 +1,9 @@
use chrono::{Duration, TimeZone, Utc};
use chrono_tz::America::Louisville;
use news_core::{ItemKind, RawItem, Source, SourceKind};
use news_store::{ClusterRepo, ItemRepo, MetricKind, PercentileTracker, SourceRepo, init_db};
use news_store::{
ClusterParams, ClusterRepo, ItemRepo, MetricKind, PercentileTracker, SourceRepo, init_db,
};
use uuid::Uuid;
use wiremock::matchers::{header, method, path};
use wiremock::{Mock, MockServer, Request, ResponseTemplate};
@@ -45,6 +47,7 @@ fn test_config(notify_opinions: bool) -> NewsConfig {
interests: Vec::new(),
blocklist: Vec::new(),
},
clustering: crate::config::ClusteringSection::default(),
}
}
@@ -71,7 +74,11 @@ fn test_item(
id: Uuid::new_v4(),
source_id,
title: title.into(),
summary: "summary".into(),
// Reusing the title keeps each seed's payload lexically distinct:
// a shared generic summary would push near-duplicate seeds into
// the clusterer's merge or gray-zone bands instead of creating
// one cluster per seed.
summary: title.into(),
link: link.into(),
published_at,
kind,
@@ -126,7 +133,7 @@ async fn seed_cluster(
let item = test_item(source_id, title, link, published_at, kind);
ItemRepo::new(pool.clone()).insert_if_new(&item).await?;
ClusterRepo::new(pool.clone())
.assign_or_create(&item, 0.5)
.assign_or_create(&item, &ClusterParams::default())
.await?;
Ok(item)
}
@@ -179,13 +186,14 @@ async fn sends_digest_with_in_band_stories() -> Result<(), NotifyError> {
let base = now.with_timezone(&Utc) - Duration::hours(1);
let mut items = Vec::new();
for i in 0..3 {
let titles = ["Alpha bulletin", "Bravo dispatch", "Charlie ledger"];
for (i, title) in titles.iter().enumerate() {
let item = seed_cluster(
&pool,
source.id,
&format!("Story {}", i),
title,
&format!("https://example.test/{}", i),
base + Duration::minutes(i),
base + Duration::minutes(i as i64),
ItemKind::News,
)
.await?;
+1 -1
View File
@@ -4,7 +4,7 @@ pub mod cycle;
pub mod loops;
pub mod metrics;
pub use cycle::{PollCycleError, poll_source_once};
pub use cycle::{PendingConsolidation, PollCycleError, poll_source_once};
pub use loops::{
PruneReport, per_source_interval, run_pruning_once, spawn_digest_loop, spawn_gate_sync_loop,
spawn_pruning_loop, spawn_source_loop,
+43 -4
View File
@@ -2,9 +2,12 @@ use std::collections::HashMap;
use std::sync::Arc;
use chrono::Utc;
use news_core::clustering::MatchVerdict;
use news_core::{RawItem, Source, StoryCluster};
use news_ingest::{FeedPoller, PollResult};
use news_store::{ClusterRepo, ItemRepo, MetricKind, PercentileTracker, SourceRepo};
use news_store::{
AssignOutcome, ClusterParams, ClusterRepo, ItemRepo, MetricKind, PercentileTracker, SourceRepo,
};
use sqlx::SqlitePool;
use tokio::sync::Mutex;
use tokio::time::Instant;
@@ -67,6 +70,18 @@ impl From<crate::ServerError> for PollCycleError {
}
}
/// One item's gray-zone clustering outcome, awaiting LLM adjudication
/// (WS2): the item plus every candidate cluster whose best member verdict
/// was `llm_candidate`. Until the LLM layer runs, gray-zone pairs stay
/// split (fail-open).
#[derive(Debug)]
pub struct PendingConsolidation {
/// The newly inserted item that matched nothing outright.
pub item: RawItem,
/// Candidate cluster ids with their best member verdicts.
pub candidates: Vec<(Uuid, MatchVerdict)>,
}
/// Polls one source, inserts items, clusters them, scores them, evaluates the
/// gate, and either prints the decision (dry-run) or publishes real
/// notifications. The gate mutex is held only during each evaluate call —
@@ -85,6 +100,7 @@ pub async fn poll_source_once(
gate: &Arc<Mutex<NotificationGate>>,
ntfy: Option<&NtfyPublisher>,
metrics: &Metrics,
clustering: &ClusterParams,
dry_run: bool,
title_width: &mut usize,
) -> Result<(), PollCycleError> {
@@ -122,15 +138,38 @@ pub async fn poll_source_once(
let mut item_by_id: HashMap<Uuid, RawItem> = HashMap::new();
let mut seen_clusters: HashMap<Uuid, StoryCluster> = HashMap::new();
let mut pending_consolidations: Vec<PendingConsolidation> = Vec::new();
for item in items {
match items_repo.insert_if_new(&item).await? {
news_store::InsertOutcome::AlreadyExists => continue,
news_store::InsertOutcome::Inserted => {}
}
let cluster = clusters_repo.assign_or_create(&item, 0.5).await?;
item_by_id.insert(item.id, item);
seen_clusters.insert(cluster.id, cluster);
match clusters_repo.assign_or_create(&item, clustering).await? {
AssignOutcome::Assigned(cluster_id) | AssignOutcome::Created(cluster_id) => {
let cluster = clusters_repo.get(cluster_id).await?.ok_or_else(
|| -> Box<dyn std::error::Error + Send + Sync> {
"cluster missing after assignment".into()
},
)?;
item_by_id.insert(item.id, item);
seen_clusters.insert(cluster.id, cluster);
}
AssignOutcome::LlmCandidates(candidates) => {
pending_consolidations.push(PendingConsolidation { item, candidates });
}
}
}
// Gray-zone outcomes have no cluster yet: WS2's LLM consolidation will
// adjudicate them. Until then they stay split, so log and move on.
for pending in &pending_consolidations {
tracing::debug!(
item_id = %pending.item.id,
title = %pending.item.title,
candidates = ?pending.candidates,
"gray-zone cluster candidates queued for LLM consolidation; staying split"
);
}
let mut scored: Vec<(StoryCluster, RawItem, f64, f64)> = Vec::new();
+4 -1
View File
@@ -42,6 +42,7 @@ pub fn spawn_source_loop(
relevance_scorer =
refresh_relevance_scorer(&pool, &config_snapshot, relevance_scorer).await;
let mut title_width = 40_usize;
let clustering = config_snapshot.clustering.cluster_params();
let result = poll_source_once(
&source,
@@ -53,6 +54,7 @@ pub fn spawn_source_loop(
&gate,
ntfy.as_deref(),
&metrics,
&clustering,
false,
&mut title_width,
)
@@ -391,7 +393,7 @@ mod tests {
}
fn empty_topics_config() -> NewsConfig {
use crate::config::{NewsConfig, NotifySection, TopicsSection};
use crate::config::{ClusteringSection, NewsConfig, NotifySection, TopicsSection};
NewsConfig {
sources: Vec::new(),
notify: NotifySection {
@@ -411,6 +413,7 @@ mod tests {
interests: Vec::new(),
blocklist: Vec::new(),
},
clustering: ClusteringSection::default(),
}
}
+19 -3
View File
@@ -209,7 +209,7 @@ impl ImportanceScorer {
#[cfg(test)]
use news_core::{ItemKind, Source, SourceKind};
#[cfg(test)]
use news_store::{ClusterRepo, SourceRepo, init_db};
use news_store::{AssignOutcome, ClusterParams, ClusterRepo, SourceRepo, init_db};
#[cfg(test)]
use pretty_assertions::assert_eq;
@@ -425,8 +425,24 @@ async fn importance_scorer_matches_direct_formula() -> Result<(), ServerError> {
items.insert_if_new(&item_a).await?;
items.insert_if_new(&item_b).await?;
let _cluster = clusters.assign_or_create(&item_a, 0.5).await?;
let cluster = clusters.assign_or_create(&item_b, 0.5).await?;
let _ = clusters
.assign_or_create(&item_a, &ClusterParams::default())
.await?;
let outcome = clusters
.assign_or_create(&item_b, &ClusterParams::default())
.await?;
let cluster_id = match outcome {
AssignOutcome::Assigned(id) | AssignOutcome::Created(id) => id,
AssignOutcome::LlmCandidates(_) => {
return Err(ServerError::DateTime(
"seed item unexpectedly landed in the gray zone".into(),
));
}
};
let cluster = clusters
.get(cluster_id)
.await?
.ok_or_else(|| ServerError::DateTime("cluster missing after assignment".into()))?;
assert_eq!(cluster.source_count, 2);
let scorer = ImportanceScorer::new(pool);
@@ -0,0 +1,15 @@
-- v0.2 headline/link columns for clusters. `llm_headline` holds the
-- LLM-written neutral headline (NULL until the consolidation layer writes
-- it); `link_item_id` points at the highest-weight source's item for the
-- cluster. `raw_items.id` is TEXT (0001_init.sql), so the FK column is
-- TEXT; ON DELETE SET NULL keeps the cluster row when a linked item is
-- pruned. Existing rows get NULL for both columns (no retro-merge).
ALTER TABLE story_clusters ADD COLUMN llm_headline TEXT;
ALTER TABLE story_clusters ADD COLUMN link_item_id TEXT
REFERENCES raw_items(id) ON DELETE SET NULL;
-- The window candidate query range-scans story_clusters(last_seen_at);
-- 0003 already covers first_seen_at (idx_story_clusters_first_seen_at)
-- but last_seen_at had no index.
CREATE INDEX idx_story_clusters_last_seen_at
ON story_clusters(last_seen_at);
+325 -141
View File
@@ -1,78 +1,158 @@
//! Story-cluster repository: assigns incoming raw items to an existing
//! cluster or starts a new one, using title-token Jaccard similarity and a
//! timestamp window anchored to the incoming item.
//! cluster or starts a new one, using weighted token-set similarity with a
//! shared-token corroboration gate and a warm-cluster window anchored to
//! the incoming item's publish time.
use chrono::{DateTime, Utc};
use chrono::{Duration, Utc};
use news_core::clustering::{ItemText, MatchVerdict, SimParams};
use news_core::{RawItem, StoryCluster};
use sqlx::{Row, SqlitePool};
use uuid::Uuid;
use crate::StoreError;
/// Per-cluster member cap for candidate comparison: only the most recent
/// 20 members of a warm cluster are compared against the incoming item.
const MEMBER_CAP: usize = 20;
/// Late-arrival grace: an item may predate a cluster's first_seen_at by at
/// most this many hours and still join it (out-of-order feed delivery).
const GRACE_HOURS: i64 = 2;
/// Repository over the `story_clusters` and `cluster_members` tables.
pub struct ClusterRepo(SqlitePool);
/// Tunables for [`ClusterRepo::assign_or_create`].
#[derive(Debug, Clone, PartialEq)]
pub struct ClusterParams {
/// Similarity-engine parameters (thresholds, corroboration, weights).
pub sim: SimParams,
/// How far back a cluster stays warm for new items, in hours.
pub window_hours: i64,
}
impl Default for ClusterParams {
fn default() -> Self {
Self {
sim: SimParams::default(),
window_hours: 48,
}
}
}
/// Outcome of one assignment attempt.
#[derive(Debug, Clone, PartialEq)]
pub enum AssignOutcome {
/// The item joined an existing cluster.
Assigned(Uuid),
/// No candidate merged; a new cluster was started.
Created(Uuid),
/// Nothing merged, but at least one candidate scored in the gray zone.
/// The LLM consolidation layer (WS2) adjudicates these; until then the
/// pair stays split (fail-open).
LlmCandidates(Vec<(Uuid, MatchVerdict)>),
}
impl ClusterRepo {
pub fn new(pool: SqlitePool) -> Self {
Self(pool)
}
/// Assigns `item` to the best matching open cluster, or creates a new
/// Loads a cluster with its member item ids, or `None` when absent.
pub async fn get(&self, cluster_id: Uuid) -> Result<Option<StoryCluster>, StoreError> {
load_cluster(&self.0, &cluster_id.to_string()).await
}
/// Assigns `item` to the best matching warm cluster, or creates a new
/// cluster if none qualifies.
///
/// A cluster is considered "open" for the incoming item when its
/// `last_seen_at` is at most the item's `published_at` and the gap is
/// strictly less than 45 minutes. The window is anchored to the item's
/// timestamp, not wall-clock time, so tests are deterministic.
/// A cluster is a candidate for the incoming item when its
/// `last_seen_at` is within `window_hours` of the item's `published_at`
/// and the item is no more than [`GRACE_HOURS`] older than the
/// cluster's `first_seen_at` (late arrivals). The window is anchored to
/// the item's timestamp, not wall-clock time, so tests are
/// deterministic.
///
/// All writes for a single assignment happen inside one sqlx transaction.
/// The item is compared against every (capped) member of each candidate
/// cluster — not just the canonical item — and the best member verdict
/// stands. All writes for a single assignment happen inside one sqlx
/// transaction.
pub async fn assign_or_create(
&self,
item: &RawItem,
threshold: f64,
) -> Result<StoryCluster, StoreError> {
params: &ClusterParams,
) -> Result<AssignOutcome, StoreError> {
let mut tx = self.0.begin().await?;
let candidate_rows = sqlx::query(
"SELECT \
sc.id AS cluster_id, \
sc.canonical_item_id, \
sc.source_count, \
sc.first_seen_at, \
sc.last_seen_at, \
ri.id AS raw_id, \
ri.source_id, \
ri.title, \
ri.summary, \
ri.link, \
ri.published_at, \
ri.kind \
// Window bounds are computed in Rust so the predicates stay plain
// string range comparisons on the RFC3339 TEXT columns and can use
// the story_clusters(last_seen_at) / (first_seen_at) indexes.
let window_floor = (item.published_at - Duration::hours(params.window_hours)).to_rfc3339();
let grace_ceiling = (item.published_at + Duration::hours(GRACE_HOURS)).to_rfc3339();
let rows = sqlx::query(
"SELECT sc.id AS cluster_id, ri.title, ri.summary \
FROM story_clusters sc \
JOIN raw_items ri ON ri.id = sc.canonical_item_id \
WHERE sc.last_seen_at <= ?1 \
AND (julianday(?1) - julianday(sc.last_seen_at)) < 45.0 / 1440.0",
JOIN cluster_members cm ON cm.cluster_id = sc.id \
JOIN raw_items ri ON ri.id = cm.item_id \
WHERE sc.last_seen_at >= ?1 AND sc.first_seen_at <= ?2 \
ORDER BY sc.id, ri.published_at DESC, ri.id DESC",
)
.bind(item.published_at.to_rfc3339())
.bind(&window_floor)
.bind(&grace_ceiling)
.fetch_all(&mut *tx)
.await?;
let mut best: Option<(String, f64)> = None;
for row in &candidate_rows {
let canonical = row_to_raw_item(row)?;
let sim = news_core::clustering::similarity(item, &canonical);
if news_core::clustering::should_cluster(sim, threshold) {
let cluster_id: String = row.try_get("cluster_id")?;
match best {
None => best = Some((cluster_id, sim)),
Some((_, best_sim)) if sim > best_sim => {
best = Some((cluster_id, sim));
let incoming = ItemText {
title: item.title.clone(),
summary: item.summary.clone(),
};
// Rows are ordered by cluster id, so each cluster's members are
// contiguous; a single pass keeps the most recent `MEMBER_CAP`.
let mut candidates: Vec<(String, Vec<ItemText>)> = Vec::new();
for row in &rows {
let cluster_id: String = row.try_get("cluster_id")?;
let member = ItemText {
title: row.try_get("title")?,
summary: row.try_get("summary")?,
};
match candidates.last_mut() {
Some((id, members)) if *id == cluster_id => {
if members.len() < MEMBER_CAP {
members.push(member);
}
Some(_) => {}
}
_ => candidates.push((cluster_id, vec![member])),
}
}
let cluster = if let Some((cluster_id, _)) = best {
// Best member verdict per cluster; the best merging cluster wins.
let mut best_merge: Option<(String, MatchVerdict)> = None;
let mut gray_zone: Vec<(String, MatchVerdict)> = Vec::new();
for (cluster_id, members) in &candidates {
let mut best: Option<MatchVerdict> = None;
for member in members {
let verdict = news_core::clustering::compare(&incoming, member, &params.sim);
if best.as_ref().is_none_or(|best| verdict.score > best.score) {
best = Some(verdict);
}
}
let Some(verdict) = best else {
continue;
};
if verdict.merge
&& best_merge
.as_ref()
.is_none_or(|(_, best)| verdict.score > best.score)
{
best_merge = Some((cluster_id.clone(), verdict));
} else if !verdict.merge && verdict.llm_candidate {
gray_zone.push((cluster_id.clone(), verdict));
}
}
let outcome = if let Some((cluster_id, _)) = best_merge {
sqlx::query("INSERT INTO cluster_members (cluster_id, item_id) VALUES (?, ?)")
.bind(&cluster_id)
.bind(item.id.to_string())
@@ -89,6 +169,7 @@ impl ClusterRepo {
.fetch_one(&mut *tx)
.await?;
// last_seen_at only ever moves forward.
sqlx::query(
"UPDATE story_clusters \
SET source_count = ?, \
@@ -101,10 +182,15 @@ impl ClusterRepo {
.execute(&mut *tx)
.await?;
load_cluster(&mut tx, &cluster_id).await?
AssignOutcome::Assigned(Uuid::parse_str(&cluster_id)?)
} else if !gray_zone.is_empty() {
let candidates = gray_zone
.into_iter()
.map(|(id, verdict)| Ok((Uuid::parse_str(&id)?, verdict)))
.collect::<Result<Vec<_>, StoreError>>()?;
AssignOutcome::LlmCandidates(candidates)
} else {
let cluster_id = Uuid::new_v4();
let cluster_id_str = cluster_id.to_string();
let cluster_id_str = Uuid::new_v4().to_string();
sqlx::query(
"INSERT INTO story_clusters \
@@ -125,67 +211,35 @@ impl ClusterRepo {
.execute(&mut *tx)
.await?;
StoryCluster {
id: cluster_id,
canonical_item_id: item.id,
member_item_ids: vec![item.id],
source_count: 1,
first_seen_at: item.published_at,
last_seen_at: item.published_at,
}
AssignOutcome::Created(Uuid::parse_str(&cluster_id_str)?)
};
tx.commit().await?;
Ok(cluster)
}
}
fn row_to_raw_item(row: &sqlx::sqlite::SqliteRow) -> Result<RawItem, StoreError> {
let id: String = row.try_get("raw_id")?;
let source_id: String = row.try_get("source_id")?;
let kind: String = row.try_get("kind")?;
let published_at: String = row.try_get("published_at")?;
Ok(RawItem {
id: Uuid::parse_str(&id)?,
source_id: Uuid::parse_str(&source_id)?,
title: row.try_get("title")?,
summary: row.try_get("summary")?,
link: row.try_get("link")?,
published_at: DateTime::parse_from_rfc3339(&published_at)
.map_err(|e| StoreError::InvalidStoredValue(format!("invalid published_at: {e}")))?
.with_timezone(&Utc),
kind: kind_from_str(&kind)?,
})
}
fn kind_from_str(kind: &str) -> Result<news_core::ItemKind, StoreError> {
match kind {
"news" => Ok(news_core::ItemKind::News),
"opinion" => Ok(news_core::ItemKind::Opinion),
other => Err(StoreError::InvalidStoredValue(format!(
"unrecognized item kind: {other}"
))),
Ok(outcome)
}
}
async fn load_cluster(
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
pool: &sqlx::SqlitePool,
cluster_id: &str,
) -> Result<StoryCluster, StoreError> {
) -> Result<Option<StoryCluster>, StoreError> {
let row = sqlx::query(
"SELECT id, canonical_item_id, source_count, first_seen_at, last_seen_at \
FROM story_clusters WHERE id = ?",
)
.bind(cluster_id)
.fetch_one(&mut **tx)
.fetch_optional(pool)
.await?;
let Some(row) = row else {
return Ok(None);
};
let member_ids: Vec<String> = sqlx::query_scalar(
"SELECT item_id FROM cluster_members WHERE cluster_id = ? ORDER BY item_id",
)
.bind(cluster_id)
.fetch_all(&mut **tx)
.fetch_all(pool)
.await?;
let id: String = row.try_get("id")?;
@@ -194,7 +248,7 @@ async fn load_cluster(
let first_seen_at: String = row.try_get("first_seen_at")?;
let last_seen_at: String = row.try_get("last_seen_at")?;
Ok(StoryCluster {
Ok(Some(StoryCluster {
id: Uuid::parse_str(&id)?,
canonical_item_id: Uuid::parse_str(&canonical_item_id)?,
member_item_ids: member_ids
@@ -208,11 +262,11 @@ async fn load_cluster(
})?,
first_seen_at: parse_datetime(&first_seen_at)?,
last_seen_at: parse_datetime(&last_seen_at)?,
})
}))
}
fn parse_datetime(value: &str) -> Result<DateTime<Utc>, StoreError> {
Ok(DateTime::parse_from_rfc3339(value)
fn parse_datetime(value: &str) -> Result<chrono::DateTime<Utc>, StoreError> {
Ok(chrono::DateTime::parse_from_rfc3339(value)
.map_err(|e| StoreError::InvalidStoredValue(format!("invalid timestamp: {e}")))?
.with_timezone(&Utc))
}
@@ -221,11 +275,13 @@ fn parse_datetime(value: &str) -> Result<DateTime<Utc>, StoreError> {
mod tests {
use super::*;
use crate::{ItemRepo, SourceRepo, StoreError, init_db};
use chrono::{Duration, Utc};
use news_core::{ItemKind, RawItem, Source, SourceKind};
use chrono::{DateTime, Duration, Utc};
use news_core::{ItemKind, Source, SourceKind};
use pretty_assertions::assert_eq;
const THRESHOLD: f64 = 0.5;
fn params() -> ClusterParams {
ClusterParams::default()
}
fn test_source(name: &str, url: &str) -> Source {
Source {
@@ -259,6 +315,34 @@ mod tests {
Ok((pool, sources, items, clusters))
}
fn require_assigned(outcome: AssignOutcome) -> Result<Uuid, StoreError> {
match outcome {
AssignOutcome::Assigned(id) => Ok(id),
other => Err(StoreError::InvalidStoredValue(format!(
"expected Assigned, got {other:?}"
))),
}
}
fn require_created(outcome: AssignOutcome) -> Result<Uuid, StoreError> {
match outcome {
AssignOutcome::Created(id) => Ok(id),
other => Err(StoreError::InvalidStoredValue(format!(
"expected Created, got {other:?}"
))),
}
}
async fn load_cluster_row(
clusters: &ClusterRepo,
id: Uuid,
) -> Result<StoryCluster, StoreError> {
clusters
.get(id)
.await?
.ok_or_else(|| StoreError::InvalidStoredValue("cluster missing after write".into()))
}
#[tokio::test]
async fn source_count_is_distinct_sources_not_item_count() -> Result<(), StoreError> {
let (_pool, sources, items, clusters) = seeded_repos().await?;
@@ -272,8 +356,8 @@ mod tests {
Utc::now(),
);
items.insert_if_new(&item_a1).await?;
let cluster1 = clusters.assign_or_create(&item_a1, THRESHOLD).await?;
assert_eq!(cluster1.source_count, 1);
let cluster1 = require_created(clusters.assign_or_create(&item_a1, &params()).await?)?;
assert_eq!(load_cluster_row(&clusters, cluster1).await?.source_count, 1);
let item_a2 = test_item(
source_a.id,
@@ -282,9 +366,9 @@ mod tests {
Utc::now(),
);
items.insert_if_new(&item_a2).await?;
let cluster2 = clusters.assign_or_create(&item_a2, THRESHOLD).await?;
assert_eq!(cluster2.id, cluster1.id);
assert_eq!(cluster2.source_count, 1);
let cluster2 = require_assigned(clusters.assign_or_create(&item_a2, &params()).await?)?;
assert_eq!(cluster2, cluster1);
assert_eq!(load_cluster_row(&clusters, cluster2).await?.source_count, 1);
let source_b = test_source("Source B", "https://b.test");
sources.upsert(&source_b).await?;
@@ -295,49 +379,16 @@ mod tests {
Utc::now(),
);
items.insert_if_new(&item_b1).await?;
let cluster3 = clusters.assign_or_create(&item_b1, THRESHOLD).await?;
assert_eq!(cluster3.id, cluster1.id);
assert_eq!(cluster3.source_count, 2);
Ok(())
}
#[tokio::test]
async fn item_outside_45_minute_window_starts_new_cluster() -> Result<(), StoreError> {
let (_pool, sources, items, clusters) = seeded_repos().await?;
let source = test_source("Source", "https://example.test");
sources.upsert(&source).await?;
let base = Utc::now();
let item1 = test_item(
source.id,
"Storm system moves toward the coast",
"https://example.test/1",
base,
);
items.insert_if_new(&item1).await?;
clusters.assign_or_create(&item1, THRESHOLD).await?;
let item2 = test_item(
source.id,
"Storm system moves toward coast",
"https://example.test/2",
base + Duration::minutes(46),
);
items.insert_if_new(&item2).await?;
clusters.assign_or_create(&item2, THRESHOLD).await?;
let count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM story_clusters")
.fetch_one(&_pool)
.await?;
assert_eq!(count, 2);
let cluster3 = require_assigned(clusters.assign_or_create(&item_b1, &params()).await?)?;
assert_eq!(cluster3, cluster1);
assert_eq!(load_cluster_row(&clusters, cluster3).await?.source_count, 2);
Ok(())
}
#[tokio::test]
async fn same_source_repeats_create_one_cluster_with_both_members() -> Result<(), StoreError> {
let (_pool, sources, items, clusters) = seeded_repos().await?;
let (pool, sources, items, clusters) = seeded_repos().await?;
let source = test_source("Al Jazeera", "https://aljazeera.test");
sources.upsert(&source).await?;
@@ -348,7 +399,7 @@ mod tests {
Utc::now(),
);
items.insert_if_new(&item1).await?;
let cluster1 = clusters.assign_or_create(&item1, THRESHOLD).await?;
let cluster1 = require_created(clusters.assign_or_create(&item1, &params()).await?)?;
let item2 = test_item(
source.id,
@@ -357,17 +408,150 @@ mod tests {
Utc::now(),
);
items.insert_if_new(&item2).await?;
let cluster2 = clusters.assign_or_create(&item2, THRESHOLD).await?;
let cluster2 = require_assigned(clusters.assign_or_create(&item2, &params()).await?)?;
assert_eq!(cluster2, cluster1);
assert_eq!(cluster2.id, cluster1.id);
assert_eq!(cluster2.source_count, 1);
assert_eq!(cluster2.member_item_ids.len(), 2);
let cluster = load_cluster_row(&clusters, cluster2).await?;
assert_eq!(cluster.source_count, 1);
assert_eq!(cluster.member_item_ids.len(), 2);
let count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM cluster_members")
.fetch_one(&_pool)
.fetch_one(&pool)
.await?;
assert_eq!(count, 2);
Ok(())
}
#[tokio::test]
async fn late_arrival_joins() -> Result<(), StoreError> {
let (_pool, sources, items, clusters) = seeded_repos().await?;
let source = test_source("Source", "https://example.test");
sources.upsert(&source).await?;
// The cluster exists first; the matching item arrives published
// 90 minutes BEFORE the cluster's first_seen_at (out-of-order
// feed delivery) — inside the 2 h late-arrival grace.
let base = Utc::now();
let item1 = test_item(
source.id,
"Floods swamp coastal town after storm",
"https://example.test/1",
base,
);
items.insert_if_new(&item1).await?;
let cluster1 = require_created(clusters.assign_or_create(&item1, &params()).await?)?;
let item2 = test_item(
source.id,
"Floods swamp coastal town",
"https://example.test/2",
base - Duration::minutes(90),
);
items.insert_if_new(&item2).await?;
let outcome = clusters.assign_or_create(&item2, &params()).await?;
let joined = require_assigned(outcome)?;
assert_eq!(joined, cluster1);
// last_seen_at stays at the forward bound (max(existing, item)).
let cluster = load_cluster_row(&clusters, cluster1).await?;
assert_eq!(cluster.last_seen_at, base);
assert_eq!(cluster.member_item_ids.len(), 2);
Ok(())
}
#[tokio::test]
async fn member_drift() -> Result<(), StoreError> {
let (_pool, sources, items, clusters) = seeded_repos().await?;
let source = test_source("Source", "https://example.test");
sources.upsert(&source).await?;
let base = Utc::now();
// Canonical member: the incoming item does NOT clear the
// corroboration gate against it (shared non-proper: {earthquake}).
let member1 = test_item(
source.id,
"Earthquake rattles Tokyo buildings",
"https://example.test/1",
base,
);
items.insert_if_new(&member1).await?;
let cluster1 = require_created(clusters.assign_or_create(&member1, &params()).await?)?;
let member2 = test_item(
source.id,
"Tokyo earthquake damages buildings",
"https://example.test/2",
base + Duration::minutes(1),
);
items.insert_if_new(&member2).await?;
require_assigned(clusters.assign_or_create(&member2, &params()).await?)?;
let member3 = test_item(
source.id,
"Powerful earthquake shakes Tokyo structures",
"https://example.test/3",
base + Duration::minutes(2),
);
items.insert_if_new(&member3).await?;
require_assigned(clusters.assign_or_create(&member3, &params()).await?)?;
// The incoming item matches the cluster's 3rd member, not the
// canonical one — member-wise comparison must still assign it.
let incoming = test_item(
source.id,
"Magnitude 7 earthquake damages Tokyo structures overnight",
"https://example.test/4",
base + Duration::minutes(3),
);
items.insert_if_new(&incoming).await?;
let outcome = clusters.assign_or_create(&incoming, &params()).await?;
let joined = require_assigned(outcome)?;
assert_eq!(joined, cluster1);
// Canonical membership is unchanged by the drift.
let cluster = load_cluster_row(&clusters, cluster1).await?;
assert_eq!(cluster.canonical_item_id, member1.id);
assert_eq!(cluster.member_item_ids.len(), 4);
Ok(())
}
#[tokio::test]
async fn window_expiry() -> Result<(), StoreError> {
let (pool, sources, items, clusters) = seeded_repos().await?;
let source = test_source("Source", "https://example.test");
sources.upsert(&source).await?;
let base = Utc::now();
let item1 = test_item(
source.id,
"Quake damages historic temple in Kyoto",
"https://example.test/1",
base,
);
items.insert_if_new(&item1).await?;
let cluster1 = require_created(clusters.assign_or_create(&item1, &params()).await?)?;
// Identical title, but the cluster went cold 50 hours ago (window:
// 48 h) — not a candidate, so a fresh cluster starts.
let item2 = test_item(
source.id,
"Quake damages historic temple in Kyoto",
"https://example.test/2",
base + Duration::hours(50),
);
items.insert_if_new(&item2).await?;
let outcome = clusters.assign_or_create(&item2, &params()).await?;
let created = require_created(outcome)?;
assert_ne!(created, cluster1);
let count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM story_clusters")
.fetch_one(&pool)
.await?;
assert_eq!(count, 2);
Ok(())
}
}
+12 -4
View File
@@ -70,7 +70,7 @@ impl FeedbackRepo {
#[cfg(test)]
mod tests {
use super::*;
use crate::{ClusterRepo, ItemRepo, SourceRepo, init_db};
use crate::{AssignOutcome, ClusterParams, ClusterRepo, ItemRepo, SourceRepo, init_db};
use chrono::Utc;
use news_core::{ItemKind, RawItem, Source, SourceKind};
use pretty_assertions::assert_eq;
@@ -105,10 +105,18 @@ mod tests {
SourceRepo::new(pool.clone()).upsert(&source).await?;
let item = test_item(source.id);
ItemRepo::new(pool.clone()).insert_if_new(&item).await?;
let cluster = ClusterRepo::new(pool.clone())
.assign_or_create(&item, 0.5)
let outcome = ClusterRepo::new(pool.clone())
.assign_or_create(&item, &ClusterParams::default())
.await?;
Ok((pool, cluster.id))
let cluster_id = match outcome {
AssignOutcome::Assigned(id) | AssignOutcome::Created(id) => id,
AssignOutcome::LlmCandidates(_) => {
return Err(StoreError::InvalidStoredValue(
"seed item unexpectedly landed in the gray zone".into(),
));
}
};
Ok((pool, cluster_id))
}
#[tokio::test]
+1 -1
View File
@@ -10,7 +10,7 @@ mod percentile;
mod prune;
mod sources;
pub use clusters::ClusterRepo;
pub use clusters::{AssignOutcome, ClusterParams, ClusterRepo};
pub use error::StoreError;
pub use feedback::FeedbackRepo;
pub use items::{InsertOutcome, ItemRepo};
+45
View File
@@ -223,6 +223,51 @@ end of a deploy that finished. Anything that fails before that point leaves no
record, so the next run does the whole thing again rather than mistaking a
pulled image for a deployed one.
## Verifying a deployment
The version shows up in three places, and after a deploy they should all tell
the same story.
The web footer shows `v{version} · {short sha}`, fetched from the API when the
page loads; if that fetch fails, no footer renders at all. The same endpoint,
asked directly:
```sh
curl https://news.rcjohnstone.com/api/version
# {"version":"0.2.0","git_sha":"…","build_time":"…"}
```
The image carries the same facts as OCI labels, and the tags show what points
where:
```sh
sudo podman inspect git.rcjohnstone.com/connor/news:latest \
--format '{{index .Config.Labels "org.opencontainers.image.version"}} {{index .Config.Labels "org.opencontainers.image.revision"}}'
sudo podman images git.rcjohnstone.com/connor/news
```
CI tags a single image id as `latest`, the short commit sha, and the release
tag (`v0.2.0`), so `podman images` lists three rows with the same image id —
and the labels are the durable record of which version and which commit an
image was actually built from.
The updater keeps its own trail: `web/dist.deployed` holds the image id of the
last deploy that got all the way through, and the service journal holds every
run, including the ones that changed nothing:
```sh
cat ~/data/news/web/dist.deployed
journalctl -u news-update.service -n 50 --no-pager
```
`systemctl list-timers news-update.timer` says when the next check fires.
None of that changes what the updater does: it still pulls `:latest`, compares
the id against `dist.deployed`, and exits when they match. The `vX.Y.Z` tags
are for people, not for the updater — a way to pin the compose file's `image:`
or to roll back to a build you can name by version instead of a commit hash
you have to go and look up.
## Rolling back
The frontend of the previous deploy is kept: