v0.2 wave 1: merge-rule-v2 clustering with LLM-candidate gray zone, 0.2.0 version plumbing, deploy verification docs
This commit is contained in:
@@ -11,6 +11,8 @@ on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
tags:
|
||||
- 'v*'
|
||||
|
||||
jobs:
|
||||
image:
|
||||
@@ -18,6 +20,10 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Compute short sha
|
||||
id: short
|
||||
run: echo "sha=$(git rev-parse --short HEAD)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- uses: docker/setup-buildx-action@v3
|
||||
|
||||
- uses: docker/login-action@v3
|
||||
@@ -28,14 +34,24 @@ jobs:
|
||||
|
||||
# Tagged with the commit as well as `latest`, so that "what is actually
|
||||
# running" has an answer, and so a rollback is a tag rather than a revert
|
||||
# and a rebuild.
|
||||
# and a rebuild. On a `v*` tag build the tag name is added too, so an
|
||||
# exact version can be pinned; on branch pushes that expression is empty
|
||||
# and the line is dropped. The version build-arg/label follow the same
|
||||
# rule and feed /api/version through the Dockerfile's ARGs.
|
||||
- uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
file: ./Dockerfile
|
||||
push: true
|
||||
build-args: |
|
||||
NEWS_BUILD_VERSION=${{ github.ref_type == 'tag' && github.ref_name || 'dev' }}
|
||||
NEWS_BUILD_SHA=${{ github.sha }}
|
||||
labels: |
|
||||
org.opencontainers.image.version=${{ github.ref_type == 'tag' && github.ref_name || 'dev' }}
|
||||
org.opencontainers.image.revision=${{ github.sha }}
|
||||
tags: |
|
||||
${{ vars.REGISTRY }}/connor/news:latest
|
||||
${{ vars.REGISTRY }}/connor/news:${{ github.sha }}
|
||||
${{ vars.REGISTRY }}/connor/news:${{ steps.short.outputs.sha }}
|
||||
${{ vars.REGISTRY }}/connor/news:${{ github.ref_type == 'tag' && github.ref_name || '' }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
|
||||
Generated
+6
-6
@@ -1630,7 +1630,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "news-cli"
|
||||
version = "0.1.0"
|
||||
version = "0.2.0"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"clap",
|
||||
@@ -1646,7 +1646,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "news-core"
|
||||
version = "0.1.0"
|
||||
version = "0.2.0"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"pretty_assertions",
|
||||
@@ -1657,7 +1657,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "news-ingest"
|
||||
version = "0.1.0"
|
||||
version = "0.2.0"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"news-core",
|
||||
@@ -1672,7 +1672,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "news-server"
|
||||
version = "0.1.0"
|
||||
version = "0.2.0"
|
||||
dependencies = [
|
||||
"axum",
|
||||
"chrono",
|
||||
@@ -1702,7 +1702,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "news-store"
|
||||
version = "0.1.0"
|
||||
version = "0.2.0"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"news-core",
|
||||
@@ -1715,7 +1715,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "news-web"
|
||||
version = "0.1.0"
|
||||
version = "0.2.0"
|
||||
dependencies = [
|
||||
"console_error_panic_hook",
|
||||
"gloo-net",
|
||||
|
||||
+1
-1
@@ -10,7 +10,7 @@ members = [
|
||||
]
|
||||
|
||||
[workspace.package]
|
||||
version = "0.1.0"
|
||||
version = "0.2.0"
|
||||
edition = "2024"
|
||||
license = "MIT"
|
||||
|
||||
|
||||
@@ -62,9 +62,18 @@ RUN --mount=type=cache,id=web-registry,target=/usr/local/cargo/registry \
|
||||
# ----------------------------------------------------------------- backend --
|
||||
FROM rust:1.89-slim-bookworm AS server
|
||||
|
||||
# Build metadata for /api/version, baked in by build.rs (see that file for
|
||||
# the fallbacks). release.yml passes the real values; a bare `docker build`
|
||||
# gets `dev`/`unknown`, matching a plain `cargo build` on a dev machine.
|
||||
ARG NEWS_BUILD_VERSION=dev
|
||||
ARG NEWS_BUILD_SHA=unknown
|
||||
|
||||
WORKDIR /app
|
||||
COPY . .
|
||||
|
||||
ENV NEWS_BUILD_VERSION=${NEWS_BUILD_VERSION} \
|
||||
NEWS_BUILD_SHA=${NEWS_BUILD_SHA}
|
||||
|
||||
# The binary is copied out of the cache mount because a cache mount is not part
|
||||
# of the layer: whatever is written there is gone by the time the next stage
|
||||
# looks. This is also why there is no dummy-source dance -- the cache does what
|
||||
|
||||
@@ -7,6 +7,7 @@ mod replay;
|
||||
#[derive(Parser)]
|
||||
#[command(name = "news-cli")]
|
||||
#[command(about = "Offline replay harness for news-triage decisions")]
|
||||
#[command(version)]
|
||||
struct Cli {
|
||||
#[command(subcommand)]
|
||||
command: Command,
|
||||
|
||||
@@ -11,13 +11,15 @@ use news_server::{
|
||||
DecisionRow, ImportanceScorer, NotificationGate, NotifyConfig, RelevanceScorer, TokenBucket,
|
||||
format_decision_header, format_decision_row,
|
||||
};
|
||||
use news_store::{ClusterRepo, ItemRepo, MetricKind, PercentileTracker, SourceRepo, init_db};
|
||||
use news_store::{
|
||||
AssignOutcome, ClusterParams, ClusterRepo, ItemRepo, MetricKind, PercentileTracker, SourceRepo,
|
||||
init_db,
|
||||
};
|
||||
use serde::Deserialize;
|
||||
use uuid::Uuid;
|
||||
|
||||
const DEFAULT_NOW: &str = "2026-01-15T12:00:00Z";
|
||||
const DEFAULT_WEIGHT: f64 = 0.8;
|
||||
const CLUSTER_THRESHOLD: f64 = 0.5;
|
||||
|
||||
#[derive(Debug, Default, Deserialize)]
|
||||
struct Manifest {
|
||||
@@ -69,9 +71,18 @@ pub async fn run(
|
||||
let items = parse_fixture(&body, &source)?;
|
||||
for item in items {
|
||||
items_repo.insert_if_new(&item).await?;
|
||||
let cluster = clusters_repo
|
||||
.assign_or_create(&item, CLUSTER_THRESHOLD)
|
||||
let outcome = clusters_repo
|
||||
.assign_or_create(&item, &ClusterParams::default())
|
||||
.await?;
|
||||
// Gray-zone pairs stay split in replay (no LLM layer).
|
||||
let cluster_id = match outcome {
|
||||
AssignOutcome::Assigned(id) | AssignOutcome::Created(id) => id,
|
||||
AssignOutcome::LlmCandidates(_) => continue,
|
||||
};
|
||||
let cluster = clusters_repo
|
||||
.get(cluster_id)
|
||||
.await?
|
||||
.ok_or("cluster missing after assignment")?;
|
||||
item_by_id.insert(item.id, item);
|
||||
seen_clusters.insert(cluster.id, cluster);
|
||||
}
|
||||
|
||||
@@ -28,5 +28,13 @@
|
||||
<description>A winter storm warning is in effect for higher elevations.</description>
|
||||
<category>World</category>
|
||||
</item>
|
||||
<item>
|
||||
<title>Summit postponed amid election dispute</title>
|
||||
<link>https://example.test/source-c/summit-dispute</link>
|
||||
<guid isPermaLink="true">https://example.test/source-c/summit-dispute</guid>
|
||||
<pubDate>Wed, 14 Jan 2026 11:25:00 GMT</pubDate>
|
||||
<description>Organizers delayed the gathering while officials review the dispute.</description>
|
||||
<category>World</category>
|
||||
</item>
|
||||
</channel>
|
||||
</rss>
|
||||
|
||||
@@ -1,47 +1,151 @@
|
||||
//! Title-similarity clustering for grouping raw feed items that describe
|
||||
//! the same breaking story.
|
||||
//! Lexical clustering for grouping raw feed items that describe the same
|
||||
//! breaking story: weighted token-set containment between title/summary
|
||||
//! pairs, gated by a minimum count of shared common (non-proper-noun)
|
||||
//! tokens. Pure functions, no I/O.
|
||||
|
||||
use std::collections::HashSet;
|
||||
|
||||
use crate::RawItem;
|
||||
/// Summary tokens folded into the scoring pool. The cap keeps short-title
|
||||
/// pairs decisive when one outlet attaches a long summary.
|
||||
const SUMMARY_TOKEN_LIMIT: usize = 40;
|
||||
|
||||
/// Tiny built-in stopword list. These words are dropped before token-set
|
||||
/// comparison because they carry almost no story-discriminating signal and
|
||||
/// dominate syndication-style title variants ("the coast" vs "coast").
|
||||
const STOPWORDS: &[&str] = &["a", "the", "of", "to", "in", "and", "for", "on", "at"];
|
||||
/// Tiny built-in stopword list. These words are dropped before comparison
|
||||
/// because they carry almost no story-discriminating signal and dominate
|
||||
/// syndication-style title variants ("the coast" vs "coast"). `new`,
|
||||
/// `first`, and `top` are deliberately NOT stopwords — they are content.
|
||||
const STOPWORDS: &[&str] = &[
|
||||
"a", "the", "of", "to", "in", "and", "for", "on", "at", //
|
||||
"is", "are", "was", "were", "be", "been", "as", "by", "from", //
|
||||
"has", "have", "had", "that", "this", "with", "after", "over", //
|
||||
"into", "amid", "during", "says", "said", "say", "tells", //
|
||||
"will", "would", "could", "can",
|
||||
];
|
||||
|
||||
/// Returns the Jaccard similarity of two titles, in the range `0.0..=1.0`.
|
||||
///
|
||||
/// Tokenisation rules:
|
||||
/// * split on non-alphanumeric characters;
|
||||
/// * lower-case each token;
|
||||
/// * drop empty tokens;
|
||||
/// * drop the small built-in stopword set documented above.
|
||||
///
|
||||
/// If both token sets are empty the similarity is defined as `0.0` so the
|
||||
/// function always returns a valid, bounded value.
|
||||
pub fn similarity(a: &RawItem, b: &RawItem) -> f64 {
|
||||
let a_tokens: HashSet<String> = tokenize(&a.title).into_iter().collect();
|
||||
let b_tokens: HashSet<String> = tokenize(&b.title).into_iter().collect();
|
||||
/// Text of one item taking part in a comparison.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ItemText {
|
||||
pub title: String,
|
||||
pub summary: String,
|
||||
}
|
||||
|
||||
if a_tokens.is_empty() && b_tokens.is_empty() {
|
||||
/// Tunables for the similarity engine.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct SimParams {
|
||||
/// Score at/above which a pair can merge (still subject to the
|
||||
/// corroboration gate).
|
||||
pub threshold: f64,
|
||||
/// Width of the below-threshold band that still routes to LLM
|
||||
/// adjudication instead of an outright split.
|
||||
pub gray_zone_width: f64,
|
||||
/// Minimum count of shared non-proper-noun content tokens for a merge.
|
||||
pub corroboration_min: usize,
|
||||
/// Weight of a proper-noun token relative to a common token.
|
||||
pub proper_noun_weight: f64,
|
||||
}
|
||||
|
||||
impl Default for SimParams {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
threshold: 0.45,
|
||||
gray_zone_width: 0.15,
|
||||
corroboration_min: 2,
|
||||
proper_noun_weight: 2.0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Verdict on whether two items describe the same story.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct MatchVerdict {
|
||||
/// Weighted containment score, `0.0..=1.0`.
|
||||
pub score: f64,
|
||||
/// Merge into one cluster without further adjudication.
|
||||
pub merge: bool,
|
||||
/// Not a merge, but close enough that LLM consolidation (WS2) should
|
||||
/// adjudicate; when no LLM runs, the pair stays split.
|
||||
pub llm_candidate: bool,
|
||||
}
|
||||
|
||||
/// One side's token pool: title tokens plus the first
|
||||
/// [`SUMMARY_TOKEN_LIMIT`] summary tokens, with the proper-noun subset.
|
||||
#[derive(Debug)]
|
||||
struct Side {
|
||||
pool: HashSet<String>,
|
||||
proper: HashSet<String>,
|
||||
}
|
||||
|
||||
impl Side {
|
||||
fn new(title: &str, summary: &str) -> Self {
|
||||
let mut pool: HashSet<String> = tokenize(title).into_iter().collect();
|
||||
pool.extend(tokenize(summary).into_iter().take(SUMMARY_TOKEN_LIMIT));
|
||||
Self {
|
||||
pool,
|
||||
proper: proper_tokens_from_title(title),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Compares two items under `p` and returns the match verdict.
|
||||
pub fn compare(a: &ItemText, b: &ItemText, p: &SimParams) -> MatchVerdict {
|
||||
let a_side = Side::new(&a.title, &a.summary);
|
||||
let b_side = Side::new(&b.title, &b.summary);
|
||||
|
||||
let score = weighted_containment(&a_side, &b_side, p.proper_noun_weight);
|
||||
let shared_non_proper = a_side
|
||||
.pool
|
||||
.intersection(&b_side.pool)
|
||||
.filter(|token| !a_side.proper.contains(*token) && !b_side.proper.contains(*token))
|
||||
.count();
|
||||
|
||||
// Merge rule v2 (user-directed, two correction rounds): corroboration is
|
||||
// the gate, proper nouns are optional weighted evidence. "Donald Trump
|
||||
// does X" vs "Donald Trump does Y" shares only {does} -> never merges;
|
||||
// "scientists clone first human" variants share {scientists, clone,
|
||||
// first, human} -> merge with zero proper nouns.
|
||||
let merge = score >= p.threshold && shared_non_proper >= p.corroboration_min;
|
||||
let llm_candidate = !merge
|
||||
&& (score >= p.threshold - p.gray_zone_width
|
||||
|| (score >= p.threshold && shared_non_proper >= 1));
|
||||
|
||||
MatchVerdict {
|
||||
score,
|
||||
merge,
|
||||
llm_candidate,
|
||||
}
|
||||
}
|
||||
|
||||
/// Weighted containment of the smaller pool in the larger pool:
|
||||
/// `sum(w(shared)) / sum(w(smaller pool))`, capped at `1.0`. Proper-noun
|
||||
/// weights come from the smaller side, which keeps the ratio bounded.
|
||||
fn weighted_containment(a: &Side, b: &Side, proper_weight: f64) -> f64 {
|
||||
let (small, large) = if a.pool.len() <= b.pool.len() {
|
||||
(a, b)
|
||||
} else {
|
||||
(b, a)
|
||||
};
|
||||
let weight = |token: &String| {
|
||||
if small.proper.contains(token) {
|
||||
proper_weight
|
||||
} else {
|
||||
1.0
|
||||
}
|
||||
};
|
||||
let denominator: f64 = small.pool.iter().map(&weight).sum();
|
||||
if denominator <= 0.0 {
|
||||
return 0.0;
|
||||
}
|
||||
|
||||
let intersection = a_tokens.intersection(&b_tokens).count();
|
||||
let union = a_tokens.union(&b_tokens).count();
|
||||
|
||||
intersection as f64 / union as f64
|
||||
let shared: f64 = small
|
||||
.pool
|
||||
.iter()
|
||||
.filter(|token| large.pool.contains(*token))
|
||||
.map(&weight)
|
||||
.sum();
|
||||
(shared / denominator).min(1.0)
|
||||
}
|
||||
|
||||
/// Returns `true` when `sim` meets or exceeds the caller-supplied threshold.
|
||||
pub fn should_cluster(sim: f64, threshold: f64) -> bool {
|
||||
sim >= threshold
|
||||
}
|
||||
|
||||
/// Tokenises `text` using the same rules as title-similarity clustering:
|
||||
/// split on non-alphanumeric characters, lower-case each token, drop empty
|
||||
/// tokens, and drop the small built-in stopword set.
|
||||
/// Tokenises `text`: split on non-alphanumeric characters, lower-case each
|
||||
/// token, drop empty tokens and the built-in stopword set. Numerals are
|
||||
/// kept as tokens (`91`, `24`).
|
||||
pub fn tokenize(text: &str) -> Vec<String> {
|
||||
text.split(|c: char| !c.is_alphanumeric())
|
||||
.map(str::to_lowercase)
|
||||
@@ -49,6 +153,44 @@ pub fn tokenize(text: &str) -> Vec<String> {
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Detects proper nouns from the original title casing: mid-title
|
||||
/// capitalized tokens that are not stopwords. The first word is ambiguous
|
||||
/// between headline-case and sentence-case, so it only counts as proper
|
||||
/// when the title opens with a capitalized multi-token sequence ("Donald
|
||||
/// Trump ..."); a lone capitalized first word is treated as grammar, not a
|
||||
/// name ("Scientists clone ...").
|
||||
fn proper_tokens_from_title(title: &str) -> HashSet<String> {
|
||||
let raw: Vec<&str> = title
|
||||
.split(|c: char| !c.is_alphanumeric())
|
||||
.filter(|token| !token.is_empty())
|
||||
.collect();
|
||||
|
||||
let mut proper = HashSet::new();
|
||||
for (index, token) in raw.iter().enumerate() {
|
||||
let Some(first_char) = token.chars().next() else {
|
||||
continue;
|
||||
};
|
||||
if !first_char.is_uppercase() {
|
||||
continue;
|
||||
}
|
||||
let lower = token.to_lowercase();
|
||||
if is_stopword(&lower) {
|
||||
continue;
|
||||
}
|
||||
if index == 0 {
|
||||
let opens_name = raw
|
||||
.get(1)
|
||||
.and_then(|next| next.chars().next())
|
||||
.is_some_and(|c| c.is_uppercase());
|
||||
if !opens_name {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
proper.insert(lower);
|
||||
}
|
||||
proper
|
||||
}
|
||||
|
||||
fn is_stopword(token: &str) -> bool {
|
||||
STOPWORDS.contains(&token)
|
||||
}
|
||||
@@ -56,54 +198,86 @@ fn is_stopword(token: &str) -> bool {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use chrono::Utc;
|
||||
use uuid::Uuid;
|
||||
|
||||
fn item(title: &str) -> RawItem {
|
||||
RawItem {
|
||||
id: Uuid::new_v4(),
|
||||
source_id: Uuid::new_v4(),
|
||||
fn text(title: &str) -> ItemText {
|
||||
ItemText {
|
||||
title: title.to_string(),
|
||||
summary: "summary".to_string(),
|
||||
link: "https://example.test/story".to_string(),
|
||||
published_at: Utc::now(),
|
||||
kind: crate::ItemKind::News,
|
||||
summary: String::new(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn identical_titles_have_similarity_one() {
|
||||
let a = item("Storm system moves toward the coast");
|
||||
let b = item("Storm system moves toward the coast");
|
||||
fn steinem_paraphrases_merge() -> Result<(), Box<dyn std::error::Error>> {
|
||||
// Shared common tokens: dies, 91. "Steinem" is proper in the first
|
||||
// title but optional weighted evidence, never required.
|
||||
let a = text("Gloria Steinem dies aged 91");
|
||||
let b = text("Steinem, feminist icon, dies at 91");
|
||||
|
||||
assert_eq!(similarity(&a, &b), 1.0);
|
||||
let verdict = compare(&a, &b, &SimParams::default());
|
||||
|
||||
assert!(verdict.merge, "expected merge, got {verdict:?}");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disjoint_titles_have_similarity_zero() {
|
||||
let a = item("red blue green");
|
||||
let b = item("one two three");
|
||||
fn trump_different_stories_split() -> Result<(), Box<dyn std::error::Error>> {
|
||||
// Shared tokens are only the entity {donald, trump}: zero common
|
||||
// corroboration, so the pair must split even though the score is
|
||||
// at the threshold.
|
||||
let a = text("Donald Trump announces new tariffs on imports");
|
||||
let b = text("Donald Trump endorses House candidate in Ohio");
|
||||
|
||||
assert_eq!(similarity(&a, &b), 0.0);
|
||||
let verdict = compare(&a, &b, &SimParams::default());
|
||||
|
||||
assert!(!verdict.merge, "expected split, got {verdict:?}");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stopword_difference_keeps_syndication_titles_high() {
|
||||
let a = item("Storm system moves toward the coast");
|
||||
let b = item("Storm system moves toward coast");
|
||||
fn no_proper_noun_merge() -> Result<(), Box<dyn std::error::Error>> {
|
||||
// Both titles are sentence-case, so no token is a proper noun; the
|
||||
// pair merges on common-word strength alone.
|
||||
let a = text("Scientists clone first human embryo");
|
||||
let b = text("First human clone created by scientists");
|
||||
|
||||
let sim = similarity(&a, &b);
|
||||
let verdict = compare(&a, &b, &SimParams::default());
|
||||
|
||||
assert!(verdict.merge, "expected merge, got {verdict:?}");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn entity_only_high_score_blocked() -> Result<(), Box<dyn std::error::Error>> {
|
||||
// High score but shared common tokens = {does} only (1 < 2):
|
||||
// entity-only overlap is never sufficient evidence, so the pair is
|
||||
// blocked from merging and routed to LLM adjudication.
|
||||
let a = text("Donald Trump does X");
|
||||
let b = text("Donald Trump does Y");
|
||||
let params = SimParams::default();
|
||||
|
||||
let verdict = compare(&a, &b, ¶ms);
|
||||
|
||||
assert!(
|
||||
sim > 0.8,
|
||||
"expected similarity > 0.8 for syndication titles, got {sim}"
|
||||
verdict.score >= params.threshold,
|
||||
"expected score at/above threshold, got {verdict:?}"
|
||||
);
|
||||
assert!(!verdict.merge, "expected split, got {verdict:?}");
|
||||
assert!(
|
||||
verdict.llm_candidate,
|
||||
"expected LLM candidate, got {verdict:?}"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn should_cluster_respects_threshold() {
|
||||
assert!(should_cluster(0.75, 0.5));
|
||||
assert!(should_cluster(0.5, 0.5));
|
||||
assert!(!should_cluster(0.49, 0.5));
|
||||
fn numerals_corroborate() -> Result<(), Box<dyn std::error::Error>> {
|
||||
// Numerals are content tokens: {24} counts toward corroboration.
|
||||
let a = text("At least 24 killed in crash");
|
||||
let b = text("24 killed in highway crash");
|
||||
|
||||
let verdict = compare(&a, &b, &SimParams::default());
|
||||
|
||||
assert!(verdict.merge, "expected merge, got {verdict:?}");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
//! Build script: bakes build metadata into the binary as compile-time
|
||||
//! environment variables, which the `/api/version` handler reads via `env!`.
|
||||
//!
|
||||
//! Values come from build-time env (`NEWS_BUILD_VERSION` / `NEWS_BUILD_SHA` /
|
||||
//! `NEWS_BUILD_TIME`) when present — the Dockerfile server stage sets the
|
||||
//! first two from CI — and fall back to `dev` / `unknown` / the current UTC
|
||||
//! timestamp, so a plain `cargo build` on a dev machine still yields a
|
||||
//! truthful (if unglamorous) `/api/version`.
|
||||
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
fn main() {
|
||||
println!("cargo:rerun-if-env-changed=NEWS_BUILD_VERSION");
|
||||
println!("cargo:rerun-if-env-changed=NEWS_BUILD_SHA");
|
||||
println!("cargo:rerun-if-env-changed=NEWS_BUILD_TIME");
|
||||
|
||||
let version = std::env::var("NEWS_BUILD_VERSION").unwrap_or_else(|_| "dev".into());
|
||||
let sha = std::env::var("NEWS_BUILD_SHA").unwrap_or_else(|_| "unknown".into());
|
||||
let time = std::env::var("NEWS_BUILD_TIME").unwrap_or_else(|_| rfc3339_now());
|
||||
|
||||
println!("cargo:rustc-env=NEWS_BUILD_VERSION={version}");
|
||||
println!("cargo:rustc-env=NEWS_BUILD_SHA={sha}");
|
||||
println!("cargo:rustc-env=NEWS_BUILD_TIME={time}");
|
||||
}
|
||||
|
||||
/// Current UTC time as an RFC3339 timestamp, std-only (chrono stays out of
|
||||
/// the build graph). Clock before 1970 degrades to the epoch rather than
|
||||
/// failing the build.
|
||||
fn rfc3339_now() -> String {
|
||||
let secs = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map(|d| d.as_secs())
|
||||
.unwrap_or(0);
|
||||
let (year, month, day) = civil_from_days((secs / 86_400) as i64);
|
||||
let rem = secs % 86_400;
|
||||
format!(
|
||||
"{year:04}-{month:02}-{day:02}T{:02}:{:02}:{:02}Z",
|
||||
rem / 3_600,
|
||||
(rem % 3_600) / 60,
|
||||
rem % 60
|
||||
)
|
||||
}
|
||||
|
||||
/// Inverse of days-from-civil (Hinnant 2017): days since the Unix epoch to a
|
||||
/// proleptic-Gregorian (year, month, day). Inputs are non-negative here.
|
||||
fn civil_from_days(days: i64) -> (i64, u32, u32) {
|
||||
let z = days + 719_468;
|
||||
let era = if z >= 0 { z } else { z - 146_096 } / 146_097;
|
||||
let doe = (z - era * 146_097) as u64;
|
||||
let yoe = (doe - doe / 1_460 + doe / 36_524 - doe / 146_096) / 365;
|
||||
let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
|
||||
let mp = (5 * doy + 2) / 153;
|
||||
let day = (doy - (153 * mp + 2) / 5 + 1) as u32;
|
||||
let month = if mp < 10 { mp + 3 } else { mp - 9 } as u32;
|
||||
let year = yoe as i64 + era * 400 + i64::from(month <= 2);
|
||||
(year, month, day)
|
||||
}
|
||||
@@ -33,7 +33,7 @@ mod tests {
|
||||
use tower::ServiceExt;
|
||||
|
||||
use crate::ServerError;
|
||||
use crate::config::{NotifySection, SourceConfig, TopicsSection};
|
||||
use crate::config::{ClusteringSection, NotifySection, SourceConfig, TopicsSection};
|
||||
|
||||
fn test_config() -> NewsConfig {
|
||||
NewsConfig {
|
||||
@@ -61,6 +61,7 @@ mod tests {
|
||||
interests: vec!["science".into()],
|
||||
blocklist: vec!["celebrity".into()],
|
||||
},
|
||||
clustering: ClusteringSection::default(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -122,7 +122,7 @@ mod tests {
|
||||
use axum::http::Request;
|
||||
use chrono::Utc;
|
||||
use news_core::{ItemKind, RawItem, Source, SourceKind};
|
||||
use news_store::{ClusterRepo, ItemRepo, SourceRepo, init_db};
|
||||
use news_store::{AssignOutcome, ClusterParams, ClusterRepo, ItemRepo, SourceRepo, init_db};
|
||||
use serde_json::json;
|
||||
use tower::ServiceExt;
|
||||
|
||||
@@ -158,10 +158,18 @@ mod tests {
|
||||
SourceRepo::new(pool.clone()).upsert(&source).await?;
|
||||
let item = test_item(source.id);
|
||||
ItemRepo::new(pool.clone()).insert_if_new(&item).await?;
|
||||
let cluster = ClusterRepo::new(pool.clone())
|
||||
.assign_or_create(&item, 0.5)
|
||||
let outcome = ClusterRepo::new(pool.clone())
|
||||
.assign_or_create(&item, &ClusterParams::default())
|
||||
.await?;
|
||||
Ok((pool, cluster.id))
|
||||
let cluster_id = match outcome {
|
||||
AssignOutcome::Assigned(id) | AssignOutcome::Created(id) => id,
|
||||
AssignOutcome::LlmCandidates(_) => {
|
||||
return Err(ServerError::DateTime(
|
||||
"seed item unexpectedly landed in the gray zone".into(),
|
||||
));
|
||||
}
|
||||
};
|
||||
Ok((pool, cluster_id))
|
||||
}
|
||||
|
||||
fn build_request(json_body: &Value) -> Result<Request<Body>, ServerError> {
|
||||
|
||||
@@ -1,5 +1,70 @@
|
||||
//! HTTP API routes.
|
||||
//!
|
||||
//! The stateless `/api/version` endpoint lives here; the endpoints that need
|
||||
//! state get their own module (`config`, `feedback`, `stories`).
|
||||
|
||||
pub mod config;
|
||||
pub mod feedback;
|
||||
pub mod stories;
|
||||
|
||||
use axum::{Json, Router, response::IntoResponse, routing::get};
|
||||
|
||||
/// Router for the endpoints that carry no state. Merged into the app beside
|
||||
/// the per-module routers in `main.rs`.
|
||||
pub fn router() -> Router {
|
||||
Router::new().route("/api/version", get(get_version))
|
||||
}
|
||||
|
||||
/// GET /api/version: build metadata for the web footer (`v{version} · {sha}`).
|
||||
/// Values are baked in at compile time by `build.rs` — env vars set at
|
||||
/// runtime have no effect.
|
||||
async fn get_version() -> impl IntoResponse {
|
||||
Json(serde_json::json!({
|
||||
"version": env!("NEWS_BUILD_VERSION"),
|
||||
"git_sha": env!("NEWS_BUILD_SHA"),
|
||||
"build_time": env!("NEWS_BUILD_TIME"),
|
||||
}))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use axum::body::Body;
|
||||
use axum::http::{Request, StatusCode};
|
||||
use tower::ServiceExt;
|
||||
|
||||
use crate::ServerError;
|
||||
|
||||
/// Compile-time proof that `build.rs` populated the three `env!`s and
|
||||
/// that the route serves them: if any `env!` were missing, this crate
|
||||
/// would not compile; if the route were miswired, the request would fail.
|
||||
#[tokio::test]
|
||||
async fn version_reports_all_build_fields() -> Result<(), ServerError> {
|
||||
let request = Request::builder()
|
||||
.method("GET")
|
||||
.uri("/api/version")
|
||||
.body(Body::empty())
|
||||
.map_err(|e| ServerError::DateTime(e.to_string()))?;
|
||||
|
||||
let response = match router().oneshot(request).await {
|
||||
Ok(res) => res,
|
||||
Err(infallible) => match infallible {},
|
||||
};
|
||||
|
||||
assert_eq!(response.status(), StatusCode::OK);
|
||||
let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)
|
||||
.await
|
||||
.map_err(|e| ServerError::DateTime(e.to_string()))?;
|
||||
let body: serde_json::Value = serde_json::from_slice(&bytes)?;
|
||||
|
||||
for field in ["version", "git_sha", "build_time"] {
|
||||
let value = body
|
||||
.get(field)
|
||||
.and_then(|v| v.as_str())
|
||||
.ok_or_else(|| ServerError::DateTime(format!("missing field: {field}")))?;
|
||||
assert!(!value.is_empty(), "{field} must be non-empty");
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@ use axum::body::Body;
|
||||
use axum::http::{Request, StatusCode};
|
||||
use chrono::Utc;
|
||||
use news_core::{ItemKind, RawItem, Source, SourceKind, StoryCluster};
|
||||
use news_store::{ClusterRepo, ItemRepo, SourceRepo, init_db};
|
||||
use news_store::{AssignOutcome, ClusterParams, ClusterRepo, ItemRepo, SourceRepo, init_db};
|
||||
use serde_json::Value;
|
||||
use tokio::sync::{Mutex, RwLock};
|
||||
use tower::ServiceExt;
|
||||
@@ -12,7 +12,7 @@ use uuid::Uuid;
|
||||
|
||||
use crate::ServerError;
|
||||
use crate::api::stories::router;
|
||||
use crate::config::{NewsConfig, NotifySection, SourceConfig, TopicsSection};
|
||||
use crate::config::{ClusteringSection, NewsConfig, NotifySection, SourceConfig, TopicsSection};
|
||||
use crate::gate::{NotificationGate, NotifyConfig, TokenBucket};
|
||||
|
||||
fn test_config() -> NewsConfig {
|
||||
@@ -41,6 +41,7 @@ fn test_config() -> NewsConfig {
|
||||
interests: vec!["science".into()],
|
||||
blocklist: vec!["celebrity".into()],
|
||||
},
|
||||
clustering: ClusteringSection::default(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -86,9 +87,22 @@ async fn seeded_pool() -> Result<(sqlx::SqlitePool, StoryCluster), ServerError>
|
||||
SourceRepo::new(pool.clone()).upsert(&source).await?;
|
||||
let item = test_item(source.id);
|
||||
ItemRepo::new(pool.clone()).insert_if_new(&item).await?;
|
||||
let cluster = ClusterRepo::new(pool.clone())
|
||||
.assign_or_create(&item, 0.95)
|
||||
let clusters = ClusterRepo::new(pool.clone());
|
||||
let outcome = clusters
|
||||
.assign_or_create(&item, &ClusterParams::default())
|
||||
.await?;
|
||||
let cluster_id = match outcome {
|
||||
AssignOutcome::Assigned(id) | AssignOutcome::Created(id) => id,
|
||||
AssignOutcome::LlmCandidates(_) => {
|
||||
return Err(ServerError::DateTime(
|
||||
"seed item unexpectedly landed in the gray zone".into(),
|
||||
));
|
||||
}
|
||||
};
|
||||
let cluster = clusters
|
||||
.get(cluster_id)
|
||||
.await?
|
||||
.ok_or_else(|| ServerError::DateTime("cluster missing after assignment".into()))?;
|
||||
Ok((pool, cluster))
|
||||
}
|
||||
|
||||
@@ -264,7 +278,7 @@ async fn blocklisted_story_serializes_relevance_as_null() -> Result<(), ServerEr
|
||||
let item = blocklisted_item(source.id);
|
||||
ItemRepo::new(pool.clone()).insert_if_new(&item).await?;
|
||||
ClusterRepo::new(pool.clone())
|
||||
.assign_or_create(&item, 0.95)
|
||||
.assign_or_create(&item, &ClusterParams::default())
|
||||
.await?;
|
||||
|
||||
let config = Arc::new(RwLock::new(test_config()));
|
||||
|
||||
@@ -20,6 +20,10 @@ pub struct NewsConfig {
|
||||
pub notify: NotifySection,
|
||||
/// Topic filters.
|
||||
pub topics: TopicsSection,
|
||||
/// Lexical clustering tunables; absent sections fall back to defaults
|
||||
/// so pre-v0.2 config files keep parsing unchanged.
|
||||
#[serde(default)]
|
||||
pub clustering: ClusteringSection,
|
||||
}
|
||||
|
||||
/// A single feed source.
|
||||
@@ -93,6 +97,75 @@ fn default_timezone() -> String {
|
||||
"America/Louisville".to_string()
|
||||
}
|
||||
|
||||
/// Lexical-clustering tunables used by the scheduler's assign step.
|
||||
#[derive(Debug, Clone, serde::Deserialize, serde::Serialize, PartialEq)]
|
||||
pub struct ClusteringSection {
|
||||
/// Score at/above which an item can merge into a cluster (still subject
|
||||
/// to the shared-token corroboration gate).
|
||||
#[serde(default = "default_similarity_threshold")]
|
||||
pub similarity_threshold: f64,
|
||||
/// How far back a cluster stays warm for new items, in hours.
|
||||
#[serde(default = "default_window_hours")]
|
||||
pub window_hours: i64,
|
||||
/// Width of the below-threshold band that routes to LLM adjudication
|
||||
/// instead of an outright split.
|
||||
#[serde(default = "default_gray_zone_width")]
|
||||
pub gray_zone_width: f64,
|
||||
/// Minimum count of shared non-proper-noun content tokens for a merge.
|
||||
#[serde(default = "default_corroboration_min")]
|
||||
pub corroboration_min: usize,
|
||||
/// Weight of a mid-title capitalized token relative to a common token.
|
||||
#[serde(default = "default_proper_noun_weight")]
|
||||
pub proper_noun_weight: f64,
|
||||
}
|
||||
|
||||
impl Default for ClusteringSection {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
similarity_threshold: default_similarity_threshold(),
|
||||
window_hours: default_window_hours(),
|
||||
gray_zone_width: default_gray_zone_width(),
|
||||
corroboration_min: default_corroboration_min(),
|
||||
proper_noun_weight: default_proper_noun_weight(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn default_similarity_threshold() -> f64 {
|
||||
0.45
|
||||
}
|
||||
|
||||
fn default_window_hours() -> i64 {
|
||||
48
|
||||
}
|
||||
|
||||
fn default_gray_zone_width() -> f64 {
|
||||
0.15
|
||||
}
|
||||
|
||||
fn default_corroboration_min() -> usize {
|
||||
2
|
||||
}
|
||||
|
||||
fn default_proper_noun_weight() -> f64 {
|
||||
2.0
|
||||
}
|
||||
|
||||
impl ClusteringSection {
|
||||
/// Builds the store-layer clustering parameters from this section.
|
||||
pub fn cluster_params(&self) -> news_store::ClusterParams {
|
||||
news_store::ClusterParams {
|
||||
sim: news_core::clustering::SimParams {
|
||||
threshold: self.similarity_threshold,
|
||||
gray_zone_width: self.gray_zone_width,
|
||||
corroboration_min: self.corroboration_min,
|
||||
proper_noun_weight: self.proper_noun_weight,
|
||||
},
|
||||
window_hours: self.window_hours,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Operator-controlled topic filters.
|
||||
#[derive(Debug, Clone, serde::Deserialize, serde::Serialize, PartialEq)]
|
||||
pub struct TopicsSection {
|
||||
@@ -251,6 +324,34 @@ blocklist = ["royal", "celebrity"]
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn clustering_fields_default_and_parse() -> Result<(), ServerError> {
|
||||
let dir = tempfile::tempdir()?;
|
||||
let path = dir.path().join("config.toml");
|
||||
std::fs::write(&path, valid_config_toml())?;
|
||||
|
||||
let config = load(&path)?;
|
||||
assert_eq!(config.clustering.similarity_threshold, 0.45);
|
||||
assert_eq!(config.clustering.window_hours, 48);
|
||||
assert_eq!(config.clustering.gray_zone_width, 0.15);
|
||||
assert_eq!(config.clustering.corroboration_min, 2);
|
||||
assert_eq!(config.clustering.proper_noun_weight, 2.0);
|
||||
|
||||
let with_explicit = format!(
|
||||
"{}\n[clustering]\nsimilarity_threshold = 0.5\nwindow_hours = 24\n\
|
||||
gray_zone_width = 0.1\ncorroboration_min = 3\nproper_noun_weight = 3.0",
|
||||
valid_config_toml()
|
||||
);
|
||||
std::fs::write(&path, with_explicit)?;
|
||||
let config = load(&path)?;
|
||||
assert_eq!(config.clustering.similarity_threshold, 0.5);
|
||||
assert_eq!(config.clustering.window_hours, 24);
|
||||
assert_eq!(config.clustering.gray_zone_width, 0.1);
|
||||
assert_eq!(config.clustering.corroboration_min, 3);
|
||||
assert_eq!(config.clustering.proper_noun_weight, 3.0);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[traced_test]
|
||||
#[tokio::test]
|
||||
async fn bad_reload_keeps_last_good_config() -> Result<(), ServerError> {
|
||||
|
||||
@@ -711,7 +711,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn from_config_maps_notify_section_and_rejects_bad_inputs() -> Result<(), ServerError> {
|
||||
use crate::config::{NewsConfig, NotifySection, TopicsSection};
|
||||
use crate::config::{ClusteringSection, NewsConfig, NotifySection, TopicsSection};
|
||||
|
||||
let config = NewsConfig {
|
||||
sources: Vec::new(),
|
||||
@@ -732,6 +732,7 @@ mod tests {
|
||||
interests: Vec::new(),
|
||||
blocklist: Vec::new(),
|
||||
},
|
||||
clustering: ClusteringSection::default(),
|
||||
};
|
||||
|
||||
let mapped = NotifyConfig::from_config(&config)?;
|
||||
|
||||
@@ -89,6 +89,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
Arc::clone(&config),
|
||||
))
|
||||
.merge(news_server::api::config::router(Arc::clone(&config)))
|
||||
.merge(news_server::api::router())
|
||||
.merge(news_server::health::router(rt.pool, rt.metrics));
|
||||
|
||||
let port: u16 = std::env::var("NEWS_PORT")
|
||||
@@ -109,6 +110,7 @@ async fn run_once(
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
use news_server::scheduler::poll_source_once;
|
||||
|
||||
let clustering = config.clustering.cluster_params();
|
||||
let mut title_width = 40_usize;
|
||||
for source in &rt.sources {
|
||||
if let Err(err) = poll_source_once(
|
||||
@@ -121,6 +123,7 @@ async fn run_once(
|
||||
&rt.gate,
|
||||
ntfy,
|
||||
&rt.metrics,
|
||||
&clustering,
|
||||
dry_run,
|
||||
&mut title_width,
|
||||
)
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
use chrono::{Duration, TimeZone, Utc};
|
||||
use chrono_tz::America::Louisville;
|
||||
use news_core::{ItemKind, RawItem, Source, SourceKind};
|
||||
use news_store::{ClusterRepo, ItemRepo, MetricKind, PercentileTracker, SourceRepo, init_db};
|
||||
use news_store::{
|
||||
ClusterParams, ClusterRepo, ItemRepo, MetricKind, PercentileTracker, SourceRepo, init_db,
|
||||
};
|
||||
use uuid::Uuid;
|
||||
use wiremock::matchers::{header, method, path};
|
||||
use wiremock::{Mock, MockServer, Request, ResponseTemplate};
|
||||
@@ -45,6 +47,7 @@ fn test_config(notify_opinions: bool) -> NewsConfig {
|
||||
interests: Vec::new(),
|
||||
blocklist: Vec::new(),
|
||||
},
|
||||
clustering: crate::config::ClusteringSection::default(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -71,7 +74,11 @@ fn test_item(
|
||||
id: Uuid::new_v4(),
|
||||
source_id,
|
||||
title: title.into(),
|
||||
summary: "summary".into(),
|
||||
// Reusing the title keeps each seed's payload lexically distinct:
|
||||
// a shared generic summary would push near-duplicate seeds into
|
||||
// the clusterer's merge or gray-zone bands instead of creating
|
||||
// one cluster per seed.
|
||||
summary: title.into(),
|
||||
link: link.into(),
|
||||
published_at,
|
||||
kind,
|
||||
@@ -126,7 +133,7 @@ async fn seed_cluster(
|
||||
let item = test_item(source_id, title, link, published_at, kind);
|
||||
ItemRepo::new(pool.clone()).insert_if_new(&item).await?;
|
||||
ClusterRepo::new(pool.clone())
|
||||
.assign_or_create(&item, 0.5)
|
||||
.assign_or_create(&item, &ClusterParams::default())
|
||||
.await?;
|
||||
Ok(item)
|
||||
}
|
||||
@@ -179,13 +186,14 @@ async fn sends_digest_with_in_band_stories() -> Result<(), NotifyError> {
|
||||
let base = now.with_timezone(&Utc) - Duration::hours(1);
|
||||
|
||||
let mut items = Vec::new();
|
||||
for i in 0..3 {
|
||||
let titles = ["Alpha bulletin", "Bravo dispatch", "Charlie ledger"];
|
||||
for (i, title) in titles.iter().enumerate() {
|
||||
let item = seed_cluster(
|
||||
&pool,
|
||||
source.id,
|
||||
&format!("Story {}", i),
|
||||
title,
|
||||
&format!("https://example.test/{}", i),
|
||||
base + Duration::minutes(i),
|
||||
base + Duration::minutes(i as i64),
|
||||
ItemKind::News,
|
||||
)
|
||||
.await?;
|
||||
|
||||
@@ -4,7 +4,7 @@ pub mod cycle;
|
||||
pub mod loops;
|
||||
pub mod metrics;
|
||||
|
||||
pub use cycle::{PollCycleError, poll_source_once};
|
||||
pub use cycle::{PendingConsolidation, PollCycleError, poll_source_once};
|
||||
pub use loops::{
|
||||
PruneReport, per_source_interval, run_pruning_once, spawn_digest_loop, spawn_gate_sync_loop,
|
||||
spawn_pruning_loop, spawn_source_loop,
|
||||
|
||||
@@ -2,9 +2,12 @@ use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use chrono::Utc;
|
||||
use news_core::clustering::MatchVerdict;
|
||||
use news_core::{RawItem, Source, StoryCluster};
|
||||
use news_ingest::{FeedPoller, PollResult};
|
||||
use news_store::{ClusterRepo, ItemRepo, MetricKind, PercentileTracker, SourceRepo};
|
||||
use news_store::{
|
||||
AssignOutcome, ClusterParams, ClusterRepo, ItemRepo, MetricKind, PercentileTracker, SourceRepo,
|
||||
};
|
||||
use sqlx::SqlitePool;
|
||||
use tokio::sync::Mutex;
|
||||
use tokio::time::Instant;
|
||||
@@ -67,6 +70,18 @@ impl From<crate::ServerError> for PollCycleError {
|
||||
}
|
||||
}
|
||||
|
||||
/// One item's gray-zone clustering outcome, awaiting LLM adjudication
|
||||
/// (WS2): the item plus every candidate cluster whose best member verdict
|
||||
/// was `llm_candidate`. Until the LLM layer runs, gray-zone pairs stay
|
||||
/// split (fail-open).
|
||||
#[derive(Debug)]
|
||||
pub struct PendingConsolidation {
|
||||
/// The newly inserted item that matched nothing outright.
|
||||
pub item: RawItem,
|
||||
/// Candidate cluster ids with their best member verdicts.
|
||||
pub candidates: Vec<(Uuid, MatchVerdict)>,
|
||||
}
|
||||
|
||||
/// Polls one source, inserts items, clusters them, scores them, evaluates the
|
||||
/// gate, and either prints the decision (dry-run) or publishes real
|
||||
/// notifications. The gate mutex is held only during each evaluate call —
|
||||
@@ -85,6 +100,7 @@ pub async fn poll_source_once(
|
||||
gate: &Arc<Mutex<NotificationGate>>,
|
||||
ntfy: Option<&NtfyPublisher>,
|
||||
metrics: &Metrics,
|
||||
clustering: &ClusterParams,
|
||||
dry_run: bool,
|
||||
title_width: &mut usize,
|
||||
) -> Result<(), PollCycleError> {
|
||||
@@ -122,15 +138,38 @@ pub async fn poll_source_once(
|
||||
|
||||
let mut item_by_id: HashMap<Uuid, RawItem> = HashMap::new();
|
||||
let mut seen_clusters: HashMap<Uuid, StoryCluster> = HashMap::new();
|
||||
let mut pending_consolidations: Vec<PendingConsolidation> = Vec::new();
|
||||
|
||||
for item in items {
|
||||
match items_repo.insert_if_new(&item).await? {
|
||||
news_store::InsertOutcome::AlreadyExists => continue,
|
||||
news_store::InsertOutcome::Inserted => {}
|
||||
}
|
||||
let cluster = clusters_repo.assign_or_create(&item, 0.5).await?;
|
||||
item_by_id.insert(item.id, item);
|
||||
seen_clusters.insert(cluster.id, cluster);
|
||||
match clusters_repo.assign_or_create(&item, clustering).await? {
|
||||
AssignOutcome::Assigned(cluster_id) | AssignOutcome::Created(cluster_id) => {
|
||||
let cluster = clusters_repo.get(cluster_id).await?.ok_or_else(
|
||||
|| -> Box<dyn std::error::Error + Send + Sync> {
|
||||
"cluster missing after assignment".into()
|
||||
},
|
||||
)?;
|
||||
item_by_id.insert(item.id, item);
|
||||
seen_clusters.insert(cluster.id, cluster);
|
||||
}
|
||||
AssignOutcome::LlmCandidates(candidates) => {
|
||||
pending_consolidations.push(PendingConsolidation { item, candidates });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Gray-zone outcomes have no cluster yet: WS2's LLM consolidation will
|
||||
// adjudicate them. Until then they stay split, so log and move on.
|
||||
for pending in &pending_consolidations {
|
||||
tracing::debug!(
|
||||
item_id = %pending.item.id,
|
||||
title = %pending.item.title,
|
||||
candidates = ?pending.candidates,
|
||||
"gray-zone cluster candidates queued for LLM consolidation; staying split"
|
||||
);
|
||||
}
|
||||
|
||||
let mut scored: Vec<(StoryCluster, RawItem, f64, f64)> = Vec::new();
|
||||
|
||||
@@ -42,6 +42,7 @@ pub fn spawn_source_loop(
|
||||
relevance_scorer =
|
||||
refresh_relevance_scorer(&pool, &config_snapshot, relevance_scorer).await;
|
||||
let mut title_width = 40_usize;
|
||||
let clustering = config_snapshot.clustering.cluster_params();
|
||||
|
||||
let result = poll_source_once(
|
||||
&source,
|
||||
@@ -53,6 +54,7 @@ pub fn spawn_source_loop(
|
||||
&gate,
|
||||
ntfy.as_deref(),
|
||||
&metrics,
|
||||
&clustering,
|
||||
false,
|
||||
&mut title_width,
|
||||
)
|
||||
@@ -391,7 +393,7 @@ mod tests {
|
||||
}
|
||||
|
||||
fn empty_topics_config() -> NewsConfig {
|
||||
use crate::config::{NewsConfig, NotifySection, TopicsSection};
|
||||
use crate::config::{ClusteringSection, NewsConfig, NotifySection, TopicsSection};
|
||||
NewsConfig {
|
||||
sources: Vec::new(),
|
||||
notify: NotifySection {
|
||||
@@ -411,6 +413,7 @@ mod tests {
|
||||
interests: Vec::new(),
|
||||
blocklist: Vec::new(),
|
||||
},
|
||||
clustering: ClusteringSection::default(),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -209,7 +209,7 @@ impl ImportanceScorer {
|
||||
#[cfg(test)]
|
||||
use news_core::{ItemKind, Source, SourceKind};
|
||||
#[cfg(test)]
|
||||
use news_store::{ClusterRepo, SourceRepo, init_db};
|
||||
use news_store::{AssignOutcome, ClusterParams, ClusterRepo, SourceRepo, init_db};
|
||||
#[cfg(test)]
|
||||
use pretty_assertions::assert_eq;
|
||||
|
||||
@@ -425,8 +425,24 @@ async fn importance_scorer_matches_direct_formula() -> Result<(), ServerError> {
|
||||
items.insert_if_new(&item_a).await?;
|
||||
items.insert_if_new(&item_b).await?;
|
||||
|
||||
let _cluster = clusters.assign_or_create(&item_a, 0.5).await?;
|
||||
let cluster = clusters.assign_or_create(&item_b, 0.5).await?;
|
||||
let _ = clusters
|
||||
.assign_or_create(&item_a, &ClusterParams::default())
|
||||
.await?;
|
||||
let outcome = clusters
|
||||
.assign_or_create(&item_b, &ClusterParams::default())
|
||||
.await?;
|
||||
let cluster_id = match outcome {
|
||||
AssignOutcome::Assigned(id) | AssignOutcome::Created(id) => id,
|
||||
AssignOutcome::LlmCandidates(_) => {
|
||||
return Err(ServerError::DateTime(
|
||||
"seed item unexpectedly landed in the gray zone".into(),
|
||||
));
|
||||
}
|
||||
};
|
||||
let cluster = clusters
|
||||
.get(cluster_id)
|
||||
.await?
|
||||
.ok_or_else(|| ServerError::DateTime("cluster missing after assignment".into()))?;
|
||||
assert_eq!(cluster.source_count, 2);
|
||||
|
||||
let scorer = ImportanceScorer::new(pool);
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
-- v0.2 headline/link columns for clusters. `llm_headline` holds the
|
||||
-- LLM-written neutral headline (NULL until the consolidation layer writes
|
||||
-- it); `link_item_id` points at the highest-weight source's item for the
|
||||
-- cluster. `raw_items.id` is TEXT (0001_init.sql), so the FK column is
|
||||
-- TEXT; ON DELETE SET NULL keeps the cluster row when a linked item is
|
||||
-- pruned. Existing rows get NULL for both columns (no retro-merge).
|
||||
ALTER TABLE story_clusters ADD COLUMN llm_headline TEXT;
|
||||
ALTER TABLE story_clusters ADD COLUMN link_item_id TEXT
|
||||
REFERENCES raw_items(id) ON DELETE SET NULL;
|
||||
|
||||
-- The window candidate query range-scans story_clusters(last_seen_at);
|
||||
-- 0003 already covers first_seen_at (idx_story_clusters_first_seen_at)
|
||||
-- but last_seen_at had no index.
|
||||
CREATE INDEX idx_story_clusters_last_seen_at
|
||||
ON story_clusters(last_seen_at);
|
||||
+325
-141
@@ -1,78 +1,158 @@
|
||||
//! Story-cluster repository: assigns incoming raw items to an existing
|
||||
//! cluster or starts a new one, using title-token Jaccard similarity and a
|
||||
//! timestamp window anchored to the incoming item.
|
||||
//! cluster or starts a new one, using weighted token-set similarity with a
|
||||
//! shared-token corroboration gate and a warm-cluster window anchored to
|
||||
//! the incoming item's publish time.
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use chrono::{Duration, Utc};
|
||||
use news_core::clustering::{ItemText, MatchVerdict, SimParams};
|
||||
use news_core::{RawItem, StoryCluster};
|
||||
use sqlx::{Row, SqlitePool};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::StoreError;
|
||||
|
||||
/// Per-cluster member cap for candidate comparison: only the most recent
|
||||
/// 20 members of a warm cluster are compared against the incoming item.
|
||||
const MEMBER_CAP: usize = 20;
|
||||
|
||||
/// Late-arrival grace: an item may predate a cluster's first_seen_at by at
|
||||
/// most this many hours and still join it (out-of-order feed delivery).
|
||||
const GRACE_HOURS: i64 = 2;
|
||||
|
||||
/// Repository over the `story_clusters` and `cluster_members` tables.
|
||||
pub struct ClusterRepo(SqlitePool);
|
||||
|
||||
/// Tunables for [`ClusterRepo::assign_or_create`].
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct ClusterParams {
|
||||
/// Similarity-engine parameters (thresholds, corroboration, weights).
|
||||
pub sim: SimParams,
|
||||
/// How far back a cluster stays warm for new items, in hours.
|
||||
pub window_hours: i64,
|
||||
}
|
||||
|
||||
impl Default for ClusterParams {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
sim: SimParams::default(),
|
||||
window_hours: 48,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Outcome of one assignment attempt.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum AssignOutcome {
|
||||
/// The item joined an existing cluster.
|
||||
Assigned(Uuid),
|
||||
/// No candidate merged; a new cluster was started.
|
||||
Created(Uuid),
|
||||
/// Nothing merged, but at least one candidate scored in the gray zone.
|
||||
/// The LLM consolidation layer (WS2) adjudicates these; until then the
|
||||
/// pair stays split (fail-open).
|
||||
LlmCandidates(Vec<(Uuid, MatchVerdict)>),
|
||||
}
|
||||
|
||||
impl ClusterRepo {
|
||||
pub fn new(pool: SqlitePool) -> Self {
|
||||
Self(pool)
|
||||
}
|
||||
|
||||
/// Assigns `item` to the best matching open cluster, or creates a new
|
||||
/// Loads a cluster with its member item ids, or `None` when absent.
|
||||
pub async fn get(&self, cluster_id: Uuid) -> Result<Option<StoryCluster>, StoreError> {
|
||||
load_cluster(&self.0, &cluster_id.to_string()).await
|
||||
}
|
||||
|
||||
/// Assigns `item` to the best matching warm cluster, or creates a new
|
||||
/// cluster if none qualifies.
|
||||
///
|
||||
/// A cluster is considered "open" for the incoming item when its
|
||||
/// `last_seen_at` is at most the item's `published_at` and the gap is
|
||||
/// strictly less than 45 minutes. The window is anchored to the item's
|
||||
/// timestamp, not wall-clock time, so tests are deterministic.
|
||||
/// A cluster is a candidate for the incoming item when its
|
||||
/// `last_seen_at` is within `window_hours` of the item's `published_at`
|
||||
/// and the item is no more than [`GRACE_HOURS`] older than the
|
||||
/// cluster's `first_seen_at` (late arrivals). The window is anchored to
|
||||
/// the item's timestamp, not wall-clock time, so tests are
|
||||
/// deterministic.
|
||||
///
|
||||
/// All writes for a single assignment happen inside one sqlx transaction.
|
||||
/// The item is compared against every (capped) member of each candidate
|
||||
/// cluster — not just the canonical item — and the best member verdict
|
||||
/// stands. All writes for a single assignment happen inside one sqlx
|
||||
/// transaction.
|
||||
pub async fn assign_or_create(
|
||||
&self,
|
||||
item: &RawItem,
|
||||
threshold: f64,
|
||||
) -> Result<StoryCluster, StoreError> {
|
||||
params: &ClusterParams,
|
||||
) -> Result<AssignOutcome, StoreError> {
|
||||
let mut tx = self.0.begin().await?;
|
||||
|
||||
let candidate_rows = sqlx::query(
|
||||
"SELECT \
|
||||
sc.id AS cluster_id, \
|
||||
sc.canonical_item_id, \
|
||||
sc.source_count, \
|
||||
sc.first_seen_at, \
|
||||
sc.last_seen_at, \
|
||||
ri.id AS raw_id, \
|
||||
ri.source_id, \
|
||||
ri.title, \
|
||||
ri.summary, \
|
||||
ri.link, \
|
||||
ri.published_at, \
|
||||
ri.kind \
|
||||
// Window bounds are computed in Rust so the predicates stay plain
|
||||
// string range comparisons on the RFC3339 TEXT columns and can use
|
||||
// the story_clusters(last_seen_at) / (first_seen_at) indexes.
|
||||
let window_floor = (item.published_at - Duration::hours(params.window_hours)).to_rfc3339();
|
||||
let grace_ceiling = (item.published_at + Duration::hours(GRACE_HOURS)).to_rfc3339();
|
||||
|
||||
let rows = sqlx::query(
|
||||
"SELECT sc.id AS cluster_id, ri.title, ri.summary \
|
||||
FROM story_clusters sc \
|
||||
JOIN raw_items ri ON ri.id = sc.canonical_item_id \
|
||||
WHERE sc.last_seen_at <= ?1 \
|
||||
AND (julianday(?1) - julianday(sc.last_seen_at)) < 45.0 / 1440.0",
|
||||
JOIN cluster_members cm ON cm.cluster_id = sc.id \
|
||||
JOIN raw_items ri ON ri.id = cm.item_id \
|
||||
WHERE sc.last_seen_at >= ?1 AND sc.first_seen_at <= ?2 \
|
||||
ORDER BY sc.id, ri.published_at DESC, ri.id DESC",
|
||||
)
|
||||
.bind(item.published_at.to_rfc3339())
|
||||
.bind(&window_floor)
|
||||
.bind(&grace_ceiling)
|
||||
.fetch_all(&mut *tx)
|
||||
.await?;
|
||||
|
||||
let mut best: Option<(String, f64)> = None;
|
||||
for row in &candidate_rows {
|
||||
let canonical = row_to_raw_item(row)?;
|
||||
let sim = news_core::clustering::similarity(item, &canonical);
|
||||
if news_core::clustering::should_cluster(sim, threshold) {
|
||||
let cluster_id: String = row.try_get("cluster_id")?;
|
||||
match best {
|
||||
None => best = Some((cluster_id, sim)),
|
||||
Some((_, best_sim)) if sim > best_sim => {
|
||||
best = Some((cluster_id, sim));
|
||||
let incoming = ItemText {
|
||||
title: item.title.clone(),
|
||||
summary: item.summary.clone(),
|
||||
};
|
||||
|
||||
// Rows are ordered by cluster id, so each cluster's members are
|
||||
// contiguous; a single pass keeps the most recent `MEMBER_CAP`.
|
||||
let mut candidates: Vec<(String, Vec<ItemText>)> = Vec::new();
|
||||
for row in &rows {
|
||||
let cluster_id: String = row.try_get("cluster_id")?;
|
||||
let member = ItemText {
|
||||
title: row.try_get("title")?,
|
||||
summary: row.try_get("summary")?,
|
||||
};
|
||||
match candidates.last_mut() {
|
||||
Some((id, members)) if *id == cluster_id => {
|
||||
if members.len() < MEMBER_CAP {
|
||||
members.push(member);
|
||||
}
|
||||
Some(_) => {}
|
||||
}
|
||||
_ => candidates.push((cluster_id, vec![member])),
|
||||
}
|
||||
}
|
||||
|
||||
let cluster = if let Some((cluster_id, _)) = best {
|
||||
// Best member verdict per cluster; the best merging cluster wins.
|
||||
let mut best_merge: Option<(String, MatchVerdict)> = None;
|
||||
let mut gray_zone: Vec<(String, MatchVerdict)> = Vec::new();
|
||||
for (cluster_id, members) in &candidates {
|
||||
let mut best: Option<MatchVerdict> = None;
|
||||
for member in members {
|
||||
let verdict = news_core::clustering::compare(&incoming, member, ¶ms.sim);
|
||||
if best.as_ref().is_none_or(|best| verdict.score > best.score) {
|
||||
best = Some(verdict);
|
||||
}
|
||||
}
|
||||
let Some(verdict) = best else {
|
||||
continue;
|
||||
};
|
||||
if verdict.merge
|
||||
&& best_merge
|
||||
.as_ref()
|
||||
.is_none_or(|(_, best)| verdict.score > best.score)
|
||||
{
|
||||
best_merge = Some((cluster_id.clone(), verdict));
|
||||
} else if !verdict.merge && verdict.llm_candidate {
|
||||
gray_zone.push((cluster_id.clone(), verdict));
|
||||
}
|
||||
}
|
||||
|
||||
let outcome = if let Some((cluster_id, _)) = best_merge {
|
||||
sqlx::query("INSERT INTO cluster_members (cluster_id, item_id) VALUES (?, ?)")
|
||||
.bind(&cluster_id)
|
||||
.bind(item.id.to_string())
|
||||
@@ -89,6 +169,7 @@ impl ClusterRepo {
|
||||
.fetch_one(&mut *tx)
|
||||
.await?;
|
||||
|
||||
// last_seen_at only ever moves forward.
|
||||
sqlx::query(
|
||||
"UPDATE story_clusters \
|
||||
SET source_count = ?, \
|
||||
@@ -101,10 +182,15 @@ impl ClusterRepo {
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
|
||||
load_cluster(&mut tx, &cluster_id).await?
|
||||
AssignOutcome::Assigned(Uuid::parse_str(&cluster_id)?)
|
||||
} else if !gray_zone.is_empty() {
|
||||
let candidates = gray_zone
|
||||
.into_iter()
|
||||
.map(|(id, verdict)| Ok((Uuid::parse_str(&id)?, verdict)))
|
||||
.collect::<Result<Vec<_>, StoreError>>()?;
|
||||
AssignOutcome::LlmCandidates(candidates)
|
||||
} else {
|
||||
let cluster_id = Uuid::new_v4();
|
||||
let cluster_id_str = cluster_id.to_string();
|
||||
let cluster_id_str = Uuid::new_v4().to_string();
|
||||
|
||||
sqlx::query(
|
||||
"INSERT INTO story_clusters \
|
||||
@@ -125,67 +211,35 @@ impl ClusterRepo {
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
|
||||
StoryCluster {
|
||||
id: cluster_id,
|
||||
canonical_item_id: item.id,
|
||||
member_item_ids: vec![item.id],
|
||||
source_count: 1,
|
||||
first_seen_at: item.published_at,
|
||||
last_seen_at: item.published_at,
|
||||
}
|
||||
AssignOutcome::Created(Uuid::parse_str(&cluster_id_str)?)
|
||||
};
|
||||
|
||||
tx.commit().await?;
|
||||
Ok(cluster)
|
||||
}
|
||||
}
|
||||
|
||||
fn row_to_raw_item(row: &sqlx::sqlite::SqliteRow) -> Result<RawItem, StoreError> {
|
||||
let id: String = row.try_get("raw_id")?;
|
||||
let source_id: String = row.try_get("source_id")?;
|
||||
let kind: String = row.try_get("kind")?;
|
||||
let published_at: String = row.try_get("published_at")?;
|
||||
|
||||
Ok(RawItem {
|
||||
id: Uuid::parse_str(&id)?,
|
||||
source_id: Uuid::parse_str(&source_id)?,
|
||||
title: row.try_get("title")?,
|
||||
summary: row.try_get("summary")?,
|
||||
link: row.try_get("link")?,
|
||||
published_at: DateTime::parse_from_rfc3339(&published_at)
|
||||
.map_err(|e| StoreError::InvalidStoredValue(format!("invalid published_at: {e}")))?
|
||||
.with_timezone(&Utc),
|
||||
kind: kind_from_str(&kind)?,
|
||||
})
|
||||
}
|
||||
|
||||
fn kind_from_str(kind: &str) -> Result<news_core::ItemKind, StoreError> {
|
||||
match kind {
|
||||
"news" => Ok(news_core::ItemKind::News),
|
||||
"opinion" => Ok(news_core::ItemKind::Opinion),
|
||||
other => Err(StoreError::InvalidStoredValue(format!(
|
||||
"unrecognized item kind: {other}"
|
||||
))),
|
||||
Ok(outcome)
|
||||
}
|
||||
}
|
||||
|
||||
async fn load_cluster(
|
||||
tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>,
|
||||
pool: &sqlx::SqlitePool,
|
||||
cluster_id: &str,
|
||||
) -> Result<StoryCluster, StoreError> {
|
||||
) -> Result<Option<StoryCluster>, StoreError> {
|
||||
let row = sqlx::query(
|
||||
"SELECT id, canonical_item_id, source_count, first_seen_at, last_seen_at \
|
||||
FROM story_clusters WHERE id = ?",
|
||||
)
|
||||
.bind(cluster_id)
|
||||
.fetch_one(&mut **tx)
|
||||
.fetch_optional(pool)
|
||||
.await?;
|
||||
|
||||
let Some(row) = row else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
let member_ids: Vec<String> = sqlx::query_scalar(
|
||||
"SELECT item_id FROM cluster_members WHERE cluster_id = ? ORDER BY item_id",
|
||||
)
|
||||
.bind(cluster_id)
|
||||
.fetch_all(&mut **tx)
|
||||
.fetch_all(pool)
|
||||
.await?;
|
||||
|
||||
let id: String = row.try_get("id")?;
|
||||
@@ -194,7 +248,7 @@ async fn load_cluster(
|
||||
let first_seen_at: String = row.try_get("first_seen_at")?;
|
||||
let last_seen_at: String = row.try_get("last_seen_at")?;
|
||||
|
||||
Ok(StoryCluster {
|
||||
Ok(Some(StoryCluster {
|
||||
id: Uuid::parse_str(&id)?,
|
||||
canonical_item_id: Uuid::parse_str(&canonical_item_id)?,
|
||||
member_item_ids: member_ids
|
||||
@@ -208,11 +262,11 @@ async fn load_cluster(
|
||||
})?,
|
||||
first_seen_at: parse_datetime(&first_seen_at)?,
|
||||
last_seen_at: parse_datetime(&last_seen_at)?,
|
||||
})
|
||||
}))
|
||||
}
|
||||
|
||||
fn parse_datetime(value: &str) -> Result<DateTime<Utc>, StoreError> {
|
||||
Ok(DateTime::parse_from_rfc3339(value)
|
||||
fn parse_datetime(value: &str) -> Result<chrono::DateTime<Utc>, StoreError> {
|
||||
Ok(chrono::DateTime::parse_from_rfc3339(value)
|
||||
.map_err(|e| StoreError::InvalidStoredValue(format!("invalid timestamp: {e}")))?
|
||||
.with_timezone(&Utc))
|
||||
}
|
||||
@@ -221,11 +275,13 @@ fn parse_datetime(value: &str) -> Result<DateTime<Utc>, StoreError> {
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::{ItemRepo, SourceRepo, StoreError, init_db};
|
||||
use chrono::{Duration, Utc};
|
||||
use news_core::{ItemKind, RawItem, Source, SourceKind};
|
||||
use chrono::{DateTime, Duration, Utc};
|
||||
use news_core::{ItemKind, Source, SourceKind};
|
||||
use pretty_assertions::assert_eq;
|
||||
|
||||
const THRESHOLD: f64 = 0.5;
|
||||
fn params() -> ClusterParams {
|
||||
ClusterParams::default()
|
||||
}
|
||||
|
||||
fn test_source(name: &str, url: &str) -> Source {
|
||||
Source {
|
||||
@@ -259,6 +315,34 @@ mod tests {
|
||||
Ok((pool, sources, items, clusters))
|
||||
}
|
||||
|
||||
fn require_assigned(outcome: AssignOutcome) -> Result<Uuid, StoreError> {
|
||||
match outcome {
|
||||
AssignOutcome::Assigned(id) => Ok(id),
|
||||
other => Err(StoreError::InvalidStoredValue(format!(
|
||||
"expected Assigned, got {other:?}"
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
fn require_created(outcome: AssignOutcome) -> Result<Uuid, StoreError> {
|
||||
match outcome {
|
||||
AssignOutcome::Created(id) => Ok(id),
|
||||
other => Err(StoreError::InvalidStoredValue(format!(
|
||||
"expected Created, got {other:?}"
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
async fn load_cluster_row(
|
||||
clusters: &ClusterRepo,
|
||||
id: Uuid,
|
||||
) -> Result<StoryCluster, StoreError> {
|
||||
clusters
|
||||
.get(id)
|
||||
.await?
|
||||
.ok_or_else(|| StoreError::InvalidStoredValue("cluster missing after write".into()))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn source_count_is_distinct_sources_not_item_count() -> Result<(), StoreError> {
|
||||
let (_pool, sources, items, clusters) = seeded_repos().await?;
|
||||
@@ -272,8 +356,8 @@ mod tests {
|
||||
Utc::now(),
|
||||
);
|
||||
items.insert_if_new(&item_a1).await?;
|
||||
let cluster1 = clusters.assign_or_create(&item_a1, THRESHOLD).await?;
|
||||
assert_eq!(cluster1.source_count, 1);
|
||||
let cluster1 = require_created(clusters.assign_or_create(&item_a1, ¶ms()).await?)?;
|
||||
assert_eq!(load_cluster_row(&clusters, cluster1).await?.source_count, 1);
|
||||
|
||||
let item_a2 = test_item(
|
||||
source_a.id,
|
||||
@@ -282,9 +366,9 @@ mod tests {
|
||||
Utc::now(),
|
||||
);
|
||||
items.insert_if_new(&item_a2).await?;
|
||||
let cluster2 = clusters.assign_or_create(&item_a2, THRESHOLD).await?;
|
||||
assert_eq!(cluster2.id, cluster1.id);
|
||||
assert_eq!(cluster2.source_count, 1);
|
||||
let cluster2 = require_assigned(clusters.assign_or_create(&item_a2, ¶ms()).await?)?;
|
||||
assert_eq!(cluster2, cluster1);
|
||||
assert_eq!(load_cluster_row(&clusters, cluster2).await?.source_count, 1);
|
||||
|
||||
let source_b = test_source("Source B", "https://b.test");
|
||||
sources.upsert(&source_b).await?;
|
||||
@@ -295,49 +379,16 @@ mod tests {
|
||||
Utc::now(),
|
||||
);
|
||||
items.insert_if_new(&item_b1).await?;
|
||||
let cluster3 = clusters.assign_or_create(&item_b1, THRESHOLD).await?;
|
||||
assert_eq!(cluster3.id, cluster1.id);
|
||||
assert_eq!(cluster3.source_count, 2);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn item_outside_45_minute_window_starts_new_cluster() -> Result<(), StoreError> {
|
||||
let (_pool, sources, items, clusters) = seeded_repos().await?;
|
||||
let source = test_source("Source", "https://example.test");
|
||||
sources.upsert(&source).await?;
|
||||
|
||||
let base = Utc::now();
|
||||
let item1 = test_item(
|
||||
source.id,
|
||||
"Storm system moves toward the coast",
|
||||
"https://example.test/1",
|
||||
base,
|
||||
);
|
||||
items.insert_if_new(&item1).await?;
|
||||
clusters.assign_or_create(&item1, THRESHOLD).await?;
|
||||
|
||||
let item2 = test_item(
|
||||
source.id,
|
||||
"Storm system moves toward coast",
|
||||
"https://example.test/2",
|
||||
base + Duration::minutes(46),
|
||||
);
|
||||
items.insert_if_new(&item2).await?;
|
||||
clusters.assign_or_create(&item2, THRESHOLD).await?;
|
||||
|
||||
let count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM story_clusters")
|
||||
.fetch_one(&_pool)
|
||||
.await?;
|
||||
assert_eq!(count, 2);
|
||||
let cluster3 = require_assigned(clusters.assign_or_create(&item_b1, ¶ms()).await?)?;
|
||||
assert_eq!(cluster3, cluster1);
|
||||
assert_eq!(load_cluster_row(&clusters, cluster3).await?.source_count, 2);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn same_source_repeats_create_one_cluster_with_both_members() -> Result<(), StoreError> {
|
||||
let (_pool, sources, items, clusters) = seeded_repos().await?;
|
||||
let (pool, sources, items, clusters) = seeded_repos().await?;
|
||||
let source = test_source("Al Jazeera", "https://aljazeera.test");
|
||||
sources.upsert(&source).await?;
|
||||
|
||||
@@ -348,7 +399,7 @@ mod tests {
|
||||
Utc::now(),
|
||||
);
|
||||
items.insert_if_new(&item1).await?;
|
||||
let cluster1 = clusters.assign_or_create(&item1, THRESHOLD).await?;
|
||||
let cluster1 = require_created(clusters.assign_or_create(&item1, ¶ms()).await?)?;
|
||||
|
||||
let item2 = test_item(
|
||||
source.id,
|
||||
@@ -357,17 +408,150 @@ mod tests {
|
||||
Utc::now(),
|
||||
);
|
||||
items.insert_if_new(&item2).await?;
|
||||
let cluster2 = clusters.assign_or_create(&item2, THRESHOLD).await?;
|
||||
let cluster2 = require_assigned(clusters.assign_or_create(&item2, ¶ms()).await?)?;
|
||||
assert_eq!(cluster2, cluster1);
|
||||
|
||||
assert_eq!(cluster2.id, cluster1.id);
|
||||
assert_eq!(cluster2.source_count, 1);
|
||||
assert_eq!(cluster2.member_item_ids.len(), 2);
|
||||
let cluster = load_cluster_row(&clusters, cluster2).await?;
|
||||
assert_eq!(cluster.source_count, 1);
|
||||
assert_eq!(cluster.member_item_ids.len(), 2);
|
||||
|
||||
let count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM cluster_members")
|
||||
.fetch_one(&_pool)
|
||||
.fetch_one(&pool)
|
||||
.await?;
|
||||
assert_eq!(count, 2);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn late_arrival_joins() -> Result<(), StoreError> {
|
||||
let (_pool, sources, items, clusters) = seeded_repos().await?;
|
||||
let source = test_source("Source", "https://example.test");
|
||||
sources.upsert(&source).await?;
|
||||
|
||||
// The cluster exists first; the matching item arrives published
|
||||
// 90 minutes BEFORE the cluster's first_seen_at (out-of-order
|
||||
// feed delivery) — inside the 2 h late-arrival grace.
|
||||
let base = Utc::now();
|
||||
let item1 = test_item(
|
||||
source.id,
|
||||
"Floods swamp coastal town after storm",
|
||||
"https://example.test/1",
|
||||
base,
|
||||
);
|
||||
items.insert_if_new(&item1).await?;
|
||||
let cluster1 = require_created(clusters.assign_or_create(&item1, ¶ms()).await?)?;
|
||||
|
||||
let item2 = test_item(
|
||||
source.id,
|
||||
"Floods swamp coastal town",
|
||||
"https://example.test/2",
|
||||
base - Duration::minutes(90),
|
||||
);
|
||||
items.insert_if_new(&item2).await?;
|
||||
let outcome = clusters.assign_or_create(&item2, ¶ms()).await?;
|
||||
|
||||
let joined = require_assigned(outcome)?;
|
||||
assert_eq!(joined, cluster1);
|
||||
|
||||
// last_seen_at stays at the forward bound (max(existing, item)).
|
||||
let cluster = load_cluster_row(&clusters, cluster1).await?;
|
||||
assert_eq!(cluster.last_seen_at, base);
|
||||
assert_eq!(cluster.member_item_ids.len(), 2);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn member_drift() -> Result<(), StoreError> {
|
||||
let (_pool, sources, items, clusters) = seeded_repos().await?;
|
||||
let source = test_source("Source", "https://example.test");
|
||||
sources.upsert(&source).await?;
|
||||
|
||||
let base = Utc::now();
|
||||
// Canonical member: the incoming item does NOT clear the
|
||||
// corroboration gate against it (shared non-proper: {earthquake}).
|
||||
let member1 = test_item(
|
||||
source.id,
|
||||
"Earthquake rattles Tokyo buildings",
|
||||
"https://example.test/1",
|
||||
base,
|
||||
);
|
||||
items.insert_if_new(&member1).await?;
|
||||
let cluster1 = require_created(clusters.assign_or_create(&member1, ¶ms()).await?)?;
|
||||
|
||||
let member2 = test_item(
|
||||
source.id,
|
||||
"Tokyo earthquake damages buildings",
|
||||
"https://example.test/2",
|
||||
base + Duration::minutes(1),
|
||||
);
|
||||
items.insert_if_new(&member2).await?;
|
||||
require_assigned(clusters.assign_or_create(&member2, ¶ms()).await?)?;
|
||||
|
||||
let member3 = test_item(
|
||||
source.id,
|
||||
"Powerful earthquake shakes Tokyo structures",
|
||||
"https://example.test/3",
|
||||
base + Duration::minutes(2),
|
||||
);
|
||||
items.insert_if_new(&member3).await?;
|
||||
require_assigned(clusters.assign_or_create(&member3, ¶ms()).await?)?;
|
||||
|
||||
// The incoming item matches the cluster's 3rd member, not the
|
||||
// canonical one — member-wise comparison must still assign it.
|
||||
let incoming = test_item(
|
||||
source.id,
|
||||
"Magnitude 7 earthquake damages Tokyo structures overnight",
|
||||
"https://example.test/4",
|
||||
base + Duration::minutes(3),
|
||||
);
|
||||
items.insert_if_new(&incoming).await?;
|
||||
let outcome = clusters.assign_or_create(&incoming, ¶ms()).await?;
|
||||
|
||||
let joined = require_assigned(outcome)?;
|
||||
assert_eq!(joined, cluster1);
|
||||
|
||||
// Canonical membership is unchanged by the drift.
|
||||
let cluster = load_cluster_row(&clusters, cluster1).await?;
|
||||
assert_eq!(cluster.canonical_item_id, member1.id);
|
||||
assert_eq!(cluster.member_item_ids.len(), 4);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn window_expiry() -> Result<(), StoreError> {
|
||||
let (pool, sources, items, clusters) = seeded_repos().await?;
|
||||
let source = test_source("Source", "https://example.test");
|
||||
sources.upsert(&source).await?;
|
||||
|
||||
let base = Utc::now();
|
||||
let item1 = test_item(
|
||||
source.id,
|
||||
"Quake damages historic temple in Kyoto",
|
||||
"https://example.test/1",
|
||||
base,
|
||||
);
|
||||
items.insert_if_new(&item1).await?;
|
||||
let cluster1 = require_created(clusters.assign_or_create(&item1, ¶ms()).await?)?;
|
||||
|
||||
// Identical title, but the cluster went cold 50 hours ago (window:
|
||||
// 48 h) — not a candidate, so a fresh cluster starts.
|
||||
let item2 = test_item(
|
||||
source.id,
|
||||
"Quake damages historic temple in Kyoto",
|
||||
"https://example.test/2",
|
||||
base + Duration::hours(50),
|
||||
);
|
||||
items.insert_if_new(&item2).await?;
|
||||
let outcome = clusters.assign_or_create(&item2, ¶ms()).await?;
|
||||
|
||||
let created = require_created(outcome)?;
|
||||
assert_ne!(created, cluster1);
|
||||
|
||||
let count: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM story_clusters")
|
||||
.fetch_one(&pool)
|
||||
.await?;
|
||||
assert_eq!(count, 2);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -70,7 +70,7 @@ impl FeedbackRepo {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::{ClusterRepo, ItemRepo, SourceRepo, init_db};
|
||||
use crate::{AssignOutcome, ClusterParams, ClusterRepo, ItemRepo, SourceRepo, init_db};
|
||||
use chrono::Utc;
|
||||
use news_core::{ItemKind, RawItem, Source, SourceKind};
|
||||
use pretty_assertions::assert_eq;
|
||||
@@ -105,10 +105,18 @@ mod tests {
|
||||
SourceRepo::new(pool.clone()).upsert(&source).await?;
|
||||
let item = test_item(source.id);
|
||||
ItemRepo::new(pool.clone()).insert_if_new(&item).await?;
|
||||
let cluster = ClusterRepo::new(pool.clone())
|
||||
.assign_or_create(&item, 0.5)
|
||||
let outcome = ClusterRepo::new(pool.clone())
|
||||
.assign_or_create(&item, &ClusterParams::default())
|
||||
.await?;
|
||||
Ok((pool, cluster.id))
|
||||
let cluster_id = match outcome {
|
||||
AssignOutcome::Assigned(id) | AssignOutcome::Created(id) => id,
|
||||
AssignOutcome::LlmCandidates(_) => {
|
||||
return Err(StoreError::InvalidStoredValue(
|
||||
"seed item unexpectedly landed in the gray zone".into(),
|
||||
));
|
||||
}
|
||||
};
|
||||
Ok((pool, cluster_id))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -10,7 +10,7 @@ mod percentile;
|
||||
mod prune;
|
||||
mod sources;
|
||||
|
||||
pub use clusters::ClusterRepo;
|
||||
pub use clusters::{AssignOutcome, ClusterParams, ClusterRepo};
|
||||
pub use error::StoreError;
|
||||
pub use feedback::FeedbackRepo;
|
||||
pub use items::{InsertOutcome, ItemRepo};
|
||||
|
||||
@@ -223,6 +223,51 @@ end of a deploy that finished. Anything that fails before that point leaves no
|
||||
record, so the next run does the whole thing again rather than mistaking a
|
||||
pulled image for a deployed one.
|
||||
|
||||
## Verifying a deployment
|
||||
|
||||
The version shows up in three places, and after a deploy they should all tell
|
||||
the same story.
|
||||
|
||||
The web footer shows `v{version} · {short sha}`, fetched from the API when the
|
||||
page loads; if that fetch fails, no footer renders at all. The same endpoint,
|
||||
asked directly:
|
||||
|
||||
```sh
|
||||
curl https://news.rcjohnstone.com/api/version
|
||||
# {"version":"0.2.0","git_sha":"…","build_time":"…"}
|
||||
```
|
||||
|
||||
The image carries the same facts as OCI labels, and the tags show what points
|
||||
where:
|
||||
|
||||
```sh
|
||||
sudo podman inspect git.rcjohnstone.com/connor/news:latest \
|
||||
--format '{{index .Config.Labels "org.opencontainers.image.version"}} {{index .Config.Labels "org.opencontainers.image.revision"}}'
|
||||
sudo podman images git.rcjohnstone.com/connor/news
|
||||
```
|
||||
|
||||
CI tags a single image id as `latest`, the short commit sha, and the release
|
||||
tag (`v0.2.0`), so `podman images` lists three rows with the same image id —
|
||||
and the labels are the durable record of which version and which commit an
|
||||
image was actually built from.
|
||||
|
||||
The updater keeps its own trail: `web/dist.deployed` holds the image id of the
|
||||
last deploy that got all the way through, and the service journal holds every
|
||||
run, including the ones that changed nothing:
|
||||
|
||||
```sh
|
||||
cat ~/data/news/web/dist.deployed
|
||||
journalctl -u news-update.service -n 50 --no-pager
|
||||
```
|
||||
|
||||
`systemctl list-timers news-update.timer` says when the next check fires.
|
||||
|
||||
None of that changes what the updater does: it still pulls `:latest`, compares
|
||||
the id against `dist.deployed`, and exits when they match. The `vX.Y.Z` tags
|
||||
are for people, not for the updater — a way to pin the compose file's `image:`
|
||||
or to roll back to a build you can name by version instead of a commit hash
|
||||
you have to go and look up.
|
||||
|
||||
## Rolling back
|
||||
|
||||
The frontend of the previous deploy is kept:
|
||||
|
||||
Reference in New Issue
Block a user