v0.2 wave 3: AP eng language filter, SourceConfig.name convergence, 12-feed example config with clustering/llm blocks

2026-09-04 09:52:14 -04:00
parent e61993fdc7
commit 2157dfc076
8 changed files with 323 additions and 9 deletions
@@ -1,11 +1,19 @@
 # Example configuration for news-triage.
 #
 # Copy to ~/config/news/config.toml (mounted read-only at /config/config.toml
-# in the container) and edit. All three sections are required; only
-# poll_interval_secs is optional (defaults to 300 seconds). The server
-# reloads this file on SIGHUP.
+# in the container) and edit. The sources, [notify], and [topics] sections
+# are required; poll_interval_secs is optional (defaults to 300 seconds).
+# The [clustering] and [llm] sections are optional too — they are spelled
+# out below with their defaults ([llm] ships disabled). The server reloads
+# this file on SIGHUP.

 # Feed sources to poll. kind is "rss" or "news-sitemap".
+#
+# name is optional: when set, it becomes the publisher label used for
+# distinct-source story counting (both BBC feeds share "BBC" so the double
+# feed counts as one publisher); when omitted, an existing row keeps its
+# name (seeded rows like "Al Jazeera" and "AP") and a new row defaults to
+# its URL.
 [[sources]]
 url = "https://www.aljazeera.com/xml/rss/all.xml"
 kind = "rss"
@@ -16,6 +24,7 @@ poll_interval_secs = 300
 [[sources]]
 url = "https://feeds.bbci.co.uk/news/world/rss.xml"
 kind = "rss"
+name = "BBC"
 weight = 1.2
 enabled = true

@@ -25,6 +34,81 @@ kind = "news-sitemap"
 weight = 0.9
 enabled = true

+# The nine feeds below were added in v0.2 (all verified live 2026-09-03).
+# Not included: DW (RSS 1.0/RDF — unsupported parser, silently yields zero
+# items) and Reuters (no public RSS).
+[[sources]]
+url = "https://feeds.bbci.co.uk/news/rss.xml"
+kind = "rss"
+name = "BBC"
+weight = 1.1
+enabled = true
+poll_interval_secs = 600
+
+[[sources]]
+url = "https://theguardian.com/world/rss"
+kind = "rss"
+name = "Guardian World"
+weight = 1.1
+enabled = true
+poll_interval_secs = 900
+
+[[sources]]
+url = "https://rss.nytimes.com/services/xml/rss/nyt/World.xml"
+kind = "rss"
+name = "NYT World"
+weight = 1.1
+enabled = true
+poll_interval_secs = 900
+
+[[sources]]
+url = "https://feeds.npr.org/1001/rss.xml"
+kind = "rss"
+name = "NPR"
+weight = 1.0
+enabled = true
+poll_interval_secs = 900
+
+[[sources]]
+url = "https://feeds.skynews.com/feeds/rss/world.xml"
+kind = "rss"
+name = "Sky World"
+weight = 0.9
+enabled = true
+poll_interval_secs = 600
+
+[[sources]]
+url = "https://cbc.ca/webfeed/rss/rss-world"
+kind = "rss"
+name = "CBC World"
+weight = 0.9
+enabled = true
+poll_interval_secs = 900
+
+[[sources]]
+url = "https://abc.net.au/news/feed/51120/rss.xml"
+kind = "rss"
+name = "ABC AU"
+weight = 0.9
+enabled = true
+poll_interval_secs = 600
+
+[[sources]]
+url = "https://pbs.org/newshour/feeds/rss/headlines"
+kind = "rss"
+name = "PBS NewsHour"
+weight = 0.9
+enabled = true
+poll_interval_secs = 1800
+
+[[sources]]
+url = "https://france24.com/en/rss"
+kind = "rss"
+name = "France24"
+weight = 0.9
+enabled = true
+poll_interval_secs = 1200
+
 [notify]
 notify_opinions = true
 quiet_hours_start = "22:00"
@@ -48,3 +132,27 @@ timezone = "America/Louisville"
 [topics]
 interests = ["middle east", "ukraine", "science"]
 blocklist = ["royal", "celebrity"]
+
+# Lexical-clustering tunables; every field is optional and defaults to the
+# value shown. A merge needs score >= similarity_threshold AND at least
+# corroboration_min shared non-proper-noun title tokens; pairs scoring in
+# the gray_zone_width band below the threshold are routed to the LLM layer.
+[clustering]
+similarity_threshold = 0.45
+window_hours = 48
+gray_zone_width = 0.15
+corroboration_min = 2
+proper_noun_weight = 2.0
+
+# LLM consolidation layer for gray-zone pairs. Ships disabled: the pipeline
+# runs exactly as without it until enabled is flipped to true. base_url is
+# the LiteLLM proxy root and model the exact flash-class model string; the
+# API key is read from api_key_env at call time. daily_budget_calls is an
+# approximate per-UTC-day call cap (a restart resets it).
+[llm]
+enabled = false
+base_url = ""
+model = ""
+api_key_env = "LITELLM_API_KEY"
+timeout_secs = 10
+daily_budget_calls = 200
@@ -19,6 +19,9 @@ struct PartialUrl {
    loc: String,
    title: String,
    published_raw: String,
+    /// `None` = no `news:language` element seen (fail-open); `Some` = the
+    /// accumulated element text to compare against `eng`.
+    language: Option<String>,
 }

 /// Parses a Google News Sitemap body (`<urlset><url><loc>` +
@@ -31,6 +34,12 @@ struct PartialUrl {
 /// downstream. A missing or unparseable publication date falls back to
 /// the fetch time.
 ///
+/// Entries whose `news:language` is present and not `eng` (compared
+/// case-insensitively after trimming) are dropped: the live AP sitemap
+/// carries English (`eng`) and Spanish (`spa`) items in one feed. An
+/// absent `news:language` keeps the entry (fail-open — never drop
+/// everything if the element is omitted).
+///
 /// Field text is accumulated across text and CDATA runs and trimmed once
 /// at item close.
 pub fn parse_news_sitemap(xml: &str, source: &Source) -> Result<Vec<RawItem>, IngestError> {
@@ -94,6 +103,7 @@ enum Field {
    Loc,
    Title,
    PublicationDate,
+    Language,
 }

 impl Field {
@@ -102,6 +112,7 @@ impl Field {
            "loc" => Some(Self::Loc),
            "news:title" => Some(Self::Title),
            "news:publication_date" => Some(Self::PublicationDate),
+            "news:language" => Some(Self::Language),
            _ => None,
        }
    }
@@ -112,6 +123,7 @@ fn apply_field(url: &mut PartialUrl, field: Field, text: &str) {
        Field::Loc => url.loc.push_str(text),
        Field::Title => url.title.push_str(text),
        Field::PublicationDate => url.published_raw.push_str(text),
+        Field::Language => url.language.get_or_insert_with(String::new).push_str(text),
    }
 }

@@ -125,6 +137,15 @@ fn parse_published_at(raw: &str) -> Result<DateTime<Utc>, IngestError> {
 }

 fn finalize(url: PartialUrl, source: &Source) -> Result<Option<RawItem>, IngestError> {
+    // Live AP sitemap (verified 2026-09-04): news:language sits at
+    // <url>/<news:news>/<news:publication>/<news:language>, values
+    // "eng" (513) and "spa" (188) in one feed. Drop only a PRESENT
+    // non-eng value; an absent element keeps the entry (fail-open).
+    if let Some(language) = &url.language
+        && !language.trim().eq_ignore_ascii_case("eng")
+    {
+        return Ok(None);
+    }
    let title = url.title.trim();
    let loc = url.loc.trim();
    if title.is_empty() || loc.is_empty() {
@@ -184,6 +205,43 @@ mod tests {
        Ok(())
    }

+    #[test]
+    fn non_eng_language_entries_are_dropped_and_absent_language_kept() -> Result<(), IngestError> {
+        let xml = include_str!("../tests/fixtures/ap_sitemap_language_filter.xml");
+        let source = sample_source();
+
+        let items = parse_news_sitemap(xml, &source)?;
+
+        assert_eq!(items.len(), 2);
+        assert_eq!(items[0].link, "https://apnews.com/article/eng-1");
+        assert_eq!(items[1].link, "https://apnews.com/article/no-language-1");
+        Ok(())
+    }
+
+    #[test]
+    fn language_match_is_trimmed_and_case_insensitive() -> Result<(), IngestError> {
+        let xml = r#"<urlset xmlns:news="http://www.google.com/schemas/sitemap-news/0.9">
+            <url>
+                <loc>https://apnews.com/article/case-1</loc>
+                <news:news>
+                    <news:publication>
+                        <news:name>Associated Press</news:name>
+                        <news:language>  ENG </news:language>
+                    </news:publication>
+                    <news:publication_date>2026-08-30T14:22:00Z</news:publication_date>
+                    <news:title>Uppercase padded language entry</news:title>
+                </news:news>
+            </url>
+        </urlset>"#;
+        let source = sample_source();
+
+        let items = parse_news_sitemap(xml, &source)?;
+
+        assert_eq!(items.len(), 1);
+        assert_eq!(items[0].title, "Uppercase padded language entry");
+        Ok(())
+    }
+
    #[test]
    fn cdata_wrapped_title_is_parsed_not_dropped() -> Result<(), IngestError> {
        let xml = r#"<urlset xmlns:news="http://www.google.com/schemas/sitemap-news/0.9">
@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Language-filter fixture. news:language position and values match the
+     live AP sitemap (verified 2026-09-04): child of <news:publication>,
+     ISO 639-2 codes "eng"/"spa". -->
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
+        xmlns:news="http://www.google.com/schemas/sitemap-news/0.9">
+  <url>
+    <loc>https://apnews.com/article/eng-1</loc>
+    <news:news>
+      <news:publication>
+        <news:name>Associated Press</news:name>
+        <news:language>eng</news:language>
+      </news:publication>
+      <news:publication_date>2026-09-03T12:00:00+00:00</news:publication_date>
+      <news:title>English item stays</news:title>
+    </news:news>
+  </url>
+  <url>
+    <loc>https://apnews.com/article/spa-1</loc>
+    <news:news>
+      <news:publication>
+        <news:name>Associated Press</news:name>
+        <news:language>spa</news:language>
+      </news:publication>
+      <news:publication_date>2026-09-03T12:30:00+00:00</news:publication_date>
+      <news:title>Spanish item is dropped</news:title>
+    </news:news>
+  </url>
+  <url>
+    <loc>https://apnews.com/article/no-language-1</loc>
+    <news:news>
+      <news:publication>
+        <news:name>Associated Press</news:name>
+      </news:publication>
+      <news:publication_date>2026-09-03T13:00:00+00:00</news:publication_date>
+      <news:title>Item without language stays (fail-open)</news:title>
+    </news:news>
+  </url>
+</urlset>
@@ -6,7 +6,7 @@
    <news:news>
      <news:publication>
        <news:name>Associated Press</news:name>
-        <news:language>en</news:language>
+        <news:language>eng</news:language>
      </news:publication>
      <news:publication_date>2026-08-30T14:22:00+00:00</news:publication_date>
      <news:title>Storm system moves toward the coast</news:title>
@@ -17,7 +17,7 @@
    <news:news>
      <news:publication>
        <news:name>Associated Press</news:name>
-        <news:language>en</news:language>
+        <news:language>eng</news:language>
      </news:publication>
      <news:publication_date>2026-08-30T13:05:00+00:00</news:publication_date>
      <news:title>Markets close mixed after volatile session</news:title>
@@ -28,7 +28,7 @@
    <news:news>
      <news:publication>
        <news:name>Associated Press</news:name>
-        <news:language>en</news:language>
+        <news:language>eng</news:language>
      </news:publication>
      <news:publication_date>2026-08-30T11:47:00+00:00</news:publication_date>
      <news:title>Election officials certify final results</news:title>
@@ -42,6 +42,7 @@ mod tests {
            sources: vec![SourceConfig {
                url: "https://example.test/rss".into(),
                kind: "rss".into(),
+                name: None,
                weight: 0.5,
                enabled: true,
                poll_interval_secs: Some(300),
@@ -20,6 +20,7 @@ fn test_config() -> NewsConfig {
        sources: vec![SourceConfig {
            url: "https://example.test/rss".into(),
            kind: "rss".into(),
+            name: None,
            weight: 0.5,
            enabled: true,
            poll_interval_secs: Some(300),
@@ -1,3 +1,5 @@
+// allow: SIZE_OK — 306 pure LOC (over 250 pre-existing); split candidate:
+// converge_sources + source_config_to_source → bin_support/converge.rs.
 use std::sync::Arc;

 use news_core::Source;
@@ -131,7 +133,15 @@ pub fn source_config_to_source(
        other => return Err(format!("unrecognized source kind: {other}").into()),
    };
    let id = existing.map_or_else(|| news_core::stable_source_id(&cfg.url), |prior| prior.id);
-    let name = existing.map_or_else(|| cfg.url.clone(), |prior| prior.name.clone());
+    // Config `name` is an explicit publisher label: when present it upserts
+    // over the row's existing name (WS5 distinct-source counting relies on
+    // both BBC feeds converging to "BBC"); when absent the row keeps its
+    // name (operator renames and seeds like "Al Jazeera" survive) and new
+    // rows fall back to the URL. `None` never overwrites by construction.
+    let name = match existing {
+        Some(prior) => cfg.name.clone().unwrap_or_else(|| prior.name.clone()),
+        None => cfg.name.clone().unwrap_or_else(|| cfg.url.clone()),
+    };
    let poll_interval_secs = cfg.poll_interval_secs.unwrap_or(300) as u32;
    Ok(Source {
        id,
@@ -146,8 +156,9 @@ pub fn source_config_to_source(

 /// Converges the `sources` table with the config: every config source is
 /// upserted, reusing the existing row's id and name when its URL is
-/// already known (so history and operator renames survive), and sources
-/// whose URL is no longer in the config are disabled rather than deleted,
+/// already known (so history and operator renames survive; a config
+/// `name` overrides the row's name for that upsert), and sources whose
+/// URL is no longer in the config are disabled rather than deleted,
 /// keeping their items and cluster history intact. Returns the enabled
 /// sources to poll; an empty config yields an empty list — nothing is
 /// seeded implicitly.
@@ -198,6 +209,7 @@ mod tests {
        SourceConfig {
            url: url.to_string(),
            kind: "rss".to_string(),
+            name: None,
            weight,
            enabled,
            poll_interval_secs: None,
@@ -284,6 +296,47 @@ mod tests {
        Ok(())
    }

+    #[tokio::test]
+    async fn config_name_overrides_existing_row_name()
+    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        let repo = fresh_repo().await?;
+        let url = "https://feeds.bbci.co.uk/news/world/rss.xml";
+        let legacy = Source {
+            id: Uuid::new_v4(),
+            name: "My BBC".to_string(),
+            url: url.to_string(),
+            kind: SourceKind::RssAtom,
+            weight: 0.9,
+            enabled: true,
+            poll_interval_secs: 900,
+        };
+        repo.upsert(&legacy).await?;
+        let mut cfg = source_cfg(url, 1.2, true);
+        cfg.name = Some("BBC".to_string());
+
+        let enabled = converge_sources(&repo, &[cfg]).await?;
+
+        assert_eq!(enabled.len(), 1);
+        assert_eq!(enabled[0].id, legacy.id);
+        assert_eq!(enabled[0].name, "BBC");
+        Ok(())
+    }
+
+    #[tokio::test]
+    async fn config_name_names_new_source_instead_of_url()
+    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
+        let repo = fresh_repo().await?;
+        let mut cfg = source_cfg("https://www.theguardian.com/world/rss", 1.1, true);
+        cfg.name = Some("Guardian World".to_string());
+
+        let enabled = converge_sources(&repo, &[cfg]).await?;
+
+        assert_eq!(enabled.len(), 1);
+        assert_eq!(enabled[0].name, "Guardian World");
+        assert_ne!(enabled[0].name, enabled[0].url);
+        Ok(())
+    }
+
    #[tokio::test]
    async fn converge_reenables_previously_disabled_source()
    -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
@@ -2,6 +2,8 @@
 //!
 //! The schema matches `.omo/drafts/news-triage.md` line 47: sources, notify
 //! settings, and topics are all required; only `poll_interval_secs` is optional.
+// allow: SIZE_OK — 346 pure LOC (over 250 pre-existing); split candidate:
+// per-section submodules (notify/clustering/llm) with their tests.

 use std::path::{Path, PathBuf};
 use std::sync::Arc;
@@ -37,6 +39,11 @@ pub struct SourceConfig {
    pub url: String,
    /// Feed kind, e.g. `rss` or `news-sitemap`.
    pub kind: String,
+    /// Optional publisher name. When set, convergence upserts it into
+    /// `sources.name` (even over an existing row); when absent, existing
+    /// rows keep their name and new rows default to the feed URL.
+    #[serde(default)]
+    pub name: Option<String>,
    /// Source prominence / trust weight.
    pub weight: f64,
    /// Whether the source is polled.
@@ -436,6 +443,53 @@ blocklist = ["royal", "celebrity"]
        Ok(())
    }

+    #[tokio::test]
+    async fn source_name_parses_and_defaults_to_none() -> Result<(), ServerError> {
+        let dir = tempfile::tempdir()?;
+        let path = dir.path().join("config.toml");
+        std::fs::write(&path, valid_config_toml())?;
+
+        let config = load(&path)?;
+        assert_eq!(config.sources[0].name, None);
+        assert_eq!(config.sources[1].name, None);
+
+        let with_name = valid_config_toml().replace("weight = 1.2", "name = \"BBC\"\nweight = 1.2");
+        std::fs::write(&path, with_name)?;
+        let config = load(&path)?;
+        assert_eq!(config.sources[0].name, None);
+        assert_eq!(config.sources[1].name, Some("BBC".to_string()));
+        Ok(())
+    }
+
+    /// Proves the shipped example file parses with the optional
+    /// [clustering]/[llm] sections and the WS5 source list. The count is
+    /// 12: AJ + AP + BBC World kept, 9 URLs added (the task's "11 total"
+    /// summary miscounts its own enumeration).
+    #[tokio::test]
+    async fn example_config_parses_with_optional_sections() -> Result<(), ServerError> {
+        let path = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../config.example.toml");
+
+        let config = load(&path)?;
+
+        assert_eq!(config.sources.len(), 12);
+        let bbc_named = config
+            .sources
+            .iter()
+            .filter(|s| s.name.as_deref() == Some("BBC"))
+            .count();
+        assert_eq!(
+            bbc_named, 2,
+            "both BBC feeds share the name for distinct-count"
+        );
+        let Some(aj) = config.sources.iter().find(|s| s.url.contains("aljazeera")) else {
+            panic!("Al Jazeera source missing from example config");
+        };
+        assert_eq!(aj.name, None, "AJ keeps its seeded row name");
+        assert_eq!(config.clustering, ClusteringSection::default());
+        assert!(!config.llm.enabled);
+        Ok(())
+    }
+
    #[traced_test]
    #[tokio::test]
    async fn bad_reload_keeps_last_good_config() -> Result<(), ServerError> {