Added the mb db download. Big upsides and downsides

frontend/src/api.rs

@@ -374,3 +374,13 @@ pub async fn ytauth_refresh() -> Result<serde_json::Value, ApiError> {
 pub async fn ytauth_clear_cookies() -> Result<(), ApiError> {
     delete(&format!("{BASE}/ytauth/cookies")).await
 }
+
+// --- MusicBrainz Local DB ---
+
+pub async fn get_mb_status() -> Result<MbStatus, ApiError> {
+    get_json(&format!("{BASE}/mb-status")).await
+}
+
+pub async fn trigger_mb_import() -> Result<TaskRef, ApiError> {
+    post_empty(&format!("{BASE}/mb-import")).await
+}

frontend/src/pages/settings.rs

@@ -3,7 +3,7 @@ use web_sys::HtmlSelectElement;
 use yew::prelude::*;
 
 use crate::api;
-use crate::types::{AppConfig, SubsonicPasswordStatus, YtAuthStatus};
+use crate::types::{AppConfig, MbStatus, SubsonicPasswordStatus, YtAuthStatus};
 
 #[function_component(SettingsPage)]
 pub fn settings_page() -> Html {
@@ -15,12 +15,15 @@ pub fn settings_page() -> Html {
     let subsonic_status = use_state(|| None::<SubsonicPasswordStatus>);
     let subsonic_password = use_state(String::new);
     let subsonic_saving = use_state(|| false);
+    let mb_status = use_state(|| None::<MbStatus>);
+    let mb_importing = use_state(|| false);
 
     {
         let config = config.clone();
         let error = error.clone();
         let ytauth = ytauth.clone();
         let subsonic_status = subsonic_status.clone();
+        let mb_status = mb_status.clone();
         use_effect_with((), move |_| {
             wasm_bindgen_futures::spawn_local(async move {
                 match api::get_config().await {
@@ -38,6 +41,11 @@ pub fn settings_page() -> Html {
                     subsonic_status.set(Some(status));
                 }
             });
+            wasm_bindgen_futures::spawn_local(async move {
+                if let Ok(status) = api::get_mb_status().await {
+                    mb_status.set(Some(status));
+                }
+            });
         });
     }
 
@@ -596,6 +604,92 @@ pub fn settings_page() -> Html {
                     </button>
                 </div>
 
+                // MusicBrainz Local Database
+                <div class="card">
+                    <h3>{ "MusicBrainz Database" }</h3>
+                    <p class="text-sm text-muted mb-1">
+                        { "Import the MusicBrainz database locally for instant artist/album lookups instead of rate-limited API calls. " }
+                        { "Makes browsing and watching artists dramatically faster." }
+                    </p>
+                    <div class="card" style="border-color: var(--warning); background: rgba(234, 179, 8, 0.08); margin: 0.5rem 0;">
+                        <p class="text-sm" style="margin:0;">
+                            <strong style="color: var(--warning);">{ "Heads up: " }</strong>
+                            { "This downloads ~24 GB of data and builds a ~10 GB local database. " }
+                            { "The initial import can take 3-6 hours depending on your hardware. " }
+                            { "Total disk usage: ~35 GB (downloads + database). " }
+                            { "After the initial import, the database is automatically refreshed weekly to stay current." }
+                        </p>
+                    </div>
+                    {
+                        if let Some(ref status) = *mb_status {
+                            if status.has_local_db {
+                                if let Some(ref stats) = status.stats {
+                                    let import_date = stats.last_import_date.clone().unwrap_or_else(|| "unknown".into());
+                                    html! {
+                                        <>
+                                            <p>
+                                                <span class="badge badge-success">{ "Loaded" }</span>
+                                                <span class="text-muted text-sm" style="margin-left: 0.5rem;">
+                                                    { format!("imported {}", import_date) }
+                                                </span>
+                                            </p>
+                                            <p class="text-sm">
+                                                { format!("{} artists, {} release groups, {} releases, {} recordings",
+                                                    stats.artists, stats.release_groups, stats.releases, stats.recordings) }
+                                            </p>
+                                        </>
+                                    }
+                                } else {
+                                    html! {
+                                        <p><span class="badge badge-success">{ "Loaded" }</span></p>
+                                    }
+                                }
+                            } else {
+                                html! {
+                                    <p class="text-sm text-muted">
+                                        { "Not configured. Import data to enable instant lookups." }
+                                    </p>
+                                }
+                            }
+                        } else {
+                            html! { <p class="text-sm text-muted">{ "Loading..." }</p> }
+                        }
+                    }
+                    <button type="button" class="btn btn-primary"
+                        disabled={*mb_importing}
+                        onclick={{
+                            let mb_importing = mb_importing.clone();
+                            let mb_status = mb_status.clone();
+                            let message = message.clone();
+                            let error = error.clone();
+                            Callback::from(move |_: MouseEvent| {
+                                let mb_importing = mb_importing.clone();
+                                let mb_status = mb_status.clone();
+                                let message = message.clone();
+                                let error = error.clone();
+                                mb_importing.set(true);
+                                wasm_bindgen_futures::spawn_local(async move {
+                                    match api::trigger_mb_import().await {
+                                        Ok(task_ref) => {
+                                            message.set(Some(format!(
+                                                "MusicBrainz import started (task {}). This will take a while.",
+                                                task_ref.task_id
+                                            )));
+                                            // Refresh status after a short delay
+                                            if let Ok(s) = api::get_mb_status().await {
+                                                mb_status.set(Some(s));
+                                            }
+                                        }
+                                        Err(e) => error.set(Some(e.0)),
+                                    }
+                                    mb_importing.set(false);
+                                });
+                            })
+                        }}>
+                        { if *mb_importing { "Starting import..." } else { "Import MusicBrainz Data" } }
+                    </button>
+                </div>
+
                 // Metadata Providers
                 <div class="card">
                     <h3>{ "Metadata Providers" }</h3>

frontend/src/types.rs

@@ -400,6 +400,8 @@ pub struct AppConfig {
     pub metadata: MetadataConfigFe,
     #[serde(default)]
     pub scheduling: SchedulingConfigFe,
+    #[serde(default)]
+    pub musicbrainz: MusicBrainzConfigFe,
 }
 
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default)]
@@ -513,3 +515,36 @@ fn default_lyrics_source() -> String {
 fn default_cover_art_source() -> String {
     "coverartarchive".into()
 }
+
+// --- MusicBrainz local DB ---
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Default)]
+pub struct MusicBrainzConfigFe {
+    #[serde(default)]
+    pub local_db_path: Option<String>,
+    #[serde(default)]
+    pub auto_update: bool,
+}
+
+#[derive(Debug, Clone, PartialEq, Deserialize)]
+pub struct MbStatus {
+    pub has_local_db: bool,
+    #[serde(default)]
+    pub stats: Option<MbLocalStats>,
+}
+
+#[derive(Debug, Clone, PartialEq, Deserialize)]
+pub struct MbLocalStats {
+    #[serde(default)]
+    pub artists: u64,
+    #[serde(default)]
+    pub release_groups: u64,
+    #[serde(default)]
+    pub releases: u64,
+    #[serde(default)]
+    pub recordings: u64,
+    #[serde(default)]
+    pub tracks: u64,
+    #[serde(default)]
+    pub last_import_date: Option<String>,
+}

src/lib.rs

@@ -8,6 +8,7 @@ pub mod auth;
 pub mod config;
 pub mod cookie_refresh;
 pub mod error;
+pub mod mb_update;
 pub mod monitor;
 pub mod pipeline;
 pub mod pipeline_scheduler;

src/main.rs

@@ -5,8 +5,8 @@ use clap::Parser;
 use tracing_actix_web::TracingLogger;
 use tracing_subscriber::EnvFilter;
 
-use shanty_data::MusicBrainzFetcher;
 use shanty_data::WikipediaFetcher;
+use shanty_data::{HybridMusicBrainzFetcher, LocalMusicBrainzFetcher, MusicBrainzFetcher};
 use shanty_db::Database;
 use shanty_search::MusicBrainzSearch;
 
@@ -54,8 +54,23 @@ async fn main() -> anyhow::Result<()> {
     tracing::info!(url = %config.database_url, "connecting to database");
     let db = Database::new(&config.database_url).await?;
 
-    let mb_client = MusicBrainzFetcher::new()?;
-    let search = MusicBrainzSearch::with_limiter(mb_client.limiter())?;
+    let mb_remote = MusicBrainzFetcher::new()?;
+    let search = MusicBrainzSearch::with_limiter(mb_remote.limiter())?;
+
+    // Set up local MB database if configured
+    let local_mb = create_local_mb_fetcher(&config);
+    let mb_client = HybridMusicBrainzFetcher::new(local_mb, mb_remote);
+
+    if mb_client.has_local_db()
+        && let Some(stats) = mb_client.local_stats()
+    {
+        tracing::info!(
+            artists = stats.artists,
+            release_groups = stats.release_groups,
+            "local MusicBrainz database loaded"
+        );
+    }
+
     let wiki_fetcher = WikipediaFetcher::new()?;
 
     let bind = format!("{}:{}", config.web.bind, config.web.port);
@@ -85,6 +100,7 @@ async fn main() -> anyhow::Result<()> {
     // Start pipeline and monitor schedulers
     shanty_web::pipeline_scheduler::spawn(state.clone());
     shanty_web::monitor::spawn(state.clone());
+    shanty_web::mb_update::spawn(state.clone());
 
     // Resolve static files directory relative to the binary location
     let static_dir = std::env::current_exe()
@@ -157,3 +173,36 @@ async fn main() -> anyhow::Result<()> {
 
     Ok(())
 }
+
+/// Create a LocalMusicBrainzFetcher from config if available.
+fn create_local_mb_fetcher(config: &AppConfig) -> Option<LocalMusicBrainzFetcher> {
+    let db_path = config
+        .musicbrainz
+        .local_db_path
+        .as_ref()
+        .map(|p| p.to_string_lossy().to_string())
+        .or_else(|| {
+            let default_path = shanty_config::data_dir().join("shanty-mb.db");
+            if default_path.exists() {
+                Some(default_path.to_string_lossy().to_string())
+            } else {
+                None
+            }
+        })?;
+
+    match LocalMusicBrainzFetcher::new(&db_path) {
+        Ok(fetcher) => {
+            if fetcher.is_available() {
+                tracing::info!(path = %db_path, "opened local MusicBrainz database");
+                Some(fetcher)
+            } else {
+                tracing::debug!(path = %db_path, "local MB database exists but has no data");
+                None
+            }
+        }
+        Err(e) => {
+            tracing::warn!(path = %db_path, error = %e, "failed to open local MB database");
+            None
+        }
+    }
+}

src/mb_update.rs

@@ -0,0 +1,120 @@
+//! Background task that periodically re-imports the MusicBrainz database.
+
+use std::time::Duration;
+
+use actix_web::web;
+
+use crate::state::AppState;
+
+/// Spawn the weekly MB database update loop.
+///
+/// Only runs if a local MB database exists (meaning the user has done an initial import).
+/// Downloads fresh dumps and re-imports weekly.
+pub fn spawn(state: web::Data<AppState>) {
+    tokio::spawn(async move {
+        // Wait 1 hour after startup before first check
+        tokio::time::sleep(Duration::from_secs(3600)).await;
+
+        loop {
+            // Check if local DB exists and auto-update is desired
+            let has_local = state.mb_client.has_local_db();
+            if !has_local {
+                // No local DB — sleep a day and check again
+                tokio::time::sleep(Duration::from_secs(86400)).await;
+                continue;
+            }
+
+            // Check how old the import is
+            let needs_update = state
+                .mb_client
+                .local_stats()
+                .and_then(|s| s.last_import_date)
+                .map(|date| {
+                    // Parse the date and check if it's older than 7 days
+                    chrono::NaiveDate::parse_from_str(&date, "%Y-%m-%d")
+                        .map(|d| {
+                            let age = chrono::Utc::now().naive_utc().date() - d;
+                            age.num_days() >= 7
+                        })
+                        .unwrap_or(true) // If we can't parse the date, update
+                })
+                .unwrap_or(false); // No stats = no local DB = skip
+
+            if !needs_update {
+                // Check again in 24 hours
+                tokio::time::sleep(Duration::from_secs(86400)).await;
+                continue;
+            }
+
+            tracing::info!("starting weekly MusicBrainz database update");
+
+            let data_dir = shanty_config::data_dir().join("mb-dumps");
+            let db_path = state
+                .config
+                .read()
+                .await
+                .musicbrainz
+                .local_db_path
+                .clone()
+                .unwrap_or_else(|| shanty_config::data_dir().join("shanty-mb.db"));
+
+            // Download fresh dumps
+            if let Err(e) = std::fs::create_dir_all(&data_dir) {
+                tracing::error!(error = %e, "failed to create dump dir for MB update");
+                tokio::time::sleep(Duration::from_secs(86400)).await;
+                continue;
+            }
+
+            let timestamp = match shanty_data::mb_import::discover_latest_dump_folder().await {
+                Ok(t) => t,
+                Err(e) => {
+                    tracing::error!(error = %e, "failed to discover latest MB dump");
+                    tokio::time::sleep(Duration::from_secs(86400)).await;
+                    continue;
+                }
+            };
+
+            let mut download_failed = false;
+            for filename in shanty_data::mb_import::DUMP_FILES {
+                if let Err(e) =
+                    shanty_data::mb_import::download_dump(filename, &timestamp, &data_dir, |msg| {
+                        tracing::info!("{msg}");
+                    })
+                    .await
+                {
+                    tracing::error!(file = filename, error = %e, "MB dump download failed");
+                    download_failed = true;
+                    break;
+                }
+            }
+
+            if download_failed {
+                tokio::time::sleep(Duration::from_secs(86400)).await;
+                continue;
+            }
+
+            // Run import in blocking task
+            let result = tokio::task::spawn_blocking(move || {
+                shanty_data::mb_import::run_import_at_path(&db_path, &data_dir, |msg| {
+                    tracing::info!("{msg}");
+                })
+            })
+            .await;
+
+            match result {
+                Ok(Ok(stats)) => {
+                    tracing::info!(%stats, "weekly MusicBrainz update complete");
+                }
+                Ok(Err(e)) => {
+                    tracing::error!(error = %e, "weekly MusicBrainz import failed");
+                }
+                Err(e) => {
+                    tracing::error!(error = %e, "weekly MusicBrainz import task panicked");
+                }
+            }
+
+            // Sleep 7 days before next check
+            tokio::time::sleep(Duration::from_secs(7 * 86400)).await;
+        }
+    });
+}

src/routes/artists.rs

@@ -193,12 +193,29 @@ async fn get_cached_album_tracks(
     let release_mbid = if let Some(rid) = first_release_id {
         rid.to_string()
     } else {
-        // Browse releases for this release group (through shared rate limiter)
-        state
-            .mb_client
-            .resolve_release_from_group(rg_id)
-            .await
-            .map_err(|e| ApiError::Internal(format!("MB error for group {rg_id}: {e}")))?
+        // Check DB cache for previously resolved release MBID
+        let resolve_cache_key = format!("release_for_rg:{rg_id}");
+        if let Ok(Some(cached_rid)) = queries::cache::get(state.db.conn(), &resolve_cache_key).await
+        {
+            cached_rid
+        } else {
+            // Browse releases for this release group (through shared rate limiter)
+            let rid = state
+                .mb_client
+                .resolve_release_from_group(rg_id)
+                .await
+                .map_err(|e| ApiError::Internal(format!("MB error for group {rg_id}: {e}")))?;
+            // Cache the resolved release MBID for 365 days — it never changes
+            let _ = queries::cache::set(
+                state.db.conn(),
+                &resolve_cache_key,
+                "musicbrainz",
+                &rid,
+                365 * 86400,
+            )
+            .await;
+            rid
+        }
     };
 
     let mb_tracks = state
@@ -259,7 +276,8 @@ pub async fn enrich_artist(
     quick_mode: bool,
 ) -> Result<serde_json::Value, ApiError> {
     // Resolve artist: local ID or MBID
-    let (artist, id, mbid) = if let Ok(local_id) = id_or_mbid.parse() {
+    // Track whether we already fetched artist info during resolution to avoid a duplicate API call
+    let (artist, id, mbid, prefetched_info) = if let Ok(local_id) = id_or_mbid.parse() {
         let artist = queries::artists::get_by_id(state.db.conn(), local_id).await?;
         let mbid = match &artist.musicbrainz_id {
             Some(m) => m.clone(),
@@ -274,7 +292,7 @@ pub async fn enrich_artist(
                 })?
             }
         };
-        (artist, Some(local_id), mbid)
+        (artist, Some(local_id), mbid, None)
     } else {
         let mbid = id_or_mbid.to_string();
 
@@ -288,9 +306,10 @@ pub async fn enrich_artist(
 
         if let Some(a) = local {
             let local_id = a.id;
-            (a, Some(local_id), mbid)
+            (a, Some(local_id), mbid, None)
         } else {
             // Look up artist info from MusicBrainz by MBID — don't create a local record
+            // This fetches url-rels too, so we reuse it below instead of calling get_artist_info() again
             let info =
                 state.mb_client.get_artist_info(&mbid).await.map_err(|e| {
                     ApiError::NotFound(format!("artist MBID {mbid} not found: {e}"))
@@ -307,24 +326,34 @@ pub async fn enrich_artist(
                 monitored: false,
                 last_checked_at: None,
             };
-            (synthetic, None, mbid)
+            (synthetic, None, mbid, Some(info))
         }
     };
 
-    // Fetch detailed artist info (country, type, URLs) — best-effort
-    let artist_info = match state.mb_client.get_artist_info(&mbid).await {
-        Ok(info) => {
-            tracing::debug!(
-                mbid = %mbid,
-                urls = info.urls.len(),
-                country = ?info.country,
-                "fetched artist info"
-            );
-            Some(info)
-        }
-        Err(e) => {
-            tracing::warn!(mbid = %mbid, error = %e, "failed to fetch artist info");
-            None
+    // Fetch detailed artist info (country, type, URLs) — reuse if already fetched during resolution
+    let artist_info = if let Some(info) = prefetched_info {
+        tracing::debug!(
+            mbid = %mbid,
+            urls = info.urls.len(),
+            country = ?info.country,
+            "reusing prefetched artist info"
+        );
+        Some(info)
+    } else {
+        match state.mb_client.get_artist_info(&mbid).await {
+            Ok(info) => {
+                tracing::debug!(
+                    mbid = %mbid,
+                    urls = info.urls.len(),
+                    country = ?info.country,
+                    "fetched artist info"
+                );
+                Some(info)
+            }
+            Err(e) => {
+                tracing::warn!(mbid = %mbid, error = %e, "failed to fetch artist info");
+                None
+            }
         }
     };
 

src/routes/system.rs

@@ -24,6 +24,8 @@ pub fn configure(cfg: &mut web::ServiceConfig) {
         .service(web::resource("/monitor/status").route(web::get().to(get_monitor_status)))
         .service(web::resource("/scheduler/skip-pipeline").route(web::post().to(skip_pipeline)))
         .service(web::resource("/scheduler/skip-monitor").route(web::post().to(skip_monitor)))
+        .service(web::resource("/mb-status").route(web::get().to(get_mb_status)))
+        .service(web::resource("/mb-import").route(web::post().to(trigger_mb_import)))
         .service(
             web::resource("/config")
                 .route(web::get().to(get_config))
@@ -327,3 +329,106 @@ async fn skip_monitor(
     sched.next_monitor = None;
     Ok(HttpResponse::Ok().json(serde_json::json!({"status": "skipped"})))
 }
+
+async fn get_mb_status(
+    state: web::Data<AppState>,
+    session: Session,
+) -> Result<HttpResponse, ApiError> {
+    auth::require_auth(&session)?;
+    let has_local = state.mb_client.has_local_db();
+    let stats = state.mb_client.local_stats();
+    Ok(HttpResponse::Ok().json(serde_json::json!({
+        "has_local_db": has_local,
+        "stats": stats,
+    })))
+}
+
+async fn trigger_mb_import(
+    state: web::Data<AppState>,
+    session: Session,
+) -> Result<HttpResponse, ApiError> {
+    auth::require_admin(&session)?;
+    let task_id = state.tasks.register("mb_import");
+    let tid = task_id.clone();
+    let config = state.config.read().await.clone();
+
+    tokio::spawn(async move {
+        state
+            .tasks
+            .update_progress(&tid, 0, 0, "Starting MusicBrainz import...");
+
+        let data_dir = shanty_config::data_dir().join("mb-dumps");
+        let db_path = config
+            .musicbrainz
+            .local_db_path
+            .clone()
+            .unwrap_or_else(|| shanty_config::data_dir().join("shanty-mb.db"));
+
+        // Download dumps
+        state
+            .tasks
+            .update_progress(&tid, 0, 4, "Downloading dumps...");
+        if let Err(e) = std::fs::create_dir_all(&data_dir) {
+            state
+                .tasks
+                .fail(&tid, format!("Failed to create data dir: {e}"));
+            return;
+        }
+
+        let timestamp = match shanty_data::mb_import::discover_latest_dump_folder().await {
+            Ok(t) => t,
+            Err(e) => {
+                state
+                    .tasks
+                    .fail(&tid, format!("Failed to discover latest dump: {e}"));
+                return;
+            }
+        };
+
+        for (i, filename) in shanty_data::mb_import::DUMP_FILES.iter().enumerate() {
+            state.tasks.update_progress(
+                &tid,
+                i as u64,
+                4 + 4, // 4 downloads + 4 imports
+                &format!("Downloading {filename}..."),
+            );
+            if let Err(e) =
+                shanty_data::mb_import::download_dump(filename, &timestamp, &data_dir, |_| {}).await
+            {
+                state
+                    .tasks
+                    .fail(&tid, format!("Failed to download {filename}: {e}"));
+                return;
+            }
+        }
+
+        // Run import
+        state
+            .tasks
+            .update_progress(&tid, 4, 8, "Importing into database...");
+
+        let tid_clone = tid.clone();
+        let state_clone = state.clone();
+        // Run import in blocking task since rusqlite is sync
+        let result = tokio::task::spawn_blocking(move || {
+            shanty_data::mb_import::run_import_at_path(&db_path, &data_dir, |msg| {
+                state_clone.tasks.update_progress(&tid_clone, 4, 8, msg);
+            })
+        })
+        .await;
+
+        match result {
+            Ok(Ok(stats)) => {
+                state.tasks.complete(&tid, format!("{stats}"));
+            }
+            Ok(Err(e)) => {
+                state.tasks.fail(&tid, format!("Import failed: {e}"));
+            }
+            Err(e) => {
+                state.tasks.fail(&tid, format!("Import task panicked: {e}"));
+            }
+        }
+    });
+
+    Ok(HttpResponse::Accepted().json(serde_json::json!({ "task_id": task_id })))
+}

src/state.rs

@@ -1,7 +1,7 @@
 use std::sync::Arc;
 use tokio::sync::{Mutex, RwLock};
 
-use shanty_data::MusicBrainzFetcher;
+use shanty_data::HybridMusicBrainzFetcher;
 use shanty_data::WikipediaFetcher;
 use shanty_db::Database;
 use shanty_search::MusicBrainzSearch;
@@ -28,7 +28,7 @@ pub struct SchedulerInfo {
 
 pub struct AppState {
     pub db: Database,
-    pub mb_client: MusicBrainzFetcher,
+    pub mb_client: HybridMusicBrainzFetcher,
     pub search: MusicBrainzSearch,
     pub wiki_fetcher: WikipediaFetcher,
     pub config: Arc<RwLock<AppConfig>>,