redux of the worker queue

src/routes/system.rs

@@ -3,12 +3,12 @@ use actix_web::{HttpResponse, web};
 use serde::Deserialize;
 
 use shanty_db::entities::download_queue::DownloadStatus;
+use shanty_db::entities::work_queue::WorkTaskType;
 use shanty_db::queries;
 
 use crate::auth;
 use crate::config::AppConfig;
 use crate::error::ApiError;
-use crate::routes::artists::enrich_all_watched_artists;
 use crate::state::AppState;
 
 pub fn configure(cfg: &mut web::ServiceConfig) {
@@ -38,13 +38,13 @@ async fn get_status(
     session: Session,
 ) -> Result<HttpResponse, ApiError> {
     auth::require_auth(&session)?;
-    let summary = shanty_watch::library_summary(state.db.conn()).await?;
-    let pending_items =
-        queries::downloads::list(state.db.conn(), Some(DownloadStatus::Pending)).await?;
+    let conn = state.db.conn();
+
+    let summary = shanty_watch::library_summary(conn).await?;
+    let pending_items = queries::downloads::list(conn, Some(DownloadStatus::Pending)).await?;
     let downloading_items =
-        queries::downloads::list(state.db.conn(), Some(DownloadStatus::Downloading)).await?;
-    let failed_items =
-        queries::downloads::list(state.db.conn(), Some(DownloadStatus::Failed)).await?;
+        queries::downloads::list(conn, Some(DownloadStatus::Downloading)).await?;
+    let failed_items = queries::downloads::list(conn, Some(DownloadStatus::Failed)).await?;
     let tasks = state.tasks.list();
 
     let mut queue_items = Vec::new();
@@ -52,15 +52,38 @@ async fn get_status(
     queue_items.extend(pending_items.iter().cloned());
     queue_items.extend(failed_items.iter().take(5).cloned());
 
-    let needs_tagging = queries::tracks::get_needing_metadata(state.db.conn()).await?;
+    let needs_tagging = queries::tracks::get_needing_metadata(conn).await?;
 
-    // Scheduled task info
-    let sched = state.scheduler.lock().await;
-    let scheduled_tasks = serde_json::json!({
-        "next_pipeline": sched.next_pipeline,
-        "next_monitor": sched.next_monitor,
-    });
-    drop(sched);
+    // Work queue counts
+    let work_queue = queries::work_queue::counts_all(conn).await.ok();
+
+    // Scheduler state from DB
+    let scheduler_jobs = queries::scheduler_state::list_all(conn).await.unwrap_or_default();
+    let scheduler_json: serde_json::Value = scheduler_jobs
+        .iter()
+        .map(|j| {
+            (
+                j.job_name.clone(),
+                serde_json::json!({
+                    "last_run": j.last_run_at,
+                    "next_run": j.next_run_at,
+                    "last_result": j.last_result,
+                    "enabled": j.enabled,
+                }),
+            )
+        })
+        .collect::<serde_json::Map<String, serde_json::Value>>()
+        .into();
+
+    // Backward-compatible scheduled field (from scheduler_state DB)
+    let next_pipeline = scheduler_jobs
+        .iter()
+        .find(|j| j.job_name == "pipeline")
+        .and_then(|j| j.next_run_at);
+    let next_monitor = scheduler_jobs
+        .iter()
+        .find(|j| j.job_name == "monitor")
+        .and_then(|j| j.next_run_at);
 
     Ok(HttpResponse::Ok().json(serde_json::json!({
         "library": summary,
@@ -75,7 +98,12 @@ async fn get_status(
             "items": needs_tagging.iter().take(20).collect::<Vec<_>>(),
         },
         "tasks": tasks,
-        "scheduled": scheduled_tasks,
+        "scheduled": {
+            "next_pipeline": next_pipeline,
+            "next_monitor": next_monitor,
+        },
+        "work_queue": work_queue,
+        "scheduler": scheduler_json,
     })))
 }
 
@@ -84,27 +112,16 @@ async fn trigger_index(
     session: Session,
 ) -> Result<HttpResponse, ApiError> {
     auth::require_auth(&session)?;
-    let task_id = state.tasks.register("index");
-    let state = state.clone();
-    let tid = task_id.clone();
-
-    tokio::spawn(async move {
-        let cfg = state.config.read().await.clone();
-        state
-            .tasks
-            .update_progress(&tid, 0, 0, "Scanning library...");
-        let scan_config = shanty_index::ScanConfig {
-            root: cfg.library_path.clone(),
-            dry_run: false,
-            concurrency: cfg.indexing.concurrency,
-        };
-        match shanty_index::run_scan(state.db.conn(), &scan_config).await {
-            Ok(stats) => state.tasks.complete(&tid, format!("{stats}")),
-            Err(e) => state.tasks.fail(&tid, e.to_string()),
-        }
-    });
-
-    Ok(HttpResponse::Accepted().json(serde_json::json!({ "task_id": task_id })))
+    let payload = serde_json::json!({"scan_all": true});
+    let item = queries::work_queue::enqueue(
+        state.db.conn(),
+        WorkTaskType::Index,
+        &payload.to_string(),
+        None,
+    )
+    .await?;
+    state.workers.notify(WorkTaskType::Index);
+    Ok(HttpResponse::Accepted().json(serde_json::json!({ "work_item_id": item.id })))
 }
 
 async fn trigger_tag(
@@ -112,35 +129,18 @@ async fn trigger_tag(
     session: Session,
 ) -> Result<HttpResponse, ApiError> {
     auth::require_auth(&session)?;
-    let task_id = state.tasks.register("tag");
-    let state = state.clone();
-    let tid = task_id.clone();
-
-    tokio::spawn(async move {
-        let cfg = state.config.read().await.clone();
-        state
-            .tasks
-            .update_progress(&tid, 0, 0, "Preparing tagger...");
-        let mb = match shanty_tag::MusicBrainzClient::new() {
-            Ok(c) => c,
-            Err(e) => {
-                state.tasks.fail(&tid, e.to_string());
-                return;
-            }
-        };
-        let tag_config = shanty_tag::TagConfig {
-            dry_run: false,
-            write_tags: cfg.tagging.write_tags,
-            confidence: cfg.tagging.confidence,
-        };
-        state.tasks.update_progress(&tid, 0, 0, "Tagging tracks...");
-        match shanty_tag::run_tagging(state.db.conn(), &mb, &tag_config, None).await {
-            Ok(stats) => state.tasks.complete(&tid, format!("{stats}")),
-            Err(e) => state.tasks.fail(&tid, e.to_string()),
-        }
-    });
-
-    Ok(HttpResponse::Accepted().json(serde_json::json!({ "task_id": task_id })))
+    let conn = state.db.conn();
+    let untagged = queries::tracks::get_needing_metadata(conn).await?;
+    let mut count = 0;
+    for track in &untagged {
+        let payload = serde_json::json!({"track_id": track.id});
+        queries::work_queue::enqueue(conn, WorkTaskType::Tag, &payload.to_string(), None).await?;
+        count += 1;
+    }
+    if count > 0 {
+        state.workers.notify(WorkTaskType::Tag);
+    }
+    Ok(HttpResponse::Accepted().json(serde_json::json!({ "enqueued": count })))
 }
 
 async fn trigger_organize(
@@ -148,39 +148,31 @@ async fn trigger_organize(
     session: Session,
 ) -> Result<HttpResponse, ApiError> {
     auth::require_auth(&session)?;
-    let task_id = state.tasks.register("organize");
-    let state = state.clone();
-    let tid = task_id.clone();
-
-    tokio::spawn(async move {
-        let cfg = state.config.read().await.clone();
-        state
-            .tasks
-            .update_progress(&tid, 0, 0, "Organizing files...");
-        let org_config = shanty_org::OrgConfig {
-            target_dir: cfg.library_path.clone(),
-            format: cfg.organization_format.clone(),
-            dry_run: false,
-            copy: false,
-        };
-        match shanty_org::organize_from_db(state.db.conn(), &org_config).await {
-            Ok(stats) => {
-                let promoted = queries::wanted::promote_downloaded_to_owned(state.db.conn())
-                    .await
-                    .unwrap_or(0);
-                let msg = if promoted > 0 {
-                    format!("{stats} — {promoted} items marked as owned")
-                } else {
-                    format!("{stats}")
-                };
-                state.tasks.complete(&tid, msg);
-                let _ = enrich_all_watched_artists(&state).await;
-            }
-            Err(e) => state.tasks.fail(&tid, e.to_string()),
+    let conn = state.db.conn();
+    let mut count = 0u64;
+    let mut offset = 0u64;
+    loop {
+        let tracks = queries::tracks::list(conn, 500, offset).await?;
+        if tracks.is_empty() {
+            break;
         }
-    });
-
-    Ok(HttpResponse::Accepted().json(serde_json::json!({ "task_id": task_id })))
+        for track in &tracks {
+            let payload = serde_json::json!({"track_id": track.id});
+            queries::work_queue::enqueue(
+                conn,
+                WorkTaskType::Organize,
+                &payload.to_string(),
+                None,
+            )
+            .await?;
+            count += 1;
+        }
+        offset += 500;
+    }
+    if count > 0 {
+        state.workers.notify(WorkTaskType::Organize);
+    }
+    Ok(HttpResponse::Accepted().json(serde_json::json!({ "enqueued": count })))
 }
 
 async fn trigger_pipeline(
@@ -188,8 +180,8 @@ async fn trigger_pipeline(
     session: Session,
 ) -> Result<HttpResponse, ApiError> {
     auth::require_auth(&session)?;
-    let task_ids = crate::pipeline::spawn_pipeline(&state);
-    Ok(HttpResponse::Accepted().json(serde_json::json!({ "task_ids": task_ids })))
+    let pipeline_id = crate::pipeline::trigger_pipeline(&state).await?;
+    Ok(HttpResponse::Accepted().json(serde_json::json!({ "pipeline_id": pipeline_id })))
 }
 
 async fn get_task(
@@ -313,10 +305,13 @@ async fn skip_pipeline(
     session: Session,
 ) -> Result<HttpResponse, ApiError> {
     auth::require_admin(&session)?;
-    let mut sched = state.scheduler.lock().await;
-    sched.skip_pipeline = true;
-    sched.next_pipeline = None;
-    Ok(HttpResponse::Ok().json(serde_json::json!({"status": "skipped"})))
+    // Push next_run_at forward by one interval
+    let cfg = state.config.read().await;
+    let hours = cfg.scheduling.pipeline_interval_hours.max(1);
+    drop(cfg);
+    let next = chrono::Utc::now().naive_utc() + chrono::Duration::hours(i64::from(hours));
+    queries::scheduler_state::update_next_run(state.db.conn(), "pipeline", Some(next)).await?;
+    Ok(HttpResponse::Ok().json(serde_json::json!({"status": "skipped", "next_run": next})))
 }
 
 async fn skip_monitor(
@@ -324,10 +319,12 @@ async fn skip_monitor(
     session: Session,
 ) -> Result<HttpResponse, ApiError> {
     auth::require_admin(&session)?;
-    let mut sched = state.scheduler.lock().await;
-    sched.skip_monitor = true;
-    sched.next_monitor = None;
-    Ok(HttpResponse::Ok().json(serde_json::json!({"status": "skipped"})))
+    let cfg = state.config.read().await;
+    let hours = cfg.scheduling.monitor_interval_hours.max(1);
+    drop(cfg);
+    let next = chrono::Utc::now().naive_utc() + chrono::Duration::hours(i64::from(hours));
+    queries::scheduler_state::update_next_run(state.db.conn(), "monitor", Some(next)).await?;
+    Ok(HttpResponse::Ok().json(serde_json::json!({"status": "skipped", "next_run": next})))
 }
 
 async fn get_mb_status(
@@ -389,7 +386,7 @@ async fn trigger_mb_import(
             state.tasks.update_progress(
                 &tid,
                 i as u64,
-                4 + 4, // 4 downloads + 4 imports
+                4 + 4,
                 &format!("Downloading {filename}..."),
             );
             if let Err(e) =
@@ -413,7 +410,6 @@ async fn trigger_mb_import(
 
         let tid_clone = tid.clone();
         let state_clone = state.clone();
-        // Run import in blocking task since rusqlite is sync
         let result = tokio::task::spawn_blocking(move || {
             shanty_data::mb_import::run_import_at_path(&db_path, &data_dir, |msg| {
                 tracing::info!("{msg}");