Added the indexer
This commit is contained in:
Connor Johnstone
3
.gitmodules
vendored
3
.gitmodules
vendored
@@ -1,3 +1,6 @@
|
||||
[submodule "shanty-db"]
|
||||
path = shanty-db
|
||||
url = ssh://connor@git.rcjohnstone.com:2222/Shanty/db.git
|
||||
[submodule "shanty-index"]
|
||||
path = shanty-index
|
||||
url = ssh://connor@git.rcjohnstone.com:2222/Shanty/index.git
|
||||
|
||||
193
Cargo.lock
generated
193
Cargo.lock
generated
@@ -2,6 +2,12 @@
|
||||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "adler2"
|
||||
version = "2.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
|
||||
|
||||
[[package]]
|
||||
name = "ahash"
|
||||
version = "0.7.8"
|
||||
@@ -409,6 +415,15 @@ version = "2.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5"
|
||||
|
||||
[[package]]
|
||||
name = "crc32fast"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-queue"
|
||||
version = "0.3.12"
|
||||
@@ -468,6 +483,12 @@ dependencies = [
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "data-encoding"
|
||||
version = "2.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea"
|
||||
|
||||
[[package]]
|
||||
name = "der"
|
||||
version = "0.7.10"
|
||||
@@ -523,6 +544,27 @@ dependencies = [
|
||||
"subtle",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dirs"
|
||||
version = "6.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e"
|
||||
dependencies = [
|
||||
"dirs-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dirs-sys"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"option-ext",
|
||||
"redox_users",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "displaydoc"
|
||||
version = "0.2.5"
|
||||
@@ -599,6 +641,16 @@ version = "0.1.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
|
||||
|
||||
[[package]]
|
||||
name = "flate2"
|
||||
version = "1.1.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c"
|
||||
dependencies = [
|
||||
"crc32fast",
|
||||
"miniz_oxide",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "flume"
|
||||
version = "0.11.1"
|
||||
@@ -1111,6 +1163,32 @@ dependencies = [
|
||||
"scopeguard",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lofty"
|
||||
version = "0.22.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ca260c51a9c71f823fbfd2e6fbc8eb2ee09834b98c00763d877ca8bfa85cde3e"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"data-encoding",
|
||||
"flate2",
|
||||
"lofty_attr",
|
||||
"log",
|
||||
"ogg_pager",
|
||||
"paste",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lofty_attr"
|
||||
version = "0.11.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ed9983e64b2358522f745c1251924e3ab7252d55637e80f6a0a3de642d6a9efc"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "log"
|
||||
version = "0.4.29"
|
||||
@@ -1142,6 +1220,16 @@ version = "2.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
|
||||
|
||||
[[package]]
|
||||
name = "miniz_oxide"
|
||||
version = "0.8.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316"
|
||||
dependencies = [
|
||||
"adler2",
|
||||
"simd-adler32",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mio"
|
||||
version = "1.1.1"
|
||||
@@ -1170,6 +1258,15 @@ dependencies = [
|
||||
"tempfile",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "nu-ansi-term"
|
||||
version = "0.50.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-bigint"
|
||||
version = "0.4.6"
|
||||
@@ -1232,6 +1329,15 @@ dependencies = [
|
||||
"libm",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ogg_pager"
|
||||
version = "0.7.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6d6d1ca8364b84e0cf725eed06b1460c44671e6c0fb28765f5262de3ece07fdc"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "once_cell"
|
||||
version = "1.21.4"
|
||||
@@ -1288,6 +1394,12 @@ dependencies = [
|
||||
"vcpkg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "option-ext"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d"
|
||||
|
||||
[[package]]
|
||||
name = "ordered-float"
|
||||
version = "4.6.0"
|
||||
@@ -1350,6 +1462,12 @@ dependencies = [
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "paste"
|
||||
version = "1.0.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
|
||||
|
||||
[[package]]
|
||||
name = "pem-rfc7468"
|
||||
version = "0.7.0"
|
||||
@@ -1589,6 +1707,17 @@ dependencies = [
|
||||
"bitflags",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redox_users"
|
||||
version = "0.5.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac"
|
||||
dependencies = [
|
||||
"getrandom 0.2.17",
|
||||
"libredox",
|
||||
"thiserror",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "1.12.3"
|
||||
@@ -1726,6 +1855,15 @@ version = "1.0.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
|
||||
|
||||
[[package]]
|
||||
name = "same-file"
|
||||
version = "1.0.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
|
||||
dependencies = [
|
||||
"winapi-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "schannel"
|
||||
version = "0.1.29"
|
||||
@@ -2047,11 +2185,20 @@ dependencies = [
|
||||
name = "shanty-index"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"chrono",
|
||||
"clap",
|
||||
"dirs",
|
||||
"lofty",
|
||||
"sea-orm",
|
||||
"serde",
|
||||
"shanty-db",
|
||||
"tempfile",
|
||||
"thiserror",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
"walkdir",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2197,6 +2344,12 @@ dependencies = [
|
||||
"rand_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "simd-adler32"
|
||||
version = "0.3.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2"
|
||||
|
||||
[[package]]
|
||||
name = "simdutf8"
|
||||
version = "0.1.5"
|
||||
@@ -2732,6 +2885,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a"
|
||||
dependencies = [
|
||||
"once_cell",
|
||||
"valuable",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tracing-log"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3"
|
||||
dependencies = [
|
||||
"log",
|
||||
"once_cell",
|
||||
"tracing-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2741,12 +2906,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319"
|
||||
dependencies = [
|
||||
"matchers",
|
||||
"nu-ansi-term",
|
||||
"once_cell",
|
||||
"regex-automata",
|
||||
"sharded-slab",
|
||||
"smallvec",
|
||||
"thread_local",
|
||||
"tracing",
|
||||
"tracing-core",
|
||||
"tracing-log",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -2823,6 +2991,12 @@ dependencies = [
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "valuable"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"
|
||||
|
||||
[[package]]
|
||||
name = "vcpkg"
|
||||
version = "0.2.15"
|
||||
@@ -2835,6 +3009,16 @@ version = "0.9.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
|
||||
|
||||
[[package]]
|
||||
name = "walkdir"
|
||||
version = "2.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
|
||||
dependencies = [
|
||||
"same-file",
|
||||
"winapi-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasi"
|
||||
version = "0.11.1+wasi-snapshot-preview1"
|
||||
@@ -2954,6 +3138,15 @@ dependencies = [
|
||||
"wasite",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi-util"
|
||||
version = "0.1.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-core"
|
||||
version = "0.62.2"
|
||||
|
||||
267
ISSUES.md
267
ISSUES.md
@@ -4,273 +4,6 @@ This file contains all planned issues for the Shanty project. Each issue is self
|
||||
|
||||
---
|
||||
|
||||
## Issue #1: Initialize Cargo workspace and project scaffolding
|
||||
|
||||
**Labels:** `mvp`, `scaffolding`, `priority:high`
|
||||
|
||||
### Description
|
||||
|
||||
Shanty is a modular music management application built in Rust. The project is organized as a Cargo workspace where each component (indexing, tagging, organization, watching, downloading, searching, web interface, etc.) lives in its own crate. Each crate serves as both a library (for integration into the larger app) and a standalone binary with its own CLI.
|
||||
|
||||
This issue covers the initial project scaffolding:
|
||||
|
||||
1. Create a top-level `Cargo.toml` defining the workspace and listing all planned member crates
|
||||
2. Create stub crates for each planned component:
|
||||
- `shanty-index` — music file indexing and metadata extraction
|
||||
- `shanty-tag` — metadata tagging via online databases
|
||||
- `shanty-org` — file organization and renaming
|
||||
- `shanty-watch` — library watchlist management
|
||||
- `shanty-dl` — music downloading
|
||||
- `shanty-search` — online music search
|
||||
- `shanty-notify` — notifications (post-MVP, but scaffold now)
|
||||
- `shanty-playlist` — playlist generation (post-MVP, but scaffold now)
|
||||
- `shanty-serve` — music serving/streaming (post-MVP, but scaffold now)
|
||||
- `shanty-play` — built-in playback (post-MVP, but scaffold now)
|
||||
- `shanty-web` — web interface backend (Actix)
|
||||
- `shanty-db` — shared database schema and access layer
|
||||
3. Each crate should have a minimal `lib.rs` and `main.rs` (where applicable) with placeholder content
|
||||
4. Create a top-level `README.md` explaining the project structure
|
||||
5. Add a `.gitignore` appropriate for Rust projects
|
||||
6. Ensure `cargo build` and `cargo test` succeed on the empty workspace
|
||||
|
||||
### Acceptance Criteria
|
||||
|
||||
- [ ] `Cargo.toml` at the workspace root lists all member crates
|
||||
- [ ] Each crate directory exists with its own `Cargo.toml`, `src/lib.rs`, and (where applicable) `src/main.rs`
|
||||
- [ ] `cargo build --workspace` succeeds
|
||||
- [ ] `cargo test --workspace` succeeds (even if no tests yet)
|
||||
- [ ] `.gitignore` excludes `target/`, `.env`, and other standard Rust artifacts
|
||||
- [ ] `README.md` exists with a brief project description and crate listing
|
||||
|
||||
### Dependencies
|
||||
|
||||
None — this is the first issue.
|
||||
|
||||
---
|
||||
|
||||
## Issue #2: Design and implement the shared database schema crate (`shanty-db`)
|
||||
|
||||
**Labels:** `mvp`, `database`, `priority:high`, `indexing`
|
||||
|
||||
### Description
|
||||
|
||||
Shanty uses a shared database to store all music metadata, library state, watchlist entries, and more. Multiple crates need to read from and write to this database. To avoid tight coupling and schema drift, there should be a dedicated `shanty-db` crate that owns the schema, migrations, and provides a typed access layer.
|
||||
|
||||
This crate should:
|
||||
|
||||
1. **Choose and integrate an ORM/query builder** — recommend `diesel` or `sea-orm` with SQLite as the default backend (SQLite is lightweight, file-based, and appropriate for a self-hosted music app). The choice should support migrations and be ergonomic for other crates to use.
|
||||
2. **Define the core schema** covering at minimum:
|
||||
- `tracks` — file path, title, artist, album, album artist, track number, disc number, duration, genre, year, codec/format, bitrate, file size, fingerprint (nullable), added_at, updated_at
|
||||
- `albums` — name, album artist, year, genre, cover art path (nullable), musicbrainz_id (nullable)
|
||||
- `artists` — name, musicbrainz_id (nullable), monitored (bool), added_at
|
||||
- `watch_items` — references to artist/album/track the user wants to monitor, with status (wanted/available/downloaded)
|
||||
- `download_queue` — items pending download, with status, source URL, error info
|
||||
- `search_cache` — optional cache for online search results to avoid excessive API calls
|
||||
3. **Provide migrations** so the schema can evolve over time without losing data
|
||||
4. **Expose a connection pool and typed query functions** that other crates import
|
||||
|
||||
### Design Considerations
|
||||
|
||||
- The schema must be versioned and migratable. Other crates depend on `shanty-db` but should not define their own tables.
|
||||
- Consider using `r2d2` or the ORM's built-in connection pooling.
|
||||
- Keep the schema normalized but pragmatic — e.g., a track belongs to an album which belongs to an artist, but allow nullable foreign keys for partially-tagged files.
|
||||
- The crate should re-export relevant types (models, enums) so other crates don't need to depend on the ORM directly for basic type usage.
|
||||
|
||||
### Acceptance Criteria
|
||||
|
||||
- [ ] `shanty-db` crate compiles and is usable as a dependency by other workspace crates
|
||||
- [ ] SQLite database is created automatically if it doesn't exist
|
||||
- [ ] Migrations can be run programmatically (and via CLI if using diesel)
|
||||
- [ ] Core tables (`tracks`, `albums`, `artists`, `watch_items`, `download_queue`) exist after migration
|
||||
- [ ] Typed Rust structs exist for each table (insertable and queryable variants)
|
||||
- [ ] Connection pooling is set up
|
||||
- [ ] At least basic integration tests exist — create a DB, run migrations, insert a track, query it back
|
||||
- [ ] Schema version is tracked so future migrations can be applied incrementally
|
||||
|
||||
### Dependencies
|
||||
|
||||
- Issue #1 (workspace scaffolding)
|
||||
|
||||
---
|
||||
|
||||
## Issue #3: Implement music file scanning in `shanty-index`
|
||||
|
||||
**Labels:** `mvp`, `indexing`, `priority:high`
|
||||
|
||||
### Description
|
||||
|
||||
The `shanty-index` crate is responsible for scanning a directory tree of music files and extracting metadata from them. This is the foundation of Shanty — before anything else can happen, the app needs to know what music the user already has.
|
||||
|
||||
This issue covers:
|
||||
|
||||
1. **Directory scanning** — recursively walk a given directory path and identify music files by extension (`.mp3`, `.flac`, `.ogg`, `.opus`, `.m4a`, `.wav`, `.wma`, `.aac`, `.alac` at minimum). Use a crate like `walkdir` for efficient traversal.
|
||||
2. **Metadata extraction** — for each music file found, extract embedded metadata (ID3 tags for MP3, Vorbis comments for FLAC/OGG, etc.). Use a crate like `lofty` or `symphonia` that handles multiple formats uniformly. Extract at minimum: title, artist, album, album artist, track number, disc number, year/date, genre, duration, codec, bitrate.
|
||||
3. **Database insertion** — insert (or update) the extracted metadata into the `shanty-db` database. Handle the case where a file has already been indexed (update if file mtime changed, skip if unchanged). Create artist and album records as needed based on the metadata found.
|
||||
4. **CLI interface** — the `shanty-index` binary should accept:
|
||||
- A path to scan (required)
|
||||
- A path to the database file (optional, with a sensible default)
|
||||
- A `--dry-run` flag that reports what would be indexed without writing to DB
|
||||
- Verbosity flags
|
||||
5. **Concurrency** — scanning and metadata extraction should be parallelized. Use `rayon` or `tokio` tasks to process multiple files concurrently. This is important because large libraries can have tens of thousands of files.
|
||||
|
||||
### Design Considerations
|
||||
|
||||
- Files with missing metadata should still be indexed — store whatever is available and leave the rest as NULL. The tagging crate will fill in gaps later.
|
||||
- Track file modification time so re-indexing can be incremental (only process changed files).
|
||||
- Consider emitting structured progress information (e.g., files scanned, files indexed, errors) that the web UI can consume later.
|
||||
- The library API should be usable without the CLI — other crates (especially the web backend) will call indexing functions directly.
|
||||
|
||||
### Acceptance Criteria
|
||||
|
||||
- [ ] Given a directory containing music files, `shanty-index` scans recursively and finds all supported formats
|
||||
- [ ] Metadata is extracted from each file and stored in the database via `shanty-db`
|
||||
- [ ] Re-running the scan on the same directory is incremental — unchanged files are skipped
|
||||
- [ ] The CLI binary works with `shanty-index /path/to/music`
|
||||
- [ ] `--dry-run` mode works and outputs what would be indexed
|
||||
- [ ] Scanning is parallelized (measurably faster than sequential on large directories)
|
||||
- [ ] Files with partial/missing metadata are still indexed (with NULLs for missing fields)
|
||||
- [ ] Artist and album records are created/linked based on extracted metadata
|
||||
- [ ] Errors on individual files (corrupt, unreadable) are logged but don't halt the scan
|
||||
- [ ] Unit tests exist for metadata extraction logic
|
||||
- [ ] Integration test: create temp dir with test audio files, scan it, verify DB contents
|
||||
|
||||
### Dependencies
|
||||
|
||||
- Issue #1 (workspace scaffolding)
|
||||
- Issue #2 (shared database schema)
|
||||
|
||||
---
|
||||
|
||||
## Issue #4: Implement online metadata lookup in `shanty-tag`
|
||||
|
||||
**Labels:** `mvp`, `tagging`, `priority:high`
|
||||
|
||||
### Description
|
||||
|
||||
The `shanty-tag` crate is responsible for filling in missing or incorrect metadata on music files. The MVP approach is "look online first" — query online databases (primarily MusicBrainz) using whatever partial metadata is available (artist + title, album name, etc.) to find the correct tags.
|
||||
|
||||
This issue covers:
|
||||
|
||||
1. **MusicBrainz client** — implement a client that queries the MusicBrainz API to look up track/album/artist metadata. MusicBrainz has a free API with rate limiting (1 request/second for unauthenticated). The client should:
|
||||
- Search by artist + title to find a matching recording
|
||||
- Search by album name + artist to find a matching release
|
||||
- Retrieve full metadata for a matched recording/release (title, artist, album, track number, year, genre, cover art URL via Cover Art Archive, MusicBrainz IDs)
|
||||
- Respect rate limits (implement a rate limiter / request queue)
|
||||
- Handle API errors gracefully
|
||||
2. **Tag matching logic** — given a track from the database (which may have partial metadata), attempt to find a match online:
|
||||
- If artist + title are available, search for the recording
|
||||
- If only a filename is available, attempt to parse artist/title from the filename (common patterns like "Artist - Title.mp3")
|
||||
- Score potential matches by similarity to existing metadata (fuzzy string matching)
|
||||
- Allow a configurable confidence threshold — only apply tags if the match confidence is above the threshold
|
||||
3. **Database update** — when a match is found and accepted, update the track (and album/artist) records in `shanty-db` with the new metadata. Also update the MusicBrainz IDs for future reference.
|
||||
4. **File tag writing** — optionally write the updated metadata back to the actual music file's embedded tags (ID3, Vorbis comments, etc.). This should be an opt-in behavior since some users may not want their files modified.
|
||||
5. **CLI interface** — the `shanty-tag` binary should accept:
|
||||
- A path to the database (optional, with default)
|
||||
- `--all` to tag all untagged/partially-tagged tracks in the database
|
||||
- `--track <id>` to tag a specific track
|
||||
- `--dry-run` to show what would be changed without applying
|
||||
- `--write-tags` to enable writing tags back to files
|
||||
- `--confidence <0.0-1.0>` to set the match threshold (default ~0.8)
|
||||
|
||||
### Design Considerations
|
||||
|
||||
- The data backend should be trait-based so that alternative providers (Last.fm, Discogs, etc.) can be added later without changing the core logic. Define a `MetadataProvider` trait with methods like `search_recording`, `search_release`, `get_recording_details`, etc.
|
||||
- MusicBrainz requires a descriptive User-Agent header — use something like `Shanty/0.1.0 (https://github.com/your-repo)`.
|
||||
- Batch operations should be parallelized where possible, but respect API rate limits.
|
||||
- Store which provider supplied the metadata so the user knows the source.
|
||||
|
||||
### Acceptance Criteria
|
||||
|
||||
- [ ] MusicBrainz API client is implemented with proper rate limiting
|
||||
- [ ] Given a track with artist + title, the tagger finds a matching recording and retrieves full metadata
|
||||
- [ ] Fuzzy matching works — minor spelling differences don't prevent matches
|
||||
- [ ] Database records are updated with new metadata and MusicBrainz IDs
|
||||
- [ ] `--write-tags` actually writes metadata back into the music file
|
||||
- [ ] `--dry-run` shows proposed changes without applying them
|
||||
- [ ] Confidence threshold filtering works
|
||||
- [ ] `MetadataProvider` trait exists, and MusicBrainz is the first implementation
|
||||
- [ ] CLI interface works as specified
|
||||
- [ ] Errors from the API (rate limits, network issues, no results) are handled gracefully
|
||||
- [ ] Tests exist for matching logic (unit tests with mocked API responses)
|
||||
|
||||
### Dependencies
|
||||
|
||||
- Issue #1 (workspace scaffolding)
|
||||
- Issue #2 (shared database schema)
|
||||
- Issue #3 (music indexing — so there are tracks in the DB to tag)
|
||||
|
||||
---
|
||||
|
||||
## Issue #5: Implement music file organization in `shanty-org`
|
||||
|
||||
**Labels:** `mvp`, `organization`, `priority:medium`
|
||||
|
||||
### Description
|
||||
|
||||
The `shanty-org` crate is responsible for organizing music files into a clean directory structure and renaming them according to a configurable format. Many music libraries are a mess of random filenames and flat folders — this crate fixes that.
|
||||
|
||||
This issue covers:
|
||||
|
||||
1. **Directory structure generation** — given a target root directory and a format template, move/copy music files into the correct structure. The default structure should be:
|
||||
```
|
||||
{root}/{artist}/{album}/{track_number} - {title}.{ext}
|
||||
```
|
||||
For example: `Music/Pink Floyd/The Dark Side of the Moon/03 - Time.flac`
|
||||
|
||||
2. **Format templates** — support a simple template syntax for both directory structure and filename. Available variables should include at minimum:
|
||||
- `{artist}` — track or album artist
|
||||
- `{album_artist}` — album artist (falls back to artist)
|
||||
- `{album}` — album name
|
||||
- `{title}` — track title
|
||||
- `{track_number}` — zero-padded track number (e.g., "03")
|
||||
- `{disc_number}` — disc number
|
||||
- `{year}` — release year
|
||||
- `{genre}` — genre
|
||||
- `{ext}` — file extension
|
||||
|
||||
3. **Safe file operations** — this crate moves actual files, so it must be very careful:
|
||||
- Sanitize filenames (remove/replace characters invalid on the target filesystem)
|
||||
- Handle conflicts (what if two files resolve to the same target path?)
|
||||
- Support dry-run mode to preview changes before executing
|
||||
- Log all file moves so the operation could theoretically be reversed
|
||||
- Update the file path in the `shanty-db` database after moving
|
||||
|
||||
4. **CLI interface** — the `shanty-org` binary should accept:
|
||||
- `--source <dir>` — source directory (or use DB paths)
|
||||
- `--target <dir>` — target root directory
|
||||
- `--format <template>` — format template (with a sensible default)
|
||||
- `--dry-run` — preview changes without moving files
|
||||
- `--copy` vs `--move` — whether to copy or move files (default: move)
|
||||
- `--from-db` — organize all tracks known to the database
|
||||
|
||||
### Design Considerations
|
||||
|
||||
- This crate can operate independently of the database — if someone just has a directory of music files with embedded tags, it should be able to read those tags and organize the files. But when used with the database, it should read metadata from there (faster than re-reading tags) and update paths afterward.
|
||||
- Handle the case where metadata is incomplete — if there's no album, put the file in an "Unknown Album" folder; if no artist, use "Unknown Artist"; etc.
|
||||
- Empty directories left behind after moving files should be cleaned up.
|
||||
- On failure mid-operation, the state should be recoverable (already-moved files should still be tracked).
|
||||
|
||||
### Acceptance Criteria
|
||||
|
||||
- [ ] Files are moved/copied into the correct directory structure based on metadata
|
||||
- [ ] Format templates work with all specified variables
|
||||
- [ ] `--dry-run` shows planned moves without executing them
|
||||
- [ ] Filenames are sanitized for filesystem safety
|
||||
- [ ] Conflicts are detected and handled (e.g., appending a number)
|
||||
- [ ] Database file paths are updated after moving (when using `--from-db`)
|
||||
- [ ] Works standalone (reading embedded tags) without requiring the database
|
||||
- [ ] Empty source directories are cleaned up after moves
|
||||
- [ ] CLI interface works as specified
|
||||
- [ ] Integration test: create temp files with metadata, organize them, verify the resulting directory structure
|
||||
|
||||
### Dependencies
|
||||
|
||||
- Issue #1 (workspace scaffolding)
|
||||
- Issue #2 (shared database schema)
|
||||
- Issue #3 (music indexing — beneficial but not strictly required)
|
||||
|
||||
---
|
||||
|
||||
## Issue #6: Implement library watchlist management in `shanty-watch`
|
||||
|
||||
**Labels:** `mvp`, `watching`, `priority:medium`
|
||||
|
||||
1
shanty-index
Submodule
1
shanty-index
Submodule
Submodule shanty-index added at 884acfcd18
@@ -1,13 +0,0 @@
|
||||
[package]
|
||||
name = "shanty-index"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
description = "Music file indexing and metadata extraction for Shanty"
|
||||
|
||||
[dependencies]
|
||||
shanty-db = { path = "../shanty-db" }
|
||||
clap = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
@@ -1,4 +0,0 @@
|
||||
//! Music file indexing and metadata extraction.
|
||||
//!
|
||||
//! Scans a directory tree of music files, extracts embedded metadata (ID3, Vorbis
|
||||
//! comments, etc.), and stores everything in the Shanty database.
|
||||
@@ -1,3 +0,0 @@
|
||||
fn main() {
|
||||
println!("shanty-index: music file indexing and metadata extraction");
|
||||
}
|
||||
Reference in New Issue
Block a user