From 21d4fddb2fb47ade1f447b2daed95ffdce252756 Mon Sep 17 00:00:00 2001 From: Test User Date: Wed, 22 Jul 2026 10:58:11 +0200 Subject: [PATCH] Design the crabidy content store; add wire + fsdy foundations Stage 1-2 of the crabidy-store dev-flow (architecture/crabidy-store.md, quality/, plan/): one /crabidy provider replacing queues/bookmarks/ captures, with track tomls linking into a content-addressed store that de-duplicates by provider id and content hash. Additive, build stays green: - proto: Track.provider_item_id + is_captured; LibraryNode.is_captured; LibraryNodeChild.is_captured (swept all literals). - fsdy: Playable::Store + PlayableSpec.store, 5-way cardinality, from_track_store, Client.with_store_root + store resolution. - crabidy_store.rs: StoreSidecar/ProviderEntry/StoreIndex/CrabidyStore type + method surface (bodies stubbed for the implement stage). - supersede bookmarks/captures/capture-deletion docs. Co-Authored-By: Claude Opus 4.8 (1M context) --- architecture/bookmarks.md | 4 + architecture/capture-deletion.md | 4 + architecture/captures.md | 13 +- architecture/crabidy-store.md | 331 ++++++++++++++++++++++++++ cbd-tui/src/app/mod.rs | 9 + cbd-tui/src/app/now_playing.rs | 2 + cbd-tui/src/app/queue.rs | 2 + cbd-web/src/state.rs | 3 + crabidy-core/crabidy/v1/crabidy.proto | 15 ++ crabidy-core/src/lib.rs | 7 + crabidy-server/src/capture_store.rs | 2 + crabidy-server/src/crabidy_store.rs | 236 ++++++++++++++++++ crabidy-server/src/lib.rs | 3 + crabidy-server/src/playback.rs | 2 + crabidy-server/src/queue_store.rs | 2 + fsdy/src/lib.rs | 100 +++++++- plan/crabidy-store.md | 118 +++++++++ quality/crabidy-store.md | 94 ++++++++ tidaldy/src/lib.rs | 8 + tidaldy/src/models.rs | 2 + ytdy/src/lib.rs | 7 + 21 files changed, 953 insertions(+), 11 deletions(-) create mode 100644 architecture/crabidy-store.md create mode 100644 crabidy-server/src/crabidy_store.rs create mode 100644 plan/crabidy-store.md create mode 100644 quality/crabidy-store.md diff --git a/architecture/bookmarks.md b/architecture/bookmarks.md index 7cbbdc7..6761520 100644 --- a/architecture/bookmarks.md +++ b/architecture/bookmarks.md @@ -1,5 +1,9 @@ # Bookmarks: capturing library subtrees +> **Superseded** by `crabidy-store.md`: `/bookmarks` is folded into the single +> `/crabidy` provider; a `w`-save writes link tomls into a `/crabidy/` +> folder. Kept for the link-vs-store rationale. + ## Context and problem statement Queue persistence (architecture/queue-persistence.md) flattens the queue diff --git a/architecture/capture-deletion.md b/architecture/capture-deletion.md index a61eaab..4ebd521 100644 --- a/architecture/capture-deletion.md +++ b/architecture/capture-deletion.md @@ -1,5 +1,9 @@ # Capture deletion +> **Superseded** by `crabidy-store.md`: audio now lives in a shared store that +> track deletion never touches, so deletion on `/crabidy` goes through directly +> with no confirmation and no disk reclamation. This whole feature is removed. + Deleting under `/captures` reclaims disk: downloaded audio is the one library content that is expensive to recreate (slow, throttled downloads — architecture/youtube-rustypipe.md), so stale captures must be diff --git a/architecture/captures.md b/architecture/captures.md index 74fba2a..2949559 100644 --- a/architecture/captures.md +++ b/architecture/captures.md @@ -1,10 +1,13 @@ # Captures (downloaded subtrees) -> **Partially superseded** by `incremental-captures.md`: download captures -> are now incremental (no tmp-and-swap, re-capturing a name resumes it), -> uncapturable tracks are recorded as *skipped* tomls instead of being -> omitted, and the capture RPC streams progress. Bookmarks keep the -> tmp-and-swap described here. +> **Superseded** by `crabidy-store.md`: captures now link into a shared +> content-addressed store under `~/.local/share/crabidy/`, de-duplicated by +> provider id and by content hash, instead of downloading audio next to each +> toml; `/captures` folds into `/crabidy`. Before that, download captures were +> already **partially superseded** by `incremental-captures.md`: they became +> incremental (re-capturing a name resumes; no tmp-and-swap), uncapturable +> tracks are recorded as *skipped* tomls instead of omitted, and the capture +> RPC streams progress. Bookmarks kept the tmp-and-swap described here. ## Context and problem statement diff --git a/architecture/crabidy-store.md b/architecture/crabidy-store.md new file mode 100644 index 0000000..42344fd --- /dev/null +++ b/architecture/crabidy-store.md @@ -0,0 +1,331 @@ +# The `crabidy` provider and the content-addressed store + +## Context and problem statement + +Today the server exposes three hand-managed subtrees — `/queues`, `/bookmarks`, +`/captures` — each a separate `fsdy::Client` instance rooted under +`~/.config/crabidy/`. Bookmarks (`w`) write *link* tomls; captures (`W`) +download audio next to each toml (`capture-deletion.md`, +`incremental-captures.md`). This has three problems the user wants fixed +green-field (no data migration): + +1. **No de-duplication.** Capturing the same track from two places (a playlist + and a search, or two saved queues) downloads and stores the audio twice. + Re-capturing after a reorder re-downloads everything. +2. **Audio lives next to metadata.** Deleting a capture folder must carefully + remove downloaded audio from disk behind a confirmation + (`capture-deletion.md`), because the audio is only referenced from that one + folder. This couples deletion to expensive-payload bookkeeping. +3. **Three UI concepts** (queues, bookmarks, captures) for what the user thinks + of as "my crabidy stuff." They want one provider. + +This design replaces all three with **one filesystem provider, `/crabidy`**, +whose track tomls *link into a single content-addressed store* of playable +files. Captures de-duplicate by provider identity and by content hash; deletion +becomes a plain toml removal that never touches the store. + +This **supersedes** `bookmarks.md`, `captures.md`, `capture-deletion.md`, and +the on-disk/resumption parts of `incremental-captures.md` (the skipped-track and +progress-stream parts of that doc survive; see D9). + +## Assumptions (confirmed by the request) + +- **Green-field.** No migration of existing `~/.config/crabidy/{queues, + bookmarks,captures}` data. On first run the new locations are simply empty. +- **Two roots, split by XDG kind.** The store (playable audio + its sidecars) + is *data* → `~/.local/share/crabidy/` (`dirs::data_dir()`). The tomls (the + `/crabidy` tree) are *state* → `~/.local/state/crabidy/` (`dirs::state_dir()`). +- **Single server, single writer.** One process owns both roots; capture + mutations are serialized. No cross-host concurrent writers. +- **The store never shrinks automatically.** Deleting a toml never deletes store + audio (D7). Orphaned store entries are accepted; a garbage collector is future + work (D10). +- **`current` stays special.** The live queue is still mirrored to a reserved, + user-untouchable folder — now `/crabidy/current`, flat as before. + +## D1 — One provider: `/crabidy` + +`ProviderOrchestrator` drops the `queues`/`bookmarks`/`captures` fields and gains +one `crabidy` field: an `fsdy::Client` rooted at `~/.local/state/crabidy/`, +mounted at `/crabidy`, built `with_editable_top_level(&["current"])` (top-level +saves are renamable/deletable; `current` is reserved) `.with_downloadable_nodes()` +`.with_deletable_tree()` (every node deletes directly — see D7). A companion +writer, `CrabidyStore`, owns *both* roots and all mutation. + +- Top-level folders under `/crabidy` are **user saves**, each created by `w`/`W`. +- **Saved queues are flat**; **saved library subtrees preserve their structure** + (falls out of walking the source — a flat queue yields a flat save). +- `/crabidy/current` is the live-queue mirror the playback loop keeps in sync + (replaces `/queues/current`). It is flat and reserved. + +Non-toml files in a save folder (e.g. client log files that also land in +`~/.local/state/crabidy/`) are ignored by listing, as today — the provider only +surfaces subdirectories and `*.cbd-track.toml` files. (Optional tidy-up, not +required: move client logs to `~/.local/state/crabidy/logs/`.) + +```d2 +direction: right +orchestrator: ProviderOrchestrator (root, path-routed) { + tidal + youtube + fs + crabidy: /crabidy (fsdy::Client, read + delete) +} +state: "~/.local/state/crabidy/\n(toml tree)" { shape: cylinder } +share: { + label: "~/.local/share/crabidy/\ncontent store: audio + sidecars" + shape: cylinder +} +store_writer: CrabidyStore (single writer, owns both roots) { + index: "StoreIndex\n(provider-id → entry,\nhash → entry)" +} +orchestrator.crabidy -> state: lists tomls +orchestrator.crabidy -> share: resolves Store playables +store_writer -> state: writes save folders / track tomls +store_writer -> share: writes audio + sidecars +store_writer.index -> share: derived by scanning sidecars at open +``` + +## D2 — The store and its sidecars + +`~/.local/share/crabidy/` is a **flat** directory. Each unique playable is a +pair: + +- `` — the audio file, named after the source's natural name (a local + file's basename, else a sanitized `.<ext>` with the extension from the + download's `Content-Type`/URL). On name collision with *different* content, + append a numeral: `song.flac`, `song (2).flac`, … (identical content never + reaches naming — it de-dupes first, D4). +- `<name>.cbd-store.toml` — the sidecar, the store entry's metadata: + +```toml +# song.flac.cbd-store.toml +hash = "blake3:1f0c…" # content hash of the audio file + +[[provider]] # one entry per provider-id that maps here +provider = "tidal" # provider name (path root of the source) +id = "125169484" # provider-internal id (see D3) +title = "Bohemian Rhapsody" +artist = "Queen" +duration = 355 +aliases = ["Bohemian Rhapsody (Remastered)"] # other titles for this id +[provider.album] +title = "A Night at the Opera" +release_date = "1975-11-21" + +[[provider]] # same content reached via a second identity +provider = "youtube" +id = "fJ9rUzIMcZQ" +title = "Queen – Bohemian Rhapsody (Official Video)" +``` + +The **sidecars are the single source of truth** — there is no separate persisted +index file. `CrabidyStore` builds an in-memory `StoreIndex` by scanning +`*.cbd-store.toml` at open and updates it on every write: + +- `by_provider_id: HashMap<(provider, id), StoreRef>` +- `by_hash: HashMap<Hash, StoreRef>` + +A `StoreRef` is the store `<name>` (which is the toml link target and the sidecar +key). Lookups are O(1). + +> **"grep without shelling out."** The request describes searching sidecars +> for a provider id / hash like ripgrep, in-process. The `StoreIndex` *is* it, +> memoized: it is derived by reading the sidecars in-process at open (no shell, +> no external index), so the store stays self-describing. A live +> content-scan-then-parse (via the `grep-searcher` crate) was considered as the +> literal realization; rejected because an in-memory map built once is simpler, +> strictly faster for repeated captures in a session, and needs no new +> dependency. If the store ever grows beyond memory, a lazy content-scan is the +> fallback (D10). + +## D3 — Provider identity on the wire + +The store keys on a **provider-internal id** — "the id that clearly identifies +the item inside the provider," which the path cannot be (the same item is +reachable via a playlist, a search, an album…). Today no such id exists; identity +is the path string. We add it to the wire model: + +- `Track.provider_item_id` (proto field 7, `string`) — set by the owning + provider when it produces a `Track`. Empty when unknown. + - **tidal** → the numeric track id. + - **youtube** → the video id. + - **fs** → the source file's canonical absolute path (two `/fs` tomls pointing + at the same file share an id → they de-dupe). + - **crabidy** (already store-backed) → the store `<name>`; used only to detect + "already captured" (D5). +- The **provider name** is the first path segment of the (resolved) source track + path — `to_track` already rewrites a link track's `path` to its target, so a + queued link routes to the real provider and carries that provider's id. + +For the capture indicator (D8) we also add: + +- `Track.is_captured` (field 8, `bool`) — the server sets it at listing time when + the store index has an entry for this track's `(provider, id)` **or** its + playable is already a store link. Cheap (one hashmap lookup) and works while + browsing *any* provider, so you can see what you have captured. +- `LibraryNode.is_captured` (field 10) and `LibraryNodeChild.is_captured` + (field 8) — a node is captured iff all its tracks and child nodes are captured + (D8 covers how the `/crabidy` provider computes this cheaply). + +## D4 — Capturing one track: the de-dup flow + +`store = true` means the track links into the store; `link` means a bookmark link +to the source provider. Capture (`W`) produces store links; bookmark (`w`) +produces plain links and never touches the store. + +For each source track a capture walk visits, in order: + +```d2 +direction: down +start: "resolve source track\n(provider, id, natural name)" +already: "already store-backed?\n(Store playable, or fs file under the store root)" +byid: "index.by_provider_id[(provider,id)] ?" +getbytes: "obtain bytes\n(download → temp, or local file)" +byhash: "index.by_hash[hash(bytes)] ?" +newentry: "NEW: copy into store\n(+numeral) + write sidecar" +addid: "add provider to sidecar,\ndiscard the temp copy" +writetoml: "write track toml with playable.store = <name>" + +start -> already +already -> writetoml: "yes → reuse target store name (no-op copy)" +already -> byid: "no" +byid -> writetoml: "HIT → reuse; record alias if title differs" +byid -> getbytes: "MISS" +getbytes -> byhash +byhash -> addid: "HIT (same content, new identity)" +byhash -> newentry: "MISS" +addid -> writetoml +newentry -> writetoml +``` + +1. **Already store-backed?** If the source track's playable is a `Store` link, or + it is an `/fs` file whose path is already under the store root, there is + nothing to fetch — the save's toml links to the same store `<name>`. (This is + the "capture on an already-captured fs item → do nothing" case.) +2. **Provider-id lookup.** `index.by_provider_id[(provider, id)]` — a hit means + we already have this exact provider item. Point the save's toml at that store + entry. If the current title differs from the stored one, append it to that + entry's `aliases`. **No download.** (Common re-capture path — makes redoing a + save cheap.) +3. **Miss → obtain bytes.** Streamed source: windowed HTTP download to a temp + file (unchanged mechanics). Local source (`/fs` pointing at a normal file, not + under the store): the file is the bytes. Hash the bytes (blake3). +4. **Hash lookup.** `index.by_hash[hash]` — a hit means identical content is + already stored under some other identity. **Add** a `[[provider]]` entry to + that sidecar, **discard** the temp download (or do not copy the fs file), + point the toml at the existing store name. No duplicate. +5. **Miss → new store entry.** Choose a store name from the natural name + (+numeral on collision), move the temp file (or copy the fs file) into the + store, write the sidecar with `hash` and the first `[[provider]]` entry, update + the index. The `/fs` case *copies* — the original music-folder file stays put. + +The byte budget (`DOWNLOAD_CAPS.max_bytes`) still counts bytes fetched *this run* +(hits cost nothing), so dedup makes big saves cheaper, never starves them. + +## D5 — Save (`w`/`W`) and conflict handling + +A **save** takes a *source* (a live-queue snapshot or a library-node path) and a +*mode* (`Link` for `w`, `Capture` for `W`) and writes a new top-level folder +`/crabidy/<name>`: + +- **`w`** on a library node or the queue → folder of **link** tomls + (`fsdy::TrackFile::from_track`), the bookmark semantics, no store, no audio. +- **`W`** on a library node or the queue → `w` **plus** run the D4 capture per + track; tomls carry `playable.store`. Works on both library nodes and the queue. + +**Conflict = refuse.** If `/crabidy/<name>` already exists, do nothing and warn +`name "<name>" already exists`; the user deletes the old folder and saves again. +This replaces the old overwrite (bookmark tmp-swap) and merge-by-name (capture +resume). The save is still built in a hidden `.tmp-<name>` sibling and swapped in +atomically, so a crashed/failed run leaves **no** blocking partial folder and the +name stays free to retry — while any audio already committed to the store +persists and makes the retry fast via D4. Resumability thus moves from the folder +to the store; the folder is all-or-nothing. + +`current` is exempt: the playback loop overwrites it on every queue change; the +user cannot save over the reserved name `current`. + +## D6 — RPC surface + +- **`CaptureLibraryNode(path, name, download)`** stays and now covers *all four* + gestures: `w`/`W` on a library node (`path = /tidal/...`), and `w`/`W` on the + queue (`path = /crabidy/current`). `download=false` → `Link`, `true` → + `Capture`. Validation errors (bad name, conflict, source not downloadable) + return synchronously; progress streams via the existing `CaptureProgress` + update (unchanged, D9). +- **`SaveQueue(name)` is removed.** Queue save/capture both go through + `CaptureLibraryNode` on `/crabidy/current`. The TUI's `w`-on-queue + (`QueueSaveAs`) and `W`-on-queue (`QueueDownloadCapture`) both emit a + `CaptureNode{ path: "/crabidy/current", name, download }`. +- `DeleteLibraryNode(path)` unchanged in shape; behavior simplified (D7). + +## D7 — Deletion + +Deletion on `/crabidy` (the only writable fs provider) **goes through directly, +no confirmation, and never touches the store**: + +- Delete a track → remove its `.cbd-track.toml` only. +- Delete a folder → `remove_dir_all` of the toml folder only. + +The existing `fsdy::delete_track_file` guard — "only delete the referenced audio +if it is contained under the instance root" — already makes this safe: a +`Store` playable resolves under `~/.local/share/…`, which is *outside* the +`/crabidy` toml root at `~/.local/state/…`, so the audio is never deleted. And +`w`-saves are links with no audio at all. + +Consequences: drop the TUI's `delete_needs_confirmation` / `ConfirmDelete` path +(the whole `capture-deletion.md` confirmation feature is gone — nothing expensive +is destroyed anymore) and drop `with_deletable_tree`'s audio-removal branch usage +for this provider (folder/toml removal remains). + +## D8 — UI: collapse to one provider, mark captured nodes + +- `/queues`, `/bookmarks`, `/captures` disappear from the root; one `/crabidy` + child appears (title `crabidy`). Inside it, `w`-saves (links) and `W`-saves + (store-backed) coexist, distinguished by the captured marker. +- **Captured marker:** a captured row is prefixed with `|` as the *first + character of the row* (before the selection padding), e.g. `|Bohemian…`. + - A **track** row is captured per `Track.is_captured` (D3): store-backed, or its + `(provider, id)` is in the store index — visible even while browsing tidal. + - A **node/child** row is captured iff all its tracks and child nodes are + captured. The `/crabidy` provider computes a node's own `is_captured` when it + lists it (it reads every track toml anyway → all `Store`?). To mark child + *folders* in a parent listing without deep recursion, a save records its mode + in a one-line marker at the save root written by `w`/`W`; nested folders under + a `W`-save are captured by construction. Exact recursion depth is an + implementation detail (see plan) — the invariant is "captured = fully local." +- The existing capture **progress** lines (`capturing <name> 12/34 …`) stay as-is. + +## D9 — What carries over from `incremental-captures.md` + +Kept: the `Skipped` playable (D1 there), skipped tracks in the queue and playback +skipping them (D3), the `CaptureProgress` stream and accept-then-stream RPC (D4), +and the focused-selection contrast fix (D7). A source that genuinely cannot be +fetched still writes a skipped toml. What changes: the *store write mode* (audio +now goes to the shared store, tomls carry `store`, dedup per D4) and *resume +semantics* (folder is atomic; store provides the savings, per D5). + +## D10 — Out of scope / future + +- **Store garbage collection.** Nothing reclaims store entries whose last + referencing toml was deleted. A future GC would scan all `/crabidy` tomls for + live `store` names and remove unreferenced pairs; needs its own design. +- **Lazy content-scan** instead of the in-memory index, for stores too large to + index in memory (D2). +- **Cross-provider captured marking of folders** (e.g. a tidal album shown + captured) beyond the cheap track-level lookup. +- **Cancelling a running capture** (unchanged from prior scope). + +## Risks + +- **Wasted download on a hash-only match** (provider-id missed but content is + identical): we download, then discard. Unavoidable for content dedup; the + provider-id path avoids it in the common case. +- **Natural-name collisions** across unrelated tracks are handled by the numeral + suffix; the store name is opaque to users (only the toml title shows in UI). +- **Index/disk skew** if something outside the server edits the store: the index + is rebuilt at every start, and the server is the sole writer, so skew is + bounded to a single run — acceptable. +- **blake3 dependency** added (fast, maintained, no C toolchain). Alternative + `sha2` rejected for speed; hashing whole tracks is on the capture hot path. diff --git a/cbd-tui/src/app/mod.rs b/cbd-tui/src/app/mod.rs index 8227d20..781a4f5 100644 --- a/cbd-tui/src/app/mod.rs +++ b/cbd-tui/src/app/mod.rs @@ -793,6 +793,7 @@ mod tests { is_creatable: true, is_downloadable: false, tracks_deletable: false, + is_captured: false, } } @@ -822,6 +823,7 @@ mod tests { is_creatable: false, is_downloadable: false, tracks_deletable: false, + is_captured: false, } } @@ -1152,11 +1154,14 @@ mod tests { duration: None, album: None, is_skipped: false, + provider_item_id: String::new(), + is_captured: false, }], is_queable: true, is_creatable: false, is_downloadable: false, tracks_deletable: true, + is_captured: false, } } @@ -1368,6 +1373,8 @@ mod tests { duration: None, album: None, is_skipped: false, + provider_item_id: String::new(), + is_captured: false, }; let listing = |downloadable| LibraryNode { tracks: vec![track.clone()], @@ -1426,6 +1433,8 @@ mod tests { duration: None, album: None, is_skipped: false, + provider_item_id: String::new(), + is_captured: false, }], resolving: false, } diff --git a/cbd-tui/src/app/now_playing.rs b/cbd-tui/src/app/now_playing.rs index 48bfd5b..2ce3fea 100644 --- a/cbd-tui/src/app/now_playing.rs +++ b/cbd-tui/src/app/now_playing.rs @@ -293,6 +293,8 @@ mod tests { duration: None, album: None, is_skipped: false, + provider_item_id: String::new(), + is_captured: false, }), spectrum: Vec::new(), spectrum_enabled: true, diff --git a/cbd-tui/src/app/queue.rs b/cbd-tui/src/app/queue.rs index d48707b..2ea3bb0 100644 --- a/cbd-tui/src/app/queue.rs +++ b/cbd-tui/src/app/queue.rs @@ -233,6 +233,8 @@ mod tests { duration: None, album: None, is_skipped: false, + provider_item_id: String::new(), + is_captured: false, }) .collect(), resolving, diff --git a/cbd-web/src/state.rs b/cbd-web/src/state.rs index 56ee09e..4eca13d 100644 --- a/cbd-web/src/state.rs +++ b/cbd-web/src/state.rs @@ -376,12 +376,15 @@ mod tests { duration: None, album: None, is_skipped: false, + provider_item_id: String::new(), + is_captured: false, }) .collect(), is_queable: true, is_creatable: false, is_downloadable: false, tracks_deletable: false, + is_captured: false, } } diff --git a/crabidy-core/crabidy/v1/crabidy.proto b/crabidy-core/crabidy/v1/crabidy.proto index fa70670..8e95ab1 100644 --- a/crabidy-core/crabidy/v1/crabidy.proto +++ b/crabidy-core/crabidy/v1/crabidy.proto @@ -254,6 +254,9 @@ message LibraryNodeChild { bool is_deletable = 6; // This node allows download captures (CaptureLibraryNode with download). bool is_downloadable = 7; + // Every track and child node below this child is captured (fully local in + // the content store). Clients mark captured rows. + bool is_captured = 8; } message QueueModifiers { @@ -306,6 +309,14 @@ message Track { // The track has no playable audio (a capture recorded its source as // uncapturable). Clients mark it; playback skips it. bool is_skipped = 6; + // Provider-internal id that identifies this item inside its provider, + // independent of the path it was reached by (playlist, search, album…). + // Set by the owning provider; empty when unknown. Keys the content store. + string provider_item_id = 7; + // The content store already holds this track (by provider id or, once + // captured, as a store-backed playable). Set at listing time; clients + // mark captured rows. See architecture/crabidy-store.md. + bool is_captured = 8; } message LibraryNode { @@ -324,4 +335,8 @@ message LibraryNode { // This node's listed tracks may be deleted (see DeleteLibraryNode) — // like is_downloadable, tracks inherit the node's flag. bool tracks_deletable = 9; + // Every track and child node below this node is captured (fully local in + // the content store). Clients mark captured nodes. See + // architecture/crabidy-store.md. + bool is_captured = 10; } diff --git a/crabidy-core/src/lib.rs b/crabidy-core/src/lib.rs index 7e53bed..b6ef03e 100644 --- a/crabidy-core/src/lib.rs +++ b/crabidy-core/src/lib.rs @@ -213,6 +213,7 @@ impl LibraryNode { is_creatable: false, is_downloadable: false, tracks_deletable: false, + is_captured: false, } } } @@ -231,6 +232,7 @@ impl LibraryNodeChild { is_editable: false, is_deletable: false, is_downloadable: false, + is_captured: false, } } } @@ -365,12 +367,15 @@ mod tests { duration: None, album: None, is_skipped: false, + provider_item_id: String::new(), + is_captured: false, }) .collect(), is_queable, is_creatable: false, is_downloadable: false, tracks_deletable: false, + is_captured: false, } } @@ -401,6 +406,8 @@ mod tests { duration: None, album: None, is_skipped: false, + provider_item_id: String::new(), + is_captured: false, }) } fn get_lib_root(&self) -> LibraryNode { diff --git a/crabidy-server/src/capture_store.rs b/crabidy-server/src/capture_store.rs index f62691f..b816bfc 100644 --- a/crabidy-server/src/capture_store.rs +++ b/crabidy-server/src/capture_store.rs @@ -310,6 +310,8 @@ mod tests { duration: Some(10), album: None, is_skipped: false, + provider_item_id: String::new(), + is_captured: false, } } } diff --git a/crabidy-server/src/crabidy_store.rs b/crabidy-server/src/crabidy_store.rs new file mode 100644 index 0000000..cce1e7d --- /dev/null +++ b/crabidy-server/src/crabidy_store.rs @@ -0,0 +1,236 @@ +//! The content-addressed store and the writer behind the single `/crabidy` +//! provider. +//! +//! This module replaces `bookmark_store`, `capture_store`, and the persistence +//! half of `queue_store`. It owns two roots (see +//! `architecture/crabidy-store.md`): +//! +//! - the **toml tree** at `~/.local/state/crabidy/` — the `/crabidy` provider's +//! folders of `*.cbd-track.toml` files, mounted read/delete by an +//! [`fsdy::Client`]; +//! - the **content store** at `~/.local/share/crabidy/` — a flat directory of +//! audio files, each paired with a `<name>.cbd-store.toml` sidecar recording +//! the content hash and every provider identity that maps to it. +//! +//! Captures de-duplicate by provider id first, then by content hash +//! (`architecture/crabidy-store.md` D4); saves are atomic and refuse to +//! overwrite an existing name (D5); track deletion never touches the store +//! (D7). +//! +//! Stage-2 (api-design) note: type definitions are real; method bodies are +//! `todo!()` stubs finalized by the implement stage. + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; + +use crabidy_core::proto::crabidy::Track; +use crabidy_core::ProviderClient; +use serde::{Deserialize, Serialize}; +use tokio::sync::Mutex; + +use crate::capture::{CaptureError, Downloader, Progress}; + +/// The reserved top-level folder holding the live-queue mirror; the user +/// cannot save over it. Replaces the old `/queues/current`. +pub const CURRENT_NAME: &str = "current"; + +/// The single library segment this provider owns. +pub const CRABIDY_PROVIDER_ROOT: &str = "/crabidy"; + +/// Whether a save writes bookmark links (`w`) or captures audio into the +/// store (`W`). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SaveMode { + /// `w`: track tomls link back to the source provider; no store, no audio. + Link, + /// `W`: track tomls carry a [`fsdy::Playable::Store`] entry; audio is + /// fetched into the content store, de-duplicated (D4). + Capture, +} + +/// Errors from opening or mutating the store that are not already a +/// [`CaptureError`]. +#[derive(Debug, thiserror::Error)] +pub enum StoreError { + #[error("no state/data directory available")] + NoDir, + #[error(transparent)] + Io(#[from] std::io::Error), + #[error("store sidecar is not valid TOML: {0}")] + Toml(#[from] toml::de::Error), +} + +/// One `(provider, id)` identity that resolves to a store entry, plus the +/// metadata that identity carried when first seen. Serialized as a +/// `[[provider]]` array element of a sidecar. +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ProviderEntry { + /// Provider name (the first path segment of the source track). + pub provider: String, + /// Provider-internal id (`Track.provider_item_id`). + pub id: String, + /// Title as seen from this identity. + pub title: String, + #[serde(default)] + pub artist: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub duration: Option<u32>, + #[serde(skip_serializing_if = "Option::is_none")] + pub album: Option<fsdy::AlbumMeta>, + /// Other titles later seen for the same `(provider, id)` (D4 step 2). + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub aliases: Vec<String>, +} + +/// The `<name>.cbd-store.toml` sidecar: the store entry's content hash and all +/// provider identities that map to it. The sidecars are the single source of +/// truth; [`StoreIndex`] is derived from them (D2). +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct StoreSidecar { + /// Content hash of the paired audio file, e.g. `"blake3:1f0c…"`. + pub hash: String, + /// Every identity mapping here; at least one. + #[serde(rename = "provider", default)] + pub providers: Vec<ProviderEntry>, +} + +/// The store `<name>` a track toml links to (the sidecar key and audio file +/// name). A bare file name, no separators. +pub type StoreName = String; + +/// In-memory index over the store's sidecars, derived by scanning +/// `*.cbd-store.toml` at open and updated on every write. No separate +/// persisted index exists — this *is* the "grep the sidecars" search, memoized +/// (D2). +#[derive(Debug, Default)] +pub struct StoreIndex { + by_provider_id: HashMap<(String, String), StoreName>, + by_hash: HashMap<String, StoreName>, +} + +impl StoreIndex { + /// Builds the index by reading every sidecar under `store_root`. + pub async fn scan(store_root: &Path) -> Result<Self, StoreError> { + let _ = store_root; + todo!("scan *.cbd-store.toml, populate by_provider_id and by_hash") + } + + /// The store entry for a `(provider, id)` identity, if any (D4 step 2). + pub fn by_provider_id(&self, provider: &str, id: &str) -> Option<&StoreName> { + self.by_provider_id + .get(&(provider.to_string(), id.to_string())) + } + + /// The store entry for a content hash, if any (D4 step 4). + pub fn by_hash(&self, hash: &str) -> Option<&StoreName> { + self.by_hash.get(hash) + } + + /// Records a freshly written or updated sidecar into the index. + pub fn insert(&mut self, name: &StoreName, sidecar: &StoreSidecar) { + let _ = (name, sidecar); + todo!("insert hash and every (provider,id) → name") + } +} + +/// Owns both store roots and serializes all mutation. +#[derive(Debug)] +pub struct CrabidyStore { + /// `~/.local/state/crabidy/` — the `/crabidy` toml tree. + tree_root: PathBuf, + /// `~/.local/share/crabidy/` — the content store (audio + sidecars). + store_root: PathBuf, + /// Serializes capture mutations; also guards the derived index. + index: Mutex<StoreIndex>, + /// Shared HTTP client for download captures. + downloader: Downloader, +} + +impl CrabidyStore { + /// The default toml-tree root: `dirs::state_dir()/crabidy`. + pub fn default_tree_root() -> Option<PathBuf> { + dirs::state_dir().map(|d| d.join("crabidy")) + } + + /// The default content-store root: `dirs::data_dir()/crabidy`. + pub fn default_store_root() -> Option<PathBuf> { + dirs::data_dir().map(|d| d.join("crabidy")) + } + + /// Opens the store: creates both roots, builds the [`StoreIndex`] from the + /// sidecars, and prepares the downloader. + pub async fn open(tree_root: PathBuf, store_root: PathBuf) -> Result<Self, StoreError> { + let _ = (&tree_root, &store_root); + todo!("create_dir_all both roots; StoreIndex::scan; Downloader::new") + } + + /// The toml-tree root the `/crabidy` [`fsdy::Client`] mounts. + pub fn tree_dir(&self) -> &Path { + &self.tree_root + } + + /// The content-store root the `/crabidy` client resolves store playables + /// against (via [`fsdy::Client::with_store_root`]). + pub fn store_dir(&self) -> &Path { + &self.store_root + } + + /// Overwrites the reserved `current` folder with the live queue as flat + /// link tomls. Called by the playback loop on every queue change. + pub async fn persist_current(&self, tracks: &[Track]) -> Result<(), CaptureError> { + let _ = tracks; + todo!("write /crabidy/current/*.cbd-track.toml (flat, links); replace atomically") + } + + /// Validates a save request synchronously so the RPC can accept/reject + /// before the (possibly long) walk: name legality, name-conflict + /// (D5: existing → refuse), and — for `Capture` — the source's download + /// blessing. + pub async fn validate( + &self, + client: &(dyn ProviderClient + Send + Sync), + source_path: &str, + name: &str, + mode: SaveMode, + ) -> Result<(), CaptureError> { + let _ = (client, source_path, name, mode); + todo!("validate_folder_name; reject existing name; Capture ⇒ source is_downloadable") + } + + /// Runs a save: walks `source_path` via `client`, writing a new atomic + /// `/crabidy/<name>` folder. `Link` writes bookmark links; `Capture` runs + /// the per-track de-dup flow (D4) into the content store and links the + /// tomls to store entries. Streams `progress`. + pub async fn save( + &self, + client: &(dyn ProviderClient + Send + Sync), + source_path: &str, + name: &str, + mode: SaveMode, + progress: Progress, + ) -> Result<(), CaptureError> { + let _ = (client, source_path, name, mode, progress); + todo!("enumerate source; per track link or capture-with-dedup; atomic swap into place") + } + + /// Marks the tracks of a wire node captured (`Track.is_captured`, + /// `LibraryNode.is_captured`) by consulting the index — cheap, works for + /// any provider (D3/D8). Called by the orchestrator after listing. + pub async fn annotate_captured(&self, node: &mut crabidy_core::proto::crabidy::LibraryNode) { + let _ = node; + todo!("set is_captured per track via index; node.is_captured when all captured") + } +} + +/// Hashes a file's contents into the sidecar `hash` string (`"blake3:…"`). +pub async fn hash_file(path: &Path) -> Result<String, StoreError> { + let _ = path; + todo!("stream the file through blake3, hex-encode with a blake3: prefix") +} + +/// Chooses a store file name from a source's natural name, appending ` (N)` on +/// collision with different content (D2/D4). +pub fn unique_store_name(store_root: &Path, natural_name: &str) -> StoreName { + let _ = (store_root, natural_name); + todo!("return natural_name, or `stem (N)ext` if the name is taken") +} diff --git a/crabidy-server/src/lib.rs b/crabidy-server/src/lib.rs index 5e7656c..59f37ad 100644 --- a/crabidy-server/src/lib.rs +++ b/crabidy-server/src/lib.rs @@ -5,6 +5,7 @@ pub mod web; pub mod capture; pub mod capture_store; +pub mod crabidy_store; pub mod playback; pub mod provider; pub mod queue_store; @@ -626,6 +627,8 @@ mod tests { duration: None, album: None, is_skipped: false, + provider_item_id: String::new(), + is_captured: false, } } diff --git a/crabidy-server/src/playback.rs b/crabidy-server/src/playback.rs index 91f9d83..290fef0 100644 --- a/crabidy-server/src/playback.rs +++ b/crabidy-server/src/playback.rs @@ -758,6 +758,8 @@ mod tests { duration: None, album: None, is_skipped: false, + provider_item_id: String::new(), + is_captured: false, } } diff --git a/crabidy-server/src/queue_store.rs b/crabidy-server/src/queue_store.rs index 345ab46..6d51b06 100644 --- a/crabidy-server/src/queue_store.rs +++ b/crabidy-server/src/queue_store.rs @@ -308,6 +308,8 @@ mod tests { release_date: None, }), is_skipped: false, + provider_item_id: String::new(), + is_captured: false, } } diff --git a/fsdy/src/lib.rs b/fsdy/src/lib.rs index 3e43daa..bcbcf96 100644 --- a/fsdy/src/lib.rs +++ b/fsdy/src/lib.rs @@ -48,8 +48,10 @@ pub struct Settings { pub enum TrackFileError { #[error("not valid TOML: {0}")] Toml(#[from] toml::de::Error), - #[error("[playable] must set exactly one of `file`, `url`, `link`, `skipped = true`")] + #[error("[playable] must set exactly one of `file`, `url`, `link`, `store`, `skipped = true`")] PlayableCardinality, + #[error("playable store name must be a non-empty bare file name: {0}")] + StoreName(String), /// Carries only the offending *scheme* — a private stream URL may /// embed a token, and this error's message ends up in logs. #[error("playable url is not http(s), scheme: {0}")] @@ -84,7 +86,7 @@ pub struct TrackFile { } /// Optional `[album]` table of a track file. -#[derive(Debug, Deserialize, Serialize)] +#[derive(Clone, Debug, Deserialize, Serialize)] pub struct AlbumMeta { pub title: String, #[serde(skip_serializing_if = "Option::is_none")] @@ -114,6 +116,13 @@ pub struct PlayableSpec { /// counts as unset and fails cardinality validation. #[serde(skip_serializing_if = "Option::is_none")] pub skipped: Option<bool>, + /// Name of an entry in the content-addressed store (a bare file name, + /// no separators), resolved against the instance's store root at play + /// time. Written by download captures instead of `file`, so audio is + /// shared and de-duplicated across saves + /// (architecture/crabidy-store.md D2). + #[serde(skip_serializing_if = "Option::is_none")] + pub store: Option<String>, } /// A validated playable reference. @@ -122,6 +131,9 @@ pub enum Playable { File(PathBuf), Url(String), Link(String), + /// A content-store entry name, resolved against the instance's store + /// root (architecture/crabidy-store.md D2). + Store(String), /// No playable audio; playback skips the track. Skipped, } @@ -150,10 +162,11 @@ impl TrackFile { &self.playable.file, &self.playable.url, &self.playable.link, + &self.playable.store, skipped, ) { - (Some(file), None, None, None) => Ok(Playable::File(file.clone())), - (None, Some(url), None, None) => { + (Some(file), None, None, None, None) => Ok(Playable::File(file.clone())), + (None, Some(url), None, None, None) => { let scheme = url::Url::parse(url) .map(|u| u.scheme().to_string()) .unwrap_or_else(|_| "<not a url>".to_string()); @@ -163,13 +176,21 @@ impl TrackFile { } Ok(Playable::Url(url.clone())) } - (None, None, Some(link), None) => { + (None, None, Some(link), None, None) => { if !link.starts_with('/') { return Err(TrackFileError::LinkNotAbsolute(link.clone())); } Ok(Playable::Link(link.clone())) } - (None, None, None, Some(())) => Ok(Playable::Skipped), + (None, None, None, Some(name), None) => { + // A store entry is a bare file name; separators would let a + // toml escape the store root at resolve time. + if name.is_empty() || name.contains(['/', '\\', '\0']) { + return Err(TrackFileError::StoreName(name.clone())); + } + Ok(Playable::Store(name.clone())) + } + (None, None, None, None, Some(())) => Ok(Playable::Skipped), _ => Err(TrackFileError::PlayableCardinality), } } @@ -197,6 +218,12 @@ impl TrackFile { release_date: a.release_date.clone(), }), is_skipped: matches!(playable, Ok(Playable::Skipped)), + // The provider that owns a link target sets this at its end; a + // local file/store/skipped track has no provider-internal id. + provider_item_id: String::new(), + // A store-backed track is captured by construction; other + // playables are marked (or not) by the server's store index. + is_captured: matches!(playable, Ok(Playable::Store(_))), } } @@ -217,6 +244,7 @@ impl TrackFile { file: None, url: None, link: None, + store: None, skipped: Some(true), } } else { @@ -224,6 +252,7 @@ impl TrackFile { file: None, url: None, link: Some(track.path.clone()), + store: None, skipped: None, } }; @@ -251,6 +280,24 @@ impl TrackFile { file: Some(file.to_path_buf()), url: None, link: None, + store: None, + skipped: None, + }; + this + } + + /// Like [`Self::from_track`], but the playable is a [`Playable::Store`] + /// entry name — used by download captures whose audio lives in the + /// shared content store, so many saves reference one file + /// (architecture/crabidy-store.md D2/D4). `name` must be a bare file + /// name (validated by [`Self::playable`] on read). + pub fn from_track_store(track: &Track, name: &str) -> Self { + let mut this = Self::from_track(track); + this.playable = PlayableSpec { + file: None, + url: None, + link: None, + store: Some(name.to_string()), skipped: None, }; this @@ -266,6 +313,7 @@ impl TrackFile { file: None, url: None, link: None, + store: None, skipped: Some(true), }; this @@ -381,6 +429,11 @@ pub struct Client { /// only — deletes there destroy downloaded data, so clients confirm /// them (architecture/capture-deletion.md). deletable_tree: bool, + /// Root of the content-addressed store this instance resolves + /// [`Playable::Store`] entries against (architecture/crabidy-store.md). + /// Set only on the `/crabidy` instance; `None` elsewhere, where a store + /// playable is a malformed reference. + store_root: Option<PathBuf>, } impl Client { @@ -412,6 +465,7 @@ impl Client { reserved: Vec::new(), downloadable_nodes: false, deletable_tree: false, + store_root: None, }) } @@ -449,6 +503,15 @@ impl Client { self } + /// Sets the content-store root against which [`Playable::Store`] entries + /// resolve (architecture/crabidy-store.md). Used by the `/crabidy` + /// instance; without it, a store playable resolves to + /// [`ProviderError::MalformedPath`]. + pub fn with_store_root(mut self, store_root: PathBuf) -> Self { + self.store_root = Some(store_root); + self + } + /// The provider root this instance owns (e.g. `/fs`), as configured /// through [`Self::new`]. pub fn provider_root(&self) -> &str { @@ -592,6 +655,7 @@ impl Client { is_creatable: false, is_downloadable: self.downloadable_nodes, tracks_deletable: self.deletable_tree, + is_captured: false, }) } @@ -780,6 +844,23 @@ impl ProviderClient for Client { Ok(vec![path]) } Playable::Url(target) => Ok(vec![target]), + Playable::Store(name) => { + // Resolve against the instance's store root; the name was + // validated as a bare file name, so it stays inside. + let Some(store_root) = self.store_root.as_ref() else { + warn!( + path = track_path, + "store playable outside the crabidy provider" + ); + return Err(ProviderError::MalformedPath); + }; + let path = store_root + .join(&name) + .to_str() + .ok_or(ProviderError::InternalError)? + .to_string(); + Ok(vec![path]) + } Playable::Link(target) => { // Link tracks carry the target path from listing time on; // reaching this arm means the caller bypassed that. @@ -819,6 +900,7 @@ impl ProviderClient for Client { is_creatable: false, is_downloadable: self.downloadable_nodes, tracks_deletable: self.deletable_tree, + is_captured: false, } } @@ -1406,6 +1488,8 @@ mod tests { release_date: Some("1977-10-28".to_string()), }), is_skipped: false, + provider_item_id: String::new(), + is_captured: false, }; let toml_text = TrackFile::from_track(&track).to_toml().expect("serialize"); let reparsed = TrackFile::parse(&toml_text).expect("reparse"); @@ -1431,6 +1515,8 @@ mod tests { release_date: None, }), is_skipped: false, + provider_item_id: String::new(), + is_captured: false, }; let file = TrackFile::from_track_with_file(&track, Path::new("0001 One.flac")); let toml_text = file.to_toml().expect("serialize"); @@ -1476,6 +1562,8 @@ mod tests { duration: None, album: None, is_skipped: false, + provider_item_id: String::new(), + is_captured: false, }; let toml_text = TrackFile::from_track(&track).to_toml().expect("serialize"); let reparsed = TrackFile::parse(&toml_text).expect("reparse"); diff --git a/plan/crabidy-store.md b/plan/crabidy-store.md new file mode 100644 index 0000000..6519555 --- /dev/null +++ b/plan/crabidy-store.md @@ -0,0 +1,118 @@ +# Plan: the `crabidy` provider and content store + +Ordered, dependency-first tasks for `architecture/crabidy-store.md`, each +mapped to the test(s) and/or `quality/crabidy-store.md` gate(s) it satisfies. +Green-field: no data migration. + +## Wire + fsdy foundations + +- [x] Proto: `Track.provider_item_id` (7), `Track.is_captured` (8), + `LibraryNode.is_captured` (10), `LibraryNodeChild.is_captured` (8). Regenerate. +- [x] fsdy: `PlayableSpec.store`, `Playable::Store`, 5-way cardinality + + `TrackFileError::StoreName`, `from_track_store`, `Client.store_root` + + `with_store_root`, `get_urls_for_track` store arm, `to_track` sets + `is_captured` for store playables. _(Gate: Playable::Store & fsdy.)_ +- [x] Sweep every `Track`/`LibraryNode`/`LibraryNodeChild` literal for the new + fields; workspace `cargo check --all-targets` green. + +## Content store core (`crabidy_store.rs`) + +- [ ] Add `blake3` dependency (Cargo.toml). _(Gate: store layout.)_ +- [ ] `hash_file` streams blake3, returns `"blake3:<hex>"`. Unit test on a temp + file with a known hash. +- [ ] `StoreSidecar`/`ProviderEntry` (de)serialize to the documented TOML shape; + round-trip test. _(Gate: store layout.)_ +- [ ] `StoreIndex::scan` reads all `*.cbd-store.toml`, skips a malformed one with + a warning; `insert` adds hash + every `(provider,id)`. Test: a store dir with + two sidecars + one broken file indexes the two, skips the broken. + _(Gates: de-dup; errors — bad sidecar never poisons the index.)_ +- [ ] `unique_store_name`: returns the natural name, else `stem (N)ext` on a + taken name. Test collisions. _(Gate: new track / numeral suffix.)_ +- [ ] `CrabidyStore::open(tree_root, store_root)`: create both dirs, scan index, + build downloader; `default_tree_root`/`default_store_root` use state/data + dirs. _(Gate: store layout — not config_dir.)_ + +## Per-track de-dup capture (D4) + +- [ ] Capture primitive `capture_track(client, track, save_dir, index, budget, + progress) -> Outcome`, implementing the D4 flow: + 1. already store-backed / fs-under-store → reuse, no fetch; + 2. `index.by_provider_id` hit → reuse, record alias if title differs; + 3. miss → fetch bytes (download via `Downloader`, or read the local fs file), + `hash_file`; + 4. `index.by_hash` hit → add `[[provider]]`, discard bytes, reuse name; + 5. miss → `unique_store_name`, move/copy into store, write sidecar, index it. + Then write the save's track toml with `from_track_store` (skipped source → + skipped toml). _(Tests: dedup-by-id, dedup-by-hash, alias, fs-copy-dedup, + fs-already-in-store no-op. Gates: de-dup section.)_ +- [ ] Budget counts only bytes fetched this run; hits cost zero. _(Test: two + captures of the same track fetch bytes once.)_ + +## Save + current (D5, D6) + +- [ ] `validate`: `validate_folder_name(name, &["current"])`; refuse if + `/crabidy/<name>` exists; `Capture` requires source `is_downloadable`. + _(Test: existing-name refused; Gate: save/conflict.)_ +- [ ] `save`: enumerate source via `client` (reuse `capture::enumerate` / + iterative walk), build into `.tmp-<name>`, per track link (`from_track`) or + `capture_track`, then atomic remove-absent + rename swap. Failure removes the + temp. _(Tests: link save writes links; capture save writes store tomls; + failed save leaves no folder but store keeps audio. Gates: save section.)_ +- [ ] `persist_current`: write `/crabidy/current` flat from a queue snapshot + (links), atomic replace. _(Test: snapshot → flat current with links.)_ +- [ ] `annotate_captured(node)`: set `Track.is_captured` per index lookup or + store playable; set node/child `is_captured` when fully captured. + _(Test: browsing a node with a captured track marks only that track.)_ + +## Provider identity (D3) + +- [ ] tidal: set `Track.provider_item_id` to the track id everywhere tidal + builds a `Track`. _(Gate: provider identity.)_ +- [ ] youtube: set it to the video id. +- [ ] fs: set it to the canonical absolute source path (for `file` playables). + _(Test: two fs tomls pointing at one file share an id.)_ + +## Orchestrator + RPC + playback rewire + +- [ ] `ProviderOrchestrator`: drop `queues`/`bookmarks`/`captures` fields; add + one `crabidy` field. Construct `CrabidyStore` + an `fsdy::Client` on its + `tree_dir()` `with_editable_top_level(&["current"]).with_downloadable_nodes() + .with_deletable_tree().with_store_root(store_dir())`. Route `crabidy_owns`; + add the single `crabidy` root child; call `annotate_captured` where nodes are + returned. Remove the old `*_owns` for the three. _(Gate: UI root; captured + marking.)_ +- [ ] `CaptureLibraryNode` handler → `crabidy_store.validate` then spawn + `crabidy_store.save`. Remove the `SaveQueue` RPC + its command, client method, + and playback handler; `save_queue` path folds into a capture on + `/crabidy/current`. _(Gate: save section.)_ +- [ ] Playback loop persists `current` through `CrabidyStore::persist_current` + (was `QueueStore`); saved-queue snapshot path removed. _(Test: existing queue + persistence tests adapted.)_ +- [ ] Remove `bookmark_store.rs`, `capture_store.rs`, `queue_store.rs`; move any + still-needed helpers into `crabidy_store`/`capture`; update `lib.rs` mods. + Keep `capture.rs` (`Downloader`, windowed download, `Progress`, `enumerate`). + +## TUI (D8) + +- [ ] Path constants → `/crabidy`, `CURRENT_QUEUE_PATH = "/crabidy/current"`. +- [ ] `library.rs::render`: prefix `|` when `is_captured`; populate `UiItem` + `is_captured` from `Track.is_captured` / child `is_captured`. _(TUI test: + a captured row renders a leading `|`.)_ +- [ ] Queue `w` → `CaptureNode{ path:"/crabidy/current", name, download:false }`; + queue `W` → same with `download:true`. Drop `QueueSaveAs`/`SaveQueue` UI path. +- [ ] Remove `delete_needs_confirmation`/`ConfirmDelete` and the `y/N` overlay; + `d` on `/crabidy` deletes directly. _(Gate: UI deletion.)_ +- [ ] Help table (`bindings.rs`): update `w`/`W`/`d` descriptions for the merged + provider. _(Existing binding tests updated.)_ + +## Docs + +- [ ] README: config/dirs table (state vs data), single `crabidy` provider, + `|` captured marker, removed queues/bookmarks/captures + capture-deletion. +- [ ] `architecture/overview.md`: provider tree updated. +- [ ] `plan/summary.md`: what was built + deviations from this plan/architecture. + +## Verification + +- [ ] `cargo test --workspace` green; `cargo clippy --all-targets -- -D warnings` + and `cargo fmt --check` clean; markdownlint clean. Run via `devenv shell`. diff --git a/quality/crabidy-store.md b/quality/crabidy-store.md new file mode 100644 index 0000000..f37157c --- /dev/null +++ b/quality/crabidy-store.md @@ -0,0 +1,94 @@ +# Quality gates: the `crabidy` provider and content store + +Criteria the implementation must satisfy beyond the automatic tests +(`fsdy/src/lib.rs`, `crabidy-server/src/crabidy_store.rs`, and the TUI tests). +Each gate is pass/fail by reading the code. See `architecture/crabidy-store.md`. +Boxes are unchecked until the implement stage verifies them. + +## Store layout and de-duplication (D2, D4) + +- [ ] The content store is a flat directory under `dirs::data_dir()/crabidy`; + the toml tree is under `dirs::state_dir()/crabidy`. Neither uses + `dirs::config_dir()`. +- [ ] Every stored audio file has a paired `<name>.cbd-store.toml` sidecar with + a `hash` and at least one `[[provider]]` entry; the sidecars are the only + persisted index (no separate index file). +- [ ] `StoreIndex` is built by scanning the sidecars at open and updated on + every write; lookups go through it (no per-capture directory rescans, no + shelling out to grep/rg). +- [ ] Capturing a track already present by `(provider, provider_item_id)` does + **no** download and reuses the existing store entry; a differing title is + appended to that entry's `aliases` (not duplicated). +- [ ] Capturing content already present by hash (provider id missed) discards + the freshly fetched bytes, adds a new `[[provider]]` entry to the existing + sidecar, and points the toml at the existing store name — no duplicate audio. +- [ ] A genuinely new track creates one audio file + one sidecar; on a + natural-name collision with *different* content the name gets a numeral + suffix (never overwrites unrelated audio). +- [ ] The download byte budget counts only bytes fetched this run; provider-id + and hash hits cost zero budget. +- [ ] An `/fs` track whose file is already under the store root captures as a + no-op reuse; one pointing at a normal file is **copied** into the store and + the original file is left in place. + +## `Playable::Store` and fsdy (D2, D7) + +- [ ] `[playable]` validation is exactly-one across `file`/`url`/`link`/`store`/ + `skipped=true`; a `store` value with a path separator or empty is + `StoreName`, not accepted. +- [ ] A `store` playable resolves only against the instance's `store_root`; an + instance without a store root (any non-`/crabidy` mount) treats it as a + malformed reference, never a path escape. +- [ ] `from_track_store` round-trips: the written toml re-reads as + `Playable::Store(name)` and `to_track` sets `is_captured = true` for it. +- [ ] Deleting a `/crabidy` track removes only its toml; deleting a folder + removes only the toml folder. Store audio (outside the toml root) is never + removed — verified via the existing "audio must be under the instance root" + guard. + +## Provider identity (D3) + +- [ ] `Track.provider_item_id` is set by tidal (track id), youtube (video id), + and fs (canonical source path); it is stable across the paths an item is + reached by (search vs playlist vs album). +- [ ] The store keys on `(provider, id)` where provider is the source track's + path root — two different provider ids never collide across providers. +- [ ] `provider_item_id` is never logged as a secret and carries no token; it is + an opaque provider id only. + +## Save, conflict, current (D5, D6) + +- [ ] `w` writes link tomls (no store, no audio); `W` writes store-backed tomls + and runs the de-dup capture. Both work on a library node and on the queue. +- [ ] A save to an existing `/crabidy/<name>` is refused with a clear warning + and changes nothing on disk; the user must delete and re-save. +- [ ] Saves are atomic: built in a hidden temp sibling and swapped into place on + success; a failed/crashed save leaves no partial top-level folder, and any + audio already committed to the store persists (making retry cheap). +- [ ] `current` is reserved: the playback loop overwrites it on every queue + change; a user save named `current` is rejected. Saved queues are flat. +- [ ] `SaveQueue` is gone; queue `w`/`W` go through `CaptureLibraryNode` on + `/crabidy/current`. Validation errors return synchronously; progress streams + via `CaptureProgress` (unchanged shape). + +## UI (D8) + +- [ ] The root library shows a single `crabidy` child; `queues`/`bookmarks`/ + `captures` no longer appear. +- [ ] Captured rows are prefixed with `|` as the first character of the row + (before selection padding), driven by `is_captured`; captured tracks are + marked even while browsing another provider (tidal/youtube). +- [ ] Deletion on `/crabidy` has no confirmation dialog and no disk-reclamation + path (the `capture-deletion.md` confirm flow is removed). + +## Errors and safety (always-on rules) + +- [ ] No panics on malformed sidecars, missing store files, partial downloads, + or unreadable sources — every defect is a typed error and a bad sidecar is + skipped with a warning, never poisoning the index. +- [ ] External calls (downloads) keep their timeouts; the capture runs on a + spawned task so the orchestrator keeps serving commands. +- [ ] Store mutation is serialized (the index mutex) so concurrent captures + cannot corrupt a sidecar or race the numeral-suffix naming. +- [ ] Stream URLs stay redacted (scheme/host only) in any new log lines; store + names and titles are fine to log, tokens are not. diff --git a/tidaldy/src/lib.rs b/tidaldy/src/lib.rs index b296ed9..5660beb 100644 --- a/tidaldy/src/lib.rs +++ b/tidaldy/src/lib.rs @@ -130,6 +130,7 @@ impl crabidy_core::ProviderClient for Client { is_creatable: false, is_downloadable: false, tracks_deletable: false, + is_captured: false, } } @@ -159,6 +160,7 @@ impl crabidy_core::ProviderClient for Client { is_creatable: false, is_downloadable: false, tracks_deletable: false, + is_captured: false, }; let user_id = user_id.ok_or(crabidy_core::ProviderError::UnknownUser)?; for playlist in self @@ -192,6 +194,7 @@ impl crabidy_core::ProviderClient for Client { is_creatable: false, is_downloadable: false, tracks_deletable: false, + is_captured: false, } } TidalPath::Artists => { @@ -205,6 +208,7 @@ impl crabidy_core::ProviderClient for Client { is_creatable: false, is_downloadable: false, tracks_deletable: false, + is_captured: false, }; let user_id = user_id.ok_or(crabidy_core::ProviderError::UnknownUser)?; for artist in self.get_users_artists(&user_id).await? { @@ -241,6 +245,7 @@ impl crabidy_core::ProviderClient for Client { is_creatable: false, is_downloadable: false, tracks_deletable: false, + is_captured: false, } } TidalPath::Album { album, .. } => { @@ -261,6 +266,7 @@ impl crabidy_core::ProviderClient for Client { is_creatable: false, is_downloadable: false, tracks_deletable: false, + is_captured: false, } } TidalPath::Search => crabidy_core::proto::crabidy::LibraryNode { @@ -289,6 +295,7 @@ impl crabidy_core::ProviderClient for Client { is_creatable: true, is_downloadable: false, tracks_deletable: false, + is_captured: false, }, TidalPath::SearchTerm(encoded) => { let term = crabidy_core::decode_segment(encoded); @@ -923,6 +930,7 @@ impl Client { is_creatable: false, is_downloadable: false, tracks_deletable: false, + is_captured: false, }) } diff --git a/tidaldy/src/models.rs b/tidaldy/src/models.rs index e95fba3..7518196 100644 --- a/tidaldy/src/models.rs +++ b/tidaldy/src/models.rs @@ -212,6 +212,8 @@ impl Track { album: self.album.clone().map(|a| a.into()), duration: self.duration.map(|d| d as u32 * 1000), is_skipped: false, + provider_item_id: String::new(), + is_captured: false, } } } diff --git a/ytdy/src/lib.rs b/ytdy/src/lib.rs index 530428d..604c55c 100644 --- a/ytdy/src/lib.rs +++ b/ytdy/src/lib.rs @@ -130,6 +130,8 @@ fn entry_to_track(entry: &VideoEntry, node_path: &str) -> Track { duration: entry.duration, album: None, is_skipped: false, + provider_item_id: String::new(), + is_captured: false, } } @@ -192,6 +194,7 @@ impl Client { is_creatable: false, is_downloadable: false, tracks_deletable: false, + is_captured: false, }) } @@ -226,6 +229,7 @@ impl Client { is_creatable: false, is_downloadable: false, tracks_deletable: false, + is_captured: false, }) } @@ -255,6 +259,7 @@ impl Client { is_creatable: false, is_downloadable: false, tracks_deletable: false, + is_captured: false, }) } @@ -432,6 +437,7 @@ impl ProviderClient for Client { is_creatable: false, is_downloadable: false, tracks_deletable: false, + is_captured: false, } } @@ -467,6 +473,7 @@ impl ProviderClient for Client { is_creatable: true, is_downloadable: false, tracks_deletable: false, + is_captured: false, }, YtPath::SearchTerm(encoded) => { let term = crabidy_core::decode_segment(encoded);