From 7f0e900c56e8c761933b86cf40bdb7f7945e4afd Mon Sep 17 00:00:00 2001 From: Test User Date: Tue, 21 Jul 2026 13:45:04 +0200 Subject: [PATCH] Capture library subtrees with downloaded audio MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit W on a downloadable library node mirrors the subtree into /captures (a fourth fsdy instance) like a bookmark, but downloads every track's audio next to its toml; the toml points at the sibling by relative name, so captures play with no provider round trip. Nodes opt in via the new is_downloadable flags — Tidal blesses queueable and track-listing nodes. The bookmark walk is now the shared capture walk parameterized by a per-track sink. Co-Authored-By: Claude Fable 5 --- Cargo.lock | 1 + architecture/captures.md | 213 +++++++++++++ cbd-tui/src/app/bindings.rs | 28 ++ cbd-tui/src/app/library.rs | 11 + cbd-tui/src/app/mod.rs | 147 ++++++++- cbd-tui/src/app/queue.rs | 1 + cbd-tui/src/main.rs | 13 +- cbd-tui/src/rpc.rs | 7 +- crabidy-core/crabidy/v1/crabidy.proto | 11 +- crabidy-core/src/lib.rs | 8 +- crabidy-server/Cargo.toml | 1 + crabidy-server/src/bookmark_store.rs | 121 +------ crabidy-server/src/capture.rs | 414 ++++++++++++++++++++++++ crabidy-server/src/capture_store.rs | 442 ++++++++++++++++++++++++++ crabidy-server/src/lib.rs | 2 + crabidy-server/src/main.rs | 10 +- crabidy-server/src/provider.rs | 121 ++++++- crabidy-server/src/rpc.rs | 27 +- fsdy/src/lib.rs | 63 ++++ plan/captures.md | 54 ++++ plan/summary.md | 47 +++ quality/captures.md | 84 +++++ tidaldy/src/lib.rs | 18 ++ 23 files changed, 1702 insertions(+), 142 deletions(-) create mode 100644 architecture/captures.md create mode 100644 crabidy-server/src/capture.rs create mode 100644 crabidy-server/src/capture_store.rs create mode 100644 plan/captures.md create mode 100644 quality/captures.md diff --git a/Cargo.lock b/Cargo.lock index b6485c9..cbb1afe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -748,6 +748,7 @@ dependencies = [ "fsdy", "futures", "rand 0.10.2", + "reqwest", "serde", "tempfile", "thiserror 2.0.19", diff --git a/architecture/captures.md b/architecture/captures.md new file mode 100644 index 0000000..0145233 --- /dev/null +++ b/architecture/captures.md @@ -0,0 +1,213 @@ +# Captures (downloaded subtrees) + +## Context and problem statement + +Bookmarks (`w`) mirror a library subtree as **link** files — replaying a +bookmark still needs the original provider. The user wants `W` on a library +node to do the same capture into a separate local provider called +**captures**, except each track's audio is **downloaded** next to its +`.cbd-track.toml`, and the toml points at that file. Playback of a capture +then needs no provider round trip at all — it is a fully local copy. + +A library node decides whether it allows `W`; Tidal implements it. + +## Assumptions (confirmed against the code) + +- `PlayableSpec.file` already supports **relative** paths, resolved against + the track file's directory at `get_urls_for_track` time (fs-provider D3). + A toml next to its audio file can say `file = "0001 Song.flac"` and the + whole capture folder stays relocatable (tmp-and-swap, rename, backup). +- `fsdy::list_dir` only surfaces directories and `*.cbd-track.toml` files; + downloaded audio siblings are invisible to the library listing. +- The bookmark walk (`bookmark_store::write_capture`) is an iterative + pre-order worklist whose only per-track action is "serialize and write + one file" — exactly the seam where a download variant plugs in. +- `reqwest` (rustls, `stream`) is already a workspace dependency; tidal + stream URLs come from `get_urls_for_track` on the orchestrator, so the + download needs no new provider methods. +- Uppercase bindings (`K`, `J`) already exist in the TUI bindings table; + `W` in `Scope::Library` is free. +- `LibraryNode`/`LibraryNodeChild` already model per-node capabilities + (`is_queable`, `is_creatable`, …) — the "does this node allow `W`" + decision extends that pattern. + +## Decisions + +### D1 — Fourth `fsdy` instance at `/captures` + +`/crabidy/captures/` is mounted read-only as `/captures` with an +editable top level (no reserved names), exactly like `/bookmarks`. Init is +non-fatal: an unopenable store disables `W` and the mount, never the +server. Loading a capture is browsing `/captures` and queueing a folder — +zero new replay mechanisms. + +### D2 — One shared walk, two track sinks + +Options considered: + +- *(a)* Copy `bookmark_store.rs` and swap the per-track write. +- *(b)* Extract the walk into a shared `capture` module parameterized by a + **track sink**; bookmarks and captures become thin stores over it. + +**Decision: (b)** — the walk (worklist, caps, temp-and-swap, all-or-nothing +cleanup, `BadSource` mapping) is behavior we already tested once and must +not fork. `crabidy-server/src/capture.rs` owns `CaptureError`, the caps, +and `write_tree(client, source, tmp, caps, sink)`; the sink is an enum +(no async-trait indirection): + +- `Sink::Link` — today's bookmark behavior, byte-identical + (`TrackFile::from_track`, link playable). +- `Sink::Download(Downloader)` — captures (D3). + +`BookmarkStore` keeps its API; `CaptureStore` (in `capture_store.rs`) is +its sibling over the captures directory. + +### D3 — Download sink: audio next to the toml, toml points at it + +Per track, in listing order: + +1. `get_urls_for_track` through the orchestrator (any provider that yields + URLs works; Tidal is the target). First URL wins. +2. HTTP GET via one shared `reqwest` client — connect timeout, one total + per-track deadline covering the whole body, **no retries** (a capture + is re-runnable and overwrite = refresh; a retry policy can come later). + The body is streamed to `NNNN .<ext>` (shared `ordered_name` + sanitizer, same 4-digit prefix as the toml so the pair sorts together). +3. The extension comes from the response `Content-Type` + (`audio/flac` → `flac`, `audio/mp4`/`audio/m4a` → `m4a`, + `audio/mpeg` → `mp3`, `audio/ogg` → `ogg`, `audio/wav` → `wav`), + falling back to the URL path's extension, then `bin` (the player probes + by content; the extension is a hint). +4. The toml is written **after** the download succeeds, with + `playable.file = "<audio file name>"` (relative, new + `TrackFile::from_track_with_file`), keeping metadata identical to a + bookmark entry. + +Caps: `MAX_CAPTURE_DIRS` stays 1 000; downloads get their own +`MAX_DOWNLOAD_TRACKS = 500` and a total byte budget +`MAX_DOWNLOAD_BYTES = 4 GiB` counted while streaming — a runaway artist +capture must not fill the disk. Downloads run sequentially (gentle on the +provider, trivially bounded memory); the whole capture already runs on a +spawned task, so the orchestrator keeps serving. + +All-or-nothing is kept: any failed download aborts the capture and removes +the temp folder. A capture that lists 30 tracks *has* 30 playable files. + +### D4 — Nodes opt in via `is_downloadable` + +New proto fields `LibraryNode.is_downloadable = 8` and +`LibraryNodeChild.is_downloadable = 7` (additive). Tidal sets the flag +centrally at the end of `get_lib_node`: a node is downloadable when it is +**queueable or lists tracks** (the "or lists tracks" covers search-term +result nodes, which are not queueable as a whole but whose track results +are downloadable); children mirror `is_queable`. Everything else (fs, +queues, bookmarks, captures, orchestrator roots) leaves the default +`false`; capturing a capture is pointless and links must not masquerade +as downloads. + +Tracks carry no flag: a listed track inherits its containing node's +`is_downloadable` (TUI) — a Tidal album's tracks are downloadable because +the album is. The server enforces at the capture **root**: a directory +source must report `is_downloadable`, a track source's parent node must +(`CaptureError::Unsupported` otherwise). Nested nodes inside the walk are +not re-checked — the pressed node's decision governs its subtree. + +### D5 — Wire: the existing rpc gains a `download` flag + +`CaptureLibraryNodeRequest` gets `bool download = 3` (additive; old +clients keep bookmarking). `ProviderCommand::CaptureLibraryNode` carries +it and the handler picks the store. Error mapping extends the bookmark +contract: `InvalidName`/`BadSource` → `invalid_argument`, +`TooLarge`/`Disabled`/`Unsupported` → `failed_precondition`, download and +disk failures → `internal`. + +### D6 — TUI: `W` on the library pane + +`Action::LibraryDownloadNode` bound to `W` in `Scope::Library` ("Download +selection as capture"). Gate: the bare selection must be queueable **and** +downloadable (`selected_downloadable()`; child flag for nodes, the current +node's flag for tracks; marks ignored like `w`). The existing input +overlay opens with `InputPurpose::Capture { path, download: true }` +(label `capture`), prefilled with the selection title. Submit sends +`MessageFromUi::CaptureNode { path, name, download }` → the rpc. Failures +are logged, never fatal to the poll loop. + +### D7 — Out of scope (explicitly) + +- Retry/resume of failed or partial downloads (re-run the capture). +- Quality/codec selection, transcoding, tagging the audio files. +- Progress display in the TUI while a capture downloads. +- Deduplicating audio across captures, or refreshing links in existing + bookmarks into downloads. +- DRM circumvention: the download uses exactly the stream URLs the + provider already serves for playback. + +## Structure + +```d2 +direction: right + +server: crabidy-server { + orch: ProviderOrchestrator + cap: "capture.rs\nshared walk + caps + swap" { + link: "Sink::Link" + dl: "Sink::Download\n(reqwest, timeouts, byte budget)" + } + bs: BookmarkStore + cs: CaptureStore +} + +tidal: "tidaldy\n(is_downloadable = is_queable)" + +disk: "config/crabidy" { + shape: cylinder + b: "bookmarks/<name>/ (link tomls)" + c: "captures/<name>/ (audio + file tomls)" +} + +cfs: "fsdy /captures\n(editable top level)" + +server.orch -> server.bs: "CaptureLibraryNode\ndownload=false" +server.orch -> server.cs: "CaptureLibraryNode\ndownload=true" +server.bs -> server.cap.link +server.cs -> server.cap.dl +server.cap.dl -> tidal: "get_urls_for_track\n+ HTTP GET stream" +server.bs -> disk.b +server.cs -> disk.c +cfs -> disk.c: "list + parse (read only)" +server.orch -> cfs: "/captures/..." +``` + +## Key flow: W on a Tidal album + +```d2 +shape: sequence_diagram +tui: TUI +rpc: gRPC +orch: Orchestrator +cs: CaptureStore +tidal: Tidal + +tui -> rpc: "CaptureLibraryNode(path, name, download=true)" +rpc -> orch: "ProviderCommand (spawned)" +orch -> cs: "capture(name)" +cs -> orch: "get_lib_node: root allows download?" +cs -> tidal: "per track: get_urls_for_track" +cs -> tidal: "HTTP GET (deadline, byte budget)" +cs -> cs: "audio + toml pair\n(toml after audio, file = relative)" +cs -> rpc: "tmp-and-swap captures/<name>/" +rpc -> tui: OK +``` + +## Risks and open questions + +- **Disk usage**: 500 tracks of FLAC can be tens of GiB; the byte budget + caps one capture, not the folder's total. Accepted — the user manages + `captures/` like any local music folder (and can delete via `d`). +- **Stream URL churn**: Tidal URLs are short-lived; the download happens + immediately after fetching each URL, so expiry only matters for very + slow transfers, which the per-track deadline already bounds. +- **Licensing**: captures are personal-use copies of streams the account + can already play; nothing here bypasses provider protection. +- Open (future): a progress event stream for long captures; retry with + classification + jitter; per-provider download quality knobs. diff --git a/cbd-tui/src/app/bindings.rs b/cbd-tui/src/app/bindings.rs index 2970aba..321a828 100644 --- a/cbd-tui/src/app/bindings.rs +++ b/cbd-tui/src/app/bindings.rs @@ -72,6 +72,10 @@ pub enum Action { /// capture the selected queueable subtree as a bookmark under /// `/bookmarks`. No-op unless the bare selection `is_queable`. LibraryCaptureNode, + /// Like [`Self::LibraryCaptureNode`], but the capture **downloads** + /// every track's audio into `/captures`. No-op unless the bare + /// selection is queueable *and* downloadable (e.g. Tidal subtrees). + LibraryDownloadNode, // Queue pane QueueInsertHere, QueueFirst, @@ -266,6 +270,13 @@ pub const BINDINGS: &[Binding] = &[ action: Action::LibraryCaptureNode, description: "Save selection as bookmark", }, + Binding { + scope: Scope::Library, + mods: KeyModifiers::SHIFT, + code: KeyCode::Char('W'), + action: Action::LibraryDownloadNode, + description: "Download selection as capture", + }, Binding { scope: Scope::Library, mods: KeyModifiers::NONE, @@ -563,6 +574,23 @@ mod tests { ), Some(Action::LibraryCaptureNode) ); + // Shift-w is the download capture, library pane only. + assert_eq!( + lookup( + UiFocus::Library, + false, + key(KeyCode::Char('W'), KeyModifiers::SHIFT) + ), + Some(Action::LibraryDownloadNode) + ); + assert_eq!( + lookup( + UiFocus::Queue, + false, + key(KeyCode::Char('W'), KeyModifiers::SHIFT) + ), + None + ); } #[test] diff --git a/cbd-tui/src/app/library.rs b/cbd-tui/src/app/library.rs index 4303352..6b1eaee 100644 --- a/cbd-tui/src/app/library.rs +++ b/cbd-tui/src/app/library.rs @@ -74,6 +74,13 @@ impl Library { item.is_queable .then(|| (item.path.clone(), item.title.clone())) } + /// The bare selection's path and title when it can be captured with a + /// download (`W`): queueable *and* downloadable. Marks are ignored, + /// like [`Self::selected_queueable`]. + pub fn selected_downloadable(&self) -> Option<(String, String)> { + let item = self.list.get(self.list_state.selected()?)?; + (item.is_queable && item.is_downloadable).then(|| (item.path.clone(), item.title.clone())) + } pub fn get_selected(&self) -> Option<Vec<String>> { if self.list.iter().any(|i| i.marked) { return Some( @@ -188,6 +195,9 @@ impl Library { is_creatable: false, is_editable: false, is_deletable: false, + // Tracks carry no wire flag: they inherit their node's + // blessing (architecture/captures.md D4). + is_downloadable: node.is_downloadable, }) .chain(node.children.iter().map(|c| UiItem { path: c.path.clone(), @@ -198,6 +208,7 @@ impl Library { is_creatable: c.is_creatable, is_editable: c.is_editable, is_deletable: c.is_deletable, + is_downloadable: c.is_downloadable, })) .collect(); diff --git a/cbd-tui/src/app/mod.rs b/cbd-tui/src/app/mod.rs index b47d24e..8e48bf5 100644 --- a/cbd-tui/src/app/mod.rs +++ b/cbd-tui/src/app/mod.rs @@ -48,6 +48,9 @@ struct UiItem { is_editable: bool, /// This item may be deleted (`d`) — part of the `[ed]` marker. is_deletable: bool, + /// This item allows download captures (`W`). Tracks inherit their + /// containing node's flag; child nodes carry their own. + is_downloadable: bool, } pub const COLOR_PRIMARY: Color = Color::Rgb(129, 161, 193); @@ -90,10 +93,13 @@ pub enum MessageFromUi { /// as `/queues/<name>` in the library on the next visit). SaveQueue(String), /// Capture the queueable subtree at `path` as the bookmark `name` - /// (structure-preserving snapshot under `/bookmarks/<name>`). + /// (structure-preserving snapshot under `/bookmarks/<name>`), or — + /// with `download` — as the capture `name` under `/captures/<name>` + /// with every track's audio downloaded next to its toml. CaptureNode { path: String, name: String, + download: bool, }, AppendTracks(Vec<String>), QueueTracks(Vec<String>), @@ -135,8 +141,10 @@ pub enum InputPurpose { /// `w`: save the current queue under the entered name. SaveQueue, /// `w` in the library: capture the queueable subtree at `path` as a - /// bookmark. The buffer starts prefilled with the selection's title. - Capture { path: String }, + /// bookmark — or, for `W` (`download`), as a download capture (the + /// selection must also be downloadable). The buffer starts prefilled + /// with the selection's title. + Capture { path: String, download: bool }, } /// State of the one-line text input overlay (node creation and rename). @@ -213,10 +221,11 @@ impl App { InputPurpose::SaveQueue => { let _ = self.tx.send(MessageFromUi::SaveQueue(title)); } - InputPurpose::Capture { path } => { + InputPurpose::Capture { path, download } => { let _ = self.tx.send(MessageFromUi::CaptureNode { path: path.clone(), name: title, + download: *download, }); } } @@ -312,7 +321,23 @@ impl App { // non-queueable selections like the other gated openers. if let Some((path, title)) = self.library.selected_queueable() { self.input = Some(InputState { - purpose: InputPurpose::Capture { path }, + purpose: InputPurpose::Capture { + path, + download: false, + }, + buffer: title, + }); + } + } + Action::LibraryDownloadNode => { + // `W`: like `w`, but the selection must also allow download + // captures (architecture/captures.md D6). + if let Some((path, title)) = self.library.selected_downloadable() { + self.input = Some(InputState { + purpose: InputPurpose::Capture { + path, + download: true, + }, buffer: title, }); } @@ -389,7 +414,10 @@ impl App { InputPurpose::Create { .. } => "new node", InputPurpose::Rename { .. } => "rename", InputPurpose::SaveQueue => "save queue", - InputPurpose::Capture { .. } => "bookmark", + InputPurpose::Capture { + download: false, .. + } => "bookmark", + InputPurpose::Capture { download: true, .. } => "capture", }; let line = Rect::new(area.x + 1, area.y + area.height - 2, area.width - 2, 1); f.render_widget(Clear, line); @@ -482,6 +510,7 @@ mod tests { tracks: Vec::new(), is_queable: false, is_creatable: true, + is_downloadable: false, } } @@ -793,9 +822,13 @@ mod tests { app.library.update(queueable_listing()); let _ = app.dispatch(Action::LibraryCaptureNode); let input = app.input.as_ref().expect("capture overlay open"); - assert!( - matches!(&input.purpose, InputPurpose::Capture { path } if path == "/tidal/artists/1") - ); + assert!(matches!( + &input.purpose, + InputPurpose::Capture { + path, + download: false + } if path == "/tidal/artists/1" + )); assert_eq!(input.buffer, "artist", "prefilled with the title"); } @@ -809,9 +842,14 @@ mod tests { app.handle_input_key(key(crossterm::event::KeyCode::Enter)); assert!(app.input.is_none()); match rx.try_recv() { - Ok(MessageFromUi::CaptureNode { path, name }) => { + Ok(MessageFromUi::CaptureNode { + path, + name, + download, + }) => { assert_eq!(path, "/tidal/artists/1"); assert_eq!(name, "artist favs"); + assert!(!download, "w captures a bookmark, not a download"); } other => panic!("expected CaptureNode, got {:?}", other.is_ok()), } @@ -822,6 +860,95 @@ mod tests { assert!(rx.try_recv().is_err(), "Esc must not capture"); } + /// Like [`queueable_listing`], but the child also allows download + /// captures (a Tidal subtree). + fn downloadable_listing() -> LibraryNode { + use crabidy_core::proto::crabidy::LibraryNodeChild; + LibraryNode { + children: vec![LibraryNodeChild { + is_downloadable: true, + ..LibraryNodeChild::new("/tidal/artists/1".to_string(), "artist".to_string(), true) + }], + ..creatable_node("/tidal/artists") + } + } + + #[test] + fn download_capture_requires_the_downloadable_flag() { + let (mut app, _rx) = app(); + // Queueable but not downloadable (e.g. an /fs folder): `W` must + // stay closed even though `w` would open. + app.library.update(queueable_listing()); + let _ = app.dispatch(Action::LibraryDownloadNode); + assert!(app.input.is_none()); + + app.library.update(downloadable_listing()); + let _ = app.dispatch(Action::LibraryDownloadNode); + let input = app.input.as_ref().expect("download overlay open"); + assert!(matches!( + &input.purpose, + InputPurpose::Capture { + path, + download: true + } if path == "/tidal/artists/1" + )); + assert_eq!(input.buffer, "artist", "prefilled with the title"); + } + + #[test] + fn tracks_inherit_their_nodes_download_blessing() { + use crabidy_core::proto::crabidy::Track; + let track = Track { + path: "/tidal/artists/1/2/3".to_string(), + artist: "a".to_string(), + title: "one".to_string(), + duration: None, + album: None, + }; + let listing = |downloadable| LibraryNode { + tracks: vec![track.clone()], + is_downloadable: downloadable, + ..creatable_node("/tidal/artists/1/2") + }; + let (mut app, _rx) = app(); + app.library.update(listing(false)); + let _ = app.dispatch(Action::LibraryDownloadNode); + assert!(app.input.is_none(), "unblessed node, unblessed tracks"); + + app.library.update(listing(true)); + let _ = app.dispatch(Action::LibraryDownloadNode); + let input = app.input.as_ref().expect("download overlay open"); + assert!(matches!( + &input.purpose, + InputPurpose::Capture { + path, + download: true + } if path == "/tidal/artists/1/2/3" + )); + } + + #[test] + fn download_capture_submits_with_the_download_flag() { + let (mut app, rx) = app(); + app.library.update(downloadable_listing()); + let _ = app.dispatch(Action::LibraryDownloadNode); + type_str(&mut app, " local "); + app.handle_input_key(key(crossterm::event::KeyCode::Enter)); + assert!(app.input.is_none()); + match rx.try_recv() { + Ok(MessageFromUi::CaptureNode { + path, + name, + download, + }) => { + assert_eq!(path, "/tidal/artists/1"); + assert_eq!(name, "artist local"); + assert!(download, "W captures a download"); + } + other => panic!("expected CaptureNode, got {:?}", other.is_ok()), + } + } + /// A one-track queue update, as the server would broadcast it. fn one_track_queue() -> crabidy_core::proto::crabidy::Queue { use crabidy_core::proto::crabidy::{Queue as ProtoQueue, Track}; diff --git a/cbd-tui/src/app/queue.rs b/cbd-tui/src/app/queue.rs index 3eb1b22..ec1377d 100644 --- a/cbd-tui/src/app/queue.rs +++ b/cbd-tui/src/app/queue.rs @@ -81,6 +81,7 @@ impl Queue { is_creatable: false, is_editable: false, is_deletable: false, + is_downloadable: false, }) .collect(); diff --git a/cbd-tui/src/main.rs b/cbd-tui/src/main.rs index 4b0614f..066c11a 100644 --- a/cbd-tui/src/main.rs +++ b/cbd-tui/src/main.rs @@ -203,11 +203,14 @@ async fn poll( error!(name, "failed to save queue: {err}"); } } - MessageFromUi::CaptureNode { path, name } => { - // A rejected capture (bad name, over-cap subtree) must - // not tear down the poll loop either. - if let Err(err) = rpc_client.capture_library_node(path.clone(), name.clone()).await { - error!(path, name, "failed to capture subtree: {err}"); + MessageFromUi::CaptureNode { path, name, download } => { + // A rejected capture (bad name, over-cap subtree, failed + // download) must not tear down the poll loop either. + if let Err(err) = rpc_client + .capture_library_node(path.clone(), name.clone(), download) + .await + { + error!(path, name, download, "failed to capture subtree: {err}"); } } } diff --git a/cbd-tui/src/rpc.rs b/cbd-tui/src/rpc.rs index 4900923..a4dff1a 100644 --- a/cbd-tui/src/rpc.rs +++ b/cbd-tui/src/rpc.rs @@ -225,8 +225,13 @@ impl RpcClient { &mut self, path: String, name: String, + download: bool, ) -> Result<(), Box<dyn Error>> { - let capture_request = Request::new(CaptureLibraryNodeRequest { path, name }); + let capture_request = Request::new(CaptureLibraryNodeRequest { + path, + name, + download, + }); self.client.capture_library_node(capture_request).await?; Ok(()) } diff --git a/crabidy-core/crabidy/v1/crabidy.proto b/crabidy-core/crabidy/v1/crabidy.proto index 3117cb8..e113bd5 100644 --- a/crabidy-core/crabidy/v1/crabidy.proto +++ b/crabidy-core/crabidy/v1/crabidy.proto @@ -105,8 +105,12 @@ message DeleteLibraryNodeResponse { message CaptureLibraryNodeRequest { // Path of the queueable node (or track) to capture. string path = 1; - // Bookmark name; becomes the top-level folder under /bookmarks. + // Capture name; becomes the top-level folder under /bookmarks (or + // /captures when download is set). string name = 2; + // Download every track's audio into the capture instead of writing + // link files; the source node must set is_downloadable. + bool download = 3; } message CaptureLibraryNodeResponse {} @@ -207,6 +211,8 @@ message LibraryNodeChild { bool is_editable = 5; // This node may be deleted (see DeleteLibraryNode). bool is_deletable = 6; + // This node allows download captures (CaptureLibraryNode with download). + bool is_downloadable = 7; } message QueueModifiers { @@ -268,4 +274,7 @@ message LibraryNode { bool is_queable = 6; // Children may be created under this node (see CreateLibraryNode). bool is_creatable = 7; + // This node allows download captures; its listed tracks inherit the + // flag (CaptureLibraryNode with download). + bool is_downloadable = 8; } diff --git a/crabidy-core/src/lib.rs b/crabidy-core/src/lib.rs index a096130..b8c9c4f 100644 --- a/crabidy-core/src/lib.rs +++ b/crabidy-core/src/lib.rs @@ -209,14 +209,16 @@ impl LibraryNode { tracks: Vec::new(), is_queable: false, is_creatable: false, + is_downloadable: false, } } } impl LibraryNodeChild { /// A regular, non-creatable, immutable child. Creatable/editable/ - /// deletable children (e.g. the search node and its terms) set the - /// capability flags explicitly via struct update. + /// deletable/downloadable children (e.g. the search node and its + /// terms, or Tidal's queueable subtrees) set the capability flags + /// explicitly via struct update. pub fn new(path: String, title: String, is_queable: bool) -> Self { Self { path, @@ -225,6 +227,7 @@ impl LibraryNodeChild { is_creatable: false, is_editable: false, is_deletable: false, + is_downloadable: false, } } } @@ -361,6 +364,7 @@ mod tests { .collect(), is_queable, is_creatable: false, + is_downloadable: false, } } diff --git a/crabidy-server/Cargo.toml b/crabidy-server/Cargo.toml index 8dd2bd8..1755da8 100644 --- a/crabidy-server/Cargo.toml +++ b/crabidy-server/Cargo.toml @@ -17,6 +17,7 @@ flume.workspace = true fsdy.workspace = true futures.workspace = true rand.workspace = true +reqwest.workspace = true serde.workspace = true thiserror.workspace = true tidaldy.workspace = true diff --git a/crabidy-server/src/bookmark_store.rs b/crabidy-server/src/bookmark_store.rs index 2b69e3a..390c3f4 100644 --- a/crabidy-server/src/bookmark_store.rs +++ b/crabidy-server/src/bookmark_store.rs @@ -17,10 +17,10 @@ pub const BOOKMARKS_PROVIDER_ROOT: &str = "/bookmarks"; /// The walk aborts beyond this many directories — a runaway provider tree /// must not fill the disk. -pub const MAX_CAPTURE_DIRS: usize = 1_000; +pub const MAX_CAPTURE_DIRS: usize = crate::capture::BOOKMARK_CAPS.max_dirs; /// The walk aborts beyond this many track files. -pub const MAX_CAPTURE_TRACKS: usize = 20_000; +pub const MAX_CAPTURE_TRACKS: usize = crate::capture::BOOKMARK_CAPS.max_tracks; /// The bookmarks directory: `bookmarks/` inside the crabidy config /// directory. `None` when the platform has no config directory. @@ -28,26 +28,7 @@ pub fn bookmarks_dir() -> Option<PathBuf> { dirs::config_dir().map(|d| d.join("crabidy").join("bookmarks")) } -/// Errors from validating or writing a capture. -/// -/// At the RPC boundary: `InvalidName`/`BadSource` → `invalid_argument`, -/// `TooLarge` → `failed_precondition`, the rest → `internal`. Messages -/// carry names and paths, never file contents. -#[derive(Debug, thiserror::Error)] -pub enum CaptureError { - #[error("invalid bookmark name: {0}")] - InvalidName(&'static str), - #[error("bookmarks are disabled")] - Disabled, - #[error("the source path cannot be captured: {0}")] - BadSource(String), - #[error("the subtree is too large to capture ({0})")] - TooLarge(&'static str), - #[error("cannot write bookmark: {0}")] - Io(#[from] std::io::Error), - #[error(transparent)] - TrackFile(#[from] fsdy::TrackFileError), -} +pub use crate::capture::CaptureError; /// Writes captured subtrees. All I/O is `tokio::fs`; the whole bookmark is /// built as a hidden temp sibling and swapped into place, so a crash never @@ -113,88 +94,20 @@ impl BookmarkStore { where C: ProviderClient + Sync, { - let name = fsdy::validate_folder_name(name, &[]).map_err(CaptureError::InvalidName)?; - let tmp = self.dir.join(format!(".tmp-{name}")); - let written = self - .write_capture(client, source_path, &tmp, max_dirs, max_tracks) - .await; - if let Err(err) = written { - // Every failure path removes the temp folder: nothing - // half-written survives, not even hidden. - let _ = tokio::fs::remove_dir_all(&tmp).await; - return Err(err); - } - let target = self.dir.join(name); - if tokio::fs::try_exists(&target).await? { - tokio::fs::remove_dir_all(&target).await?; - } - tokio::fs::rename(&tmp, &target).await?; - Ok(()) - } - - /// Builds the mirrored tree inside `tmp`. All-or-nothing: any provider - /// or write failure aborts the whole capture (the caller removes - /// `tmp`) — a bookmark that *looks* complete must *be* complete. - async fn write_capture<C>( - &self, - client: &C, - source_path: &str, - tmp: &Path, - max_dirs: usize, - max_tracks: usize, - ) -> Result<(), CaptureError> - where - C: ProviderClient + Sync, - { - // A leftover temp folder from a crashed or racing capture is stale. - if tokio::fs::try_exists(tmp).await? { - tokio::fs::remove_dir_all(tmp).await?; - } - tokio::fs::create_dir_all(tmp).await?; - - // A track source captures as a folder with one file. - if client.is_track_path(source_path) { - let track = client - .get_metadata_for_track(source_path) - .await - .map_err(|err| CaptureError::BadSource(format!("{source_path}: {err}")))?; - let text = fsdy::TrackFile::from_track(&track).to_toml()?; - let file = tmp.join(fsdy::track_file_name(0, &track.title)); - tokio::fs::write(file, text).await?; - return Ok(()); - } - - let mut dirs = 1usize; - let mut tracks = 0usize; - // Iterative pre-order: a deep provider tree must not overflow the - // stack. Each entry pairs a library path with its mirror folder. - let mut worklist: Vec<(String, PathBuf)> = - vec![(source_path.to_string(), tmp.to_path_buf())]; - while let Some((lib_path, dir)) = worklist.pop() { - let node = client - .get_lib_node(&lib_path) - .await - .map_err(|err| CaptureError::BadSource(format!("{lib_path}: {err}")))?; - for (index, track) in node.tracks.iter().enumerate() { - tracks += 1; - if tracks > max_tracks { - return Err(CaptureError::TooLarge("too many tracks")); - } - let text = fsdy::TrackFile::from_track(track).to_toml()?; - let file = dir.join(fsdy::track_file_name(index, &track.title)); - tokio::fs::write(file, text).await?; - } - for (index, child) in node.children.iter().enumerate() { - dirs += 1; - if dirs > max_dirs { - return Err(CaptureError::TooLarge("too many directories")); - } - let child_dir = dir.join(fsdy::dir_name(index, &child.title)); - tokio::fs::create_dir(&child_dir).await?; - worklist.push((child.path.clone(), child_dir)); - } - } - Ok(()) + let caps = crate::capture::Caps { + max_dirs, + max_tracks, + ..crate::capture::BOOKMARK_CAPS + }; + crate::capture::capture_into( + &self.dir, + client, + source_path, + name, + caps, + &crate::capture::Sink::Link, + ) + .await } } diff --git a/crabidy-server/src/capture.rs b/crabidy-server/src/capture.rs new file mode 100644 index 0000000..30f9297 --- /dev/null +++ b/crabidy-server/src/capture.rs @@ -0,0 +1,414 @@ +//! The shared subtree-capture walk (see `architecture/captures.md` D2). +//! +//! Both bookmark captures (`w`, link files) and download captures (`W`, +//! audio files next to their tomls) mirror a library subtree into a folder: +//! an iterative pre-order walk over [`ProviderClient::get_lib_node`] with +//! order-prefixed names, size caps, a hidden tmp-and-swap write, and +//! all-or-nothing cleanup. This module owns that walk; the two stores only +//! differ in the per-track [`Sink`]. + +use std::path::{Path, PathBuf}; + +use crabidy_core::proto::crabidy::Track; +use crabidy_core::ProviderClient; + +/// Connect timeout for download requests. +pub const DOWNLOAD_CONNECT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); + +/// Total per-track deadline: URL fetch, request, and streaming the whole +/// body. A stalled transfer aborts the capture instead of hanging it. +pub const DOWNLOAD_TRACK_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(600); + +/// Size limits for one capture. The walk aborts with +/// [`CaptureError::TooLarge`] when a limit trips — a runaway provider tree +/// or oversized stream must not fill the disk. +#[derive(Debug, Clone, Copy)] +pub struct Caps { + /// Maximum directories (the capture root counts as the first). + pub max_dirs: usize, + /// Maximum track files. + pub max_tracks: usize, + /// Maximum total downloaded bytes (ignored by [`Sink::Link`]). + pub max_bytes: u64, +} + +/// Caps for bookmark (link) captures: link files are tiny, so only the +/// tree size is bounded. +pub const BOOKMARK_CAPS: Caps = Caps { + max_dirs: 1_000, + max_tracks: 20_000, + max_bytes: u64::MAX, +}; + +/// Caps for download captures: fewer tracks and a 4 GiB byte budget. +pub const DOWNLOAD_CAPS: Caps = Caps { + max_dirs: 1_000, + max_tracks: 500, + max_bytes: 4 * 1024 * 1024 * 1024, +}; + +/// Errors from validating or writing a capture. +/// +/// At the RPC boundary: `InvalidName`/`BadSource` → `invalid_argument`, +/// `TooLarge`/`Disabled`/`Unsupported` → `failed_precondition`, the rest → +/// `internal`. Messages carry names, paths, and counts, never file +/// contents. +#[derive(Debug, thiserror::Error)] +pub enum CaptureError { + #[error("invalid name: {0}")] + InvalidName(&'static str), + #[error("the store is disabled")] + Disabled, + #[error("the source path cannot be captured: {0}")] + BadSource(String), + #[error("the source does not allow downloads")] + Unsupported, + #[error("the subtree is too large to capture ({0})")] + TooLarge(&'static str), + #[error("download failed: {0}")] + Download(String), + #[error("cannot write capture: {0}")] + Io(#[from] std::io::Error), + #[error(transparent)] + TrackFile(#[from] fsdy::TrackFileError), +} + +/// What happens to each track the walk visits. +#[derive(Debug)] +pub enum Sink { + /// Write an order-prefixed link file ([`fsdy::TrackFile::from_track`]) + /// — the bookmark behavior. + Link, + /// Download the track's audio next to an order-prefixed toml that + /// points at it — the captures behavior. + Download(Downloader), +} + +/// Downloads one track's audio via the provider's stream URL. +/// +/// One shared HTTP client with a connect timeout; each track is bounded by +/// [`DOWNLOAD_TRACK_TIMEOUT`] end to end and never retried (a capture is +/// re-runnable; overwrite = refresh). Bodies are streamed to disk against +/// the capture's remaining byte budget. +#[derive(Debug)] +pub struct Downloader { + http: reqwest::Client, +} + +impl Downloader { + /// Builds the shared HTTP client. Fails only when the TLS backend + /// cannot initialize. + pub fn new() -> Result<Self, reqwest::Error> { + let http = reqwest::Client::builder() + .connect_timeout(DOWNLOAD_CONNECT_TIMEOUT) + .build()?; + Ok(Self { http }) + } + + /// Downloads one track: audio file first, then the toml pointing at + /// it — a toml never exists without its audio. The whole operation is + /// bounded by [`DOWNLOAD_TRACK_TIMEOUT`]. + /// + /// Error messages carry the track's library path, never the stream + /// URL (it may embed a token) — reqwest errors are stripped with + /// [`reqwest::Error::without_url`]. + async fn download_track<C>( + &self, + client: &C, + track: &Track, + dir: &Path, + index: usize, + bytes_left: &mut u64, + ) -> Result<(), CaptureError> + where + C: ProviderClient + Sync, + { + let fetched = tokio::time::timeout( + DOWNLOAD_TRACK_TIMEOUT, + self.fetch_track(client, track, dir, index, bytes_left), + ) + .await; + match fetched { + Ok(result) => result, + Err(_) => Err(CaptureError::Download(format!( + "{}: timed out after {}s", + track.path, + DOWNLOAD_TRACK_TIMEOUT.as_secs() + ))), + } + } + + /// The unbounded body of [`Self::download_track`]: resolve the stream + /// URL, stream the response to disk against the byte budget, then + /// write the toml. + async fn fetch_track<C>( + &self, + client: &C, + track: &Track, + dir: &Path, + index: usize, + bytes_left: &mut u64, + ) -> Result<(), CaptureError> + where + C: ProviderClient + Sync, + { + let track_path = track.path.as_str(); + let urls = client + .get_urls_for_track(track_path) + .await + .map_err(|err| CaptureError::BadSource(format!("{track_path}: {err}")))?; + let url = urls + .first() + .ok_or_else(|| CaptureError::BadSource(format!("{track_path}: no stream url")))?; + let download_err = |err: reqwest::Error| { + CaptureError::Download(format!("{track_path}: {}", err.without_url())) + }; + let mut response = self + .http + .get(url) + .send() + .await + .map_err(download_err)? + .error_for_status() + .map_err(download_err)?; + let content_type = response + .headers() + .get(reqwest::header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .map(str::to_string); + let ext = extension_for(content_type.as_deref(), url); + let audio_name = audio_file_name(index, &track.title, &ext); + let mut audio = tokio::fs::File::create(dir.join(&audio_name)).await?; + while let Some(chunk) = response.chunk().await.map_err(download_err)? { + let len = chunk.len() as u64; + if len > *bytes_left { + return Err(CaptureError::TooLarge("download budget exhausted")); + } + *bytes_left -= len; + tokio::io::AsyncWriteExt::write_all(&mut audio, &chunk).await?; + } + tokio::io::AsyncWriteExt::flush(&mut audio).await?; + drop(audio); + + let text = + fsdy::TrackFile::from_track_with_file(track, Path::new(&audio_name)).to_toml()?; + tokio::fs::write(dir.join(fsdy::track_file_name(index, &track.title)), text).await?; + Ok(()) + } +} + +/// Captures the subtree at `source_path` as `dir/<name>/`, overwriting an +/// existing folder of that name. +/// +/// Validates `name` ([`fsdy::validate_folder_name`], nothing reserved), +/// builds the whole capture in a hidden `.tmp-<name>` sibling via +/// [`write_tree`], and swaps it into place. Every failure removes the temp +/// folder — nothing half-written survives, not even hidden. +pub async fn capture_into<C>( + dir: &Path, + client: &C, + source_path: &str, + name: &str, + caps: Caps, + sink: &Sink, +) -> Result<(), CaptureError> +where + C: ProviderClient + Sync, +{ + let name = fsdy::validate_folder_name(name, &[]).map_err(CaptureError::InvalidName)?; + let tmp = dir.join(format!(".tmp-{name}")); + let written = write_tree(client, source_path, &tmp, caps, sink).await; + if let Err(err) = written { + // Every failure path removes the temp folder: nothing + // half-written survives, not even hidden. + let _ = tokio::fs::remove_dir_all(&tmp).await; + return Err(err); + } + let target = dir.join(name); + if tokio::fs::try_exists(&target).await? { + tokio::fs::remove_dir_all(&target).await?; + } + tokio::fs::rename(&tmp, &target).await?; + Ok(()) +} + +/// Builds the mirrored tree inside `tmp`: iterative pre-order over +/// [`ProviderClient::get_lib_node`] (a deep tree must not overflow the +/// stack), every child node an order-prefixed folder ([`fsdy::dir_name`]), +/// every track handed to `sink` with its listing index. A `source_path` +/// that is a track captures as a folder with one entry. All-or-nothing: +/// any provider, download, or write failure aborts the whole capture. +async fn write_tree<C>( + client: &C, + source_path: &str, + tmp: &Path, + caps: Caps, + sink: &Sink, +) -> Result<(), CaptureError> +where + C: ProviderClient + Sync, +{ + // A leftover temp folder from a crashed or racing capture is stale. + if tokio::fs::try_exists(tmp).await? { + tokio::fs::remove_dir_all(tmp).await?; + } + tokio::fs::create_dir_all(tmp).await?; + let mut bytes_left = caps.max_bytes; + + // A track source captures as a folder with one entry. + if client.is_track_path(source_path) { + let track = client + .get_metadata_for_track(source_path) + .await + .map_err(|err| CaptureError::BadSource(format!("{source_path}: {err}")))?; + return write_track(sink, client, &track, tmp, 0, &mut bytes_left).await; + } + + let mut dirs = 1usize; + let mut tracks = 0usize; + // Iterative pre-order: a deep provider tree must not overflow the + // stack. Each entry pairs a library path with its mirror folder. + let mut worklist: Vec<(String, PathBuf)> = vec![(source_path.to_string(), tmp.to_path_buf())]; + while let Some((lib_path, dir)) = worklist.pop() { + let node = client + .get_lib_node(&lib_path) + .await + .map_err(|err| CaptureError::BadSource(format!("{lib_path}: {err}")))?; + for (index, track) in node.tracks.iter().enumerate() { + tracks += 1; + if tracks > caps.max_tracks { + return Err(CaptureError::TooLarge("too many tracks")); + } + write_track(sink, client, track, &dir, index, &mut bytes_left).await?; + } + for (index, child) in node.children.iter().enumerate() { + dirs += 1; + if dirs > caps.max_dirs { + return Err(CaptureError::TooLarge("too many directories")); + } + let child_dir = dir.join(fsdy::dir_name(index, &child.title)); + tokio::fs::create_dir(&child_dir).await?; + worklist.push((child.path.clone(), child_dir)); + } + } + Ok(()) +} + +/// Writes one visited track into `dir` at listing position `index`, +/// drawing downloaded bytes from `bytes_left`. +async fn write_track<C>( + sink: &Sink, + client: &C, + track: &Track, + dir: &Path, + index: usize, + bytes_left: &mut u64, +) -> Result<(), CaptureError> +where + C: ProviderClient + Sync, +{ + match sink { + Sink::Link => { + let text = fsdy::TrackFile::from_track(track).to_toml()?; + let file = dir.join(fsdy::track_file_name(index, &track.title)); + tokio::fs::write(file, text).await?; + Ok(()) + } + Sink::Download(downloader) => { + downloader + .download_track(client, track, dir, index, bytes_left) + .await + } + } +} + +/// Picks the audio file extension: the response `Content-Type` first +/// (`audio/flac` → `flac`, `audio/mp4`/`audio/m4a` → `m4a`, `audio/mpeg` → +/// `mp3`, `audio/ogg` → `ogg`, `audio/wav` → `wav`), then the URL path's +/// extension, then `bin` — the player probes by content, the extension is +/// a hint. +fn extension_for(content_type: Option<&str>, url: &str) -> String { + let mapped = content_type + .and_then(|ct| ct.split(';').next()) + .map(|essence| essence.trim().to_ascii_lowercase()) + .and_then(|essence| match essence.as_str() { + "audio/flac" | "audio/x-flac" => Some("flac"), + "audio/mp4" | "audio/m4a" | "audio/x-m4a" => Some("m4a"), + "audio/mpeg" | "audio/mp3" => Some("mp3"), + "audio/ogg" => Some("ogg"), + "audio/wav" | "audio/x-wav" => Some("wav"), + _ => None, + }); + match mapped.or_else(|| url_extension(url)) { + Some(ext) => ext.to_string(), + None => "bin".to_string(), + } +} + +/// The extension of a URL's last path segment (query and fragment +/// stripped), when it looks like one: short and alphanumeric. +fn url_extension(url: &str) -> Option<&str> { + let path = url.split(['?', '#']).next()?; + let segment = path.rsplit('/').next()?; + let (stem, ext) = segment.rsplit_once('.')?; + let plausible = !stem.is_empty() + && !ext.is_empty() + && ext.len() <= 5 + && ext.chars().all(|c| c.is_ascii_alphanumeric()); + plausible.then_some(ext) +} + +/// The audio file name for track `index` titled `title` with `ext`, next +/// to its toml: `NNNN <title>.<ext>` through the same sanitizer as the +/// toml name, so the pair sorts together — and what the toml's relative +/// `file` playable points at. +fn audio_file_name(index: usize, title: &str, ext: &str) -> String { + format!("{}.{ext}", fsdy::dir_name(index, title)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn extension_prefers_content_type_then_url_then_bin() { + for (ct, ext) in [ + ("audio/flac", "flac"), + ("audio/mp4", "m4a"), + ("audio/m4a", "m4a"), + ("audio/mpeg", "mp3"), + ("audio/ogg", "ogg"), + ("audio/wav", "wav"), + ] { + assert_eq!(extension_for(Some(ct), "https://x.test/s"), ext); + } + // Content-Type parameters must not confuse the mapping. + assert_eq!( + extension_for(Some("audio/flac; charset=binary"), "https://x.test/s"), + "flac" + ); + // Unknown or missing types fall back to the URL path's extension… + assert_eq!( + extension_for(None, "https://x.test/media/track.m4a?token=abc"), + "m4a" + ); + assert_eq!( + extension_for(Some("application/octet-stream"), "https://x.test/a.flac"), + "flac" + ); + // …and to `bin` when the URL has none either. + assert_eq!(extension_for(None, "https://x.test/stream"), "bin"); + assert_eq!(extension_for(None, "not a url"), "bin"); + } + + #[test] + fn audio_files_pair_with_their_toml_names() { + // Same prefix and sanitized stem as `fsdy::track_file_name`, so the + // audio file sorts right next to its toml. + assert_eq!( + audio_file_name(0, "We Will Rock You", "flac"), + "0001 We Will Rock You.flac" + ); + assert_eq!(audio_file_name(11, "a/b", "mp3"), "0012 a_b.mp3"); + } +} diff --git a/crabidy-server/src/capture_store.rs b/crabidy-server/src/capture_store.rs new file mode 100644 index 0000000..646a4de --- /dev/null +++ b/crabidy-server/src/capture_store.rs @@ -0,0 +1,442 @@ +//! Downloaded library subtrees ("captures") on disk +//! (see `architecture/captures.md`). +//! +//! Every capture is a folder under `<config>/crabidy/captures/` that +//! mirrors the captured subtree like a bookmark, except each track's audio +//! is **downloaded** next to its order-prefixed `*.cbd-track.toml`, and the +//! toml's playable is a relative `file` pointing at it — replaying a +//! capture needs no provider round trip. The same directory is mounted +//! read-only into the library as `/captures` by an `fsdy` instance (with +//! editable top-level folders); this module is the only writer. + +use std::path::{Path, PathBuf}; + +use crabidy_core::ProviderClient; + +use crate::capture::{Caps, CaptureError}; + +/// The library mount point of the captures directory. +pub const CAPTURES_PROVIDER_ROOT: &str = "/captures"; + +/// The captures directory: `captures/` inside the crabidy config +/// directory. `None` when the platform has no config directory. +pub fn captures_dir() -> Option<PathBuf> { + dirs::config_dir().map(|d| d.join("crabidy").join("captures")) +} + +/// Writes download captures. All audio is fetched through one shared HTTP +/// client; the whole capture is built as a hidden temp sibling and swapped +/// into place, so a crash or failed download never leaves a half-written +/// capture next to intact ones. +#[derive(Debug)] +pub struct CaptureStore { + dir: PathBuf, + sink: crate::capture::Sink, +} + +impl CaptureStore { + /// Opens the store at `dir`, creating the directory (and parents) if + /// missing, and builds the shared HTTP client. + pub async fn open(dir: PathBuf) -> Result<Self, CaptureError> { + tokio::fs::create_dir_all(&dir).await?; + let downloader = crate::capture::Downloader::new() + .map_err(|err| CaptureError::Download(format!("cannot build http client: {err}")))?; + Ok(Self { + dir, + sink: crate::capture::Sink::Download(downloader), + }) + } + + /// The store directory (what the `/captures` provider instance + /// mounts). + pub fn dir(&self) -> &Path { + &self.dir + } + + /// Captures the subtree at `source_path` as the capture `name`, + /// downloading every track's audio; overwrites an existing capture of + /// that name. + /// + /// The capture **root** must opt in (`architecture/captures.md` D4): a + /// directory source must report `is_downloadable`, a track source's + /// parent node must — otherwise [`CaptureError::Unsupported`]. The + /// walk, caps ([`crate::capture::DOWNLOAD_CAPS`]), tmp-and-swap, and + /// all-or-nothing cleanup are [`crate::capture::capture_into`]'s. + pub async fn capture<C>( + &self, + client: &C, + source_path: &str, + name: &str, + ) -> Result<(), CaptureError> + where + C: ProviderClient + Sync, + { + self.capture_with_caps(client, source_path, name, crate::capture::DOWNLOAD_CAPS) + .await + } + + /// [`Self::capture`] with explicit caps — the seam the cap tests use. + async fn capture_with_caps<C>( + &self, + client: &C, + source_path: &str, + name: &str, + caps: Caps, + ) -> Result<(), CaptureError> + where + C: ProviderClient + Sync, + { + source_allows_download(client, source_path).await?; + crate::capture::capture_into(&self.dir, client, source_path, name, caps, &self.sink).await + } +} + +/// Checks that the capture root allows downloads: the node at +/// `source_path` (or, for a track path, its parent node) must set +/// `is_downloadable`. An unreadable root is [`CaptureError::BadSource`]. +async fn source_allows_download<C>(client: &C, source_path: &str) -> Result<(), CaptureError> +where + C: ProviderClient + Sync, +{ + let node_path = if client.is_track_path(source_path) { + crabidy_core::parent_path(source_path).unwrap_or(source_path) + } else { + source_path + }; + let node = client + .get_lib_node(node_path) + .await + .map_err(|err| CaptureError::BadSource(format!("{node_path}: {err}")))?; + if node.is_downloadable { + Ok(()) + } else { + Err(CaptureError::Unsupported) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::capture::{Caps, DOWNLOAD_CAPS}; + use async_trait::async_trait; + use crabidy_core::proto::crabidy::{LibraryNode, LibraryNodeChild, Track}; + use crabidy_core::ProviderError; + use std::collections::HashMap; + use std::fs; + use std::path::Path; + use tempfile::TempDir; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + /// Serves every request with one fixed response and returns a URL for + /// it. Minimal HTTP/1.1 on a loopback socket — enough for reqwest. + async fn serve(status: &'static str, content_type: &'static str, body: Vec<u8>) -> String { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind test server"); + let addr = listener.local_addr().expect("test server addr"); + tokio::spawn(async move { + while let Ok((mut sock, _)) = listener.accept().await { + let body = body.clone(); + tokio::spawn(async move { + // Read until the header terminator; the request itself + // is irrelevant. + let mut buf = Vec::new(); + let mut chunk = [0u8; 1024]; + loop { + match sock.read(&mut chunk).await { + Ok(0) | Err(_) => break, + Ok(n) => { + buf.extend_from_slice(&chunk[..n]); + if buf.windows(4).any(|w| w == b"\r\n\r\n") { + break; + } + } + } + } + let head = format!( + "HTTP/1.1 {status}\r\nContent-Type: {content_type}\r\n\ + Content-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ); + let _ = sock.write_all(head.as_bytes()).await; + let _ = sock.write_all(&body).await; + let _ = sock.shutdown().await; + }); + } + }); + format!("http://{addr}/stream") + } + + /// A provider with one album (`/mock/a`: tracks `one`, `two`) whose + /// stream URLs are the test server's, and a node-level download + /// blessing toggle. + #[derive(Debug)] + struct MockProvider { + urls: HashMap<String, String>, + downloadable: bool, + } + + impl MockProvider { + fn new(urls: &[(&str, &str)], downloadable: bool) -> Self { + Self { + urls: urls + .iter() + .map(|(p, u)| (p.to_string(), u.to_string())) + .collect(), + downloadable, + } + } + + fn track(&self, path: &str) -> Track { + let title = match path { + "/mock/a/1" => "one", + "/mock/a/2" => "two", + other => other, + }; + Track { + path: path.to_string(), + artist: "mock".to_string(), + title: title.to_string(), + duration: Some(10), + album: None, + } + } + } + + #[async_trait] + impl ProviderClient for MockProvider { + async fn init(_s: &str) -> Result<Self, ProviderError> { + Err(ProviderError::NotSupported) + } + fn settings(&self) -> String { + String::new() + } + fn is_track_path(&self, path: &str) -> bool { + path.starts_with("/mock/a/") && path.len() > "/mock/a/".len() + } + async fn get_urls_for_track(&self, track_path: &str) -> Result<Vec<String>, ProviderError> { + self.urls + .get(track_path) + .map(|u| vec![u.clone()]) + .ok_or(ProviderError::FetchError) + } + async fn get_metadata_for_track(&self, track_path: &str) -> Result<Track, ProviderError> { + if !self.is_track_path(track_path) { + return Err(ProviderError::MalformedPath); + } + Ok(self.track(track_path)) + } + fn get_lib_root(&self) -> LibraryNode { + LibraryNode::new() + } + async fn get_lib_node(&self, path: &str) -> Result<LibraryNode, ProviderError> { + let mut node = LibraryNode::new(); + node.path = path.to_string(); + node.is_downloadable = self.downloadable; + match path { + "/mock" => { + node.title = "mock".to_string(); + node.children = vec![LibraryNodeChild { + is_downloadable: self.downloadable, + ..LibraryNodeChild::new("/mock/a".to_string(), "a".to_string(), true) + }]; + } + "/mock/a" => { + node.title = "a".to_string(); + node.is_queable = true; + node.tracks = vec![self.track("/mock/a/1"), self.track("/mock/a/2")]; + } + _ => return Err(ProviderError::MalformedPath), + } + Ok(node) + } + async fn create_lib_node( + &self, + _parent_path: &str, + _title: &str, + ) -> Result<LibraryNode, ProviderError> { + Err(ProviderError::NotSupported) + } + async fn rename_lib_node( + &self, + _path: &str, + _new_title: &str, + ) -> Result<LibraryNode, ProviderError> { + Err(ProviderError::NotSupported) + } + async fn delete_lib_node(&self, _path: &str) -> Result<LibraryNode, ProviderError> { + Err(ProviderError::NotSupported) + } + } + + async fn store() -> (CaptureStore, TempDir) { + let dir = TempDir::new().expect("store tempdir"); + let store = CaptureStore::open(dir.path().join("captures")) + .await + .expect("open creates the directory"); + (store, dir) + } + + fn visible(dir: &Path) -> Vec<String> { + let mut names: Vec<String> = fs::read_dir(dir) + .expect("dir") + .map(|e| e.expect("entry").file_name().to_string_lossy().into_owned()) + .filter(|n| !n.starts_with('.')) + .collect(); + names.sort_by_key(|n| n.to_lowercase()); + names + } + + #[tokio::test] + async fn download_capture_writes_audio_next_to_pointing_tomls() { + let url = serve("200 OK", "audio/flac", b"flacbytes".to_vec()).await; + let mock = MockProvider::new(&[("/mock/a/1", &url), ("/mock/a/2", &url)], true); + let (store, _dir) = store().await; + store + .capture(&mock, "/mock/a", "faves") + .await + .expect("capture"); + + let root = store.dir().join("faves"); + assert_eq!( + visible(&root), + vec![ + "0001 one.cbd-track.toml".to_string(), + "0001 one.flac".into(), + "0002 two.cbd-track.toml".into(), + "0002 two.flac".into(), + ] + ); + assert_eq!( + fs::read(root.join("0001 one.flac")).expect("audio"), + b"flacbytes" + ); + // The toml points at its sibling with a *relative* file playable. + let toml = fs::read_to_string(root.join("0002 two.cbd-track.toml")).expect("toml"); + let file = fsdy::TrackFile::parse(&toml).expect("parses"); + assert_eq!( + file.playable().expect("playable"), + fsdy::Playable::File("0002 two.flac".into()) + ); + assert_eq!(file.title, "two"); + + // Replay: the capture resolves through a /captures instance and the + // audio resolves to the absolute sibling path — no provider round + // trip left. + let captures = fsdy::Client::new(CAPTURES_PROVIDER_ROOT, store.dir().to_path_buf()) + .expect("captures instance"); + let (chunk_tx, chunk_rx) = flume::bounded(8); + captures + .resolve_tracks_into("/captures/faves", chunk_tx) + .await + .expect("resolve"); + let tracks: Vec<Track> = chunk_rx.into_iter().flatten().collect(); + assert_eq!(tracks.len(), 2); + let urls = captures + .get_urls_for_track(&tracks[0].path) + .await + .expect("urls"); + assert_eq!(urls, vec![root.join("0001 one.flac").display().to_string()]); + } + + #[tokio::test] + async fn capture_requires_the_root_download_blessing() { + let url = serve("200 OK", "audio/flac", b"x".to_vec()).await; + let mock = MockProvider::new(&[("/mock/a/1", &url), ("/mock/a/2", &url)], false); + let (store, _dir) = store().await; + assert!(matches!( + store.capture(&mock, "/mock/a", "faves").await, + Err(CaptureError::Unsupported) + )); + assert!(matches!( + store.capture(&mock, "/mock/a/1", "faves").await, + Err(CaptureError::Unsupported) + )); + assert_eq!(fs::read_dir(store.dir()).expect("store dir").count(), 0); + } + + #[tokio::test] + async fn capturing_a_single_blessed_track_writes_one_pair() { + let url = serve("200 OK", "audio/mpeg", b"mp3bytes".to_vec()).await; + let mock = MockProvider::new(&[("/mock/a/1", &url)], true); + let (store, _dir) = store().await; + store + .capture(&mock, "/mock/a/1", "just one") + .await + .expect("capture track"); + assert_eq!( + visible(&store.dir().join("just one")), + vec!["0001 one.cbd-track.toml".to_string(), "0001 one.mp3".into()] + ); + } + + #[tokio::test] + async fn download_capture_is_all_or_nothing() { + let ok = serve("200 OK", "audio/flac", b"x".to_vec()).await; + let gone = serve("404 Not Found", "text/plain", Vec::new()).await; + let mock = MockProvider::new(&[("/mock/a/1", &ok), ("/mock/a/2", &gone)], true); + let (store, _dir) = store().await; + let err = store + .capture(&mock, "/mock/a", "faves") + .await + .expect_err("a failed download aborts the capture"); + assert!(matches!(err, CaptureError::Download(_)), "got {err:?}"); + // Nothing half-written survives, not even hidden temp folders. + assert_eq!(fs::read_dir(store.dir()).expect("store dir").count(), 0); + } + + #[tokio::test] + async fn download_capture_enforces_its_caps() { + let url = serve("200 OK", "audio/flac", b"0123456789".to_vec()).await; + let mock = MockProvider::new(&[("/mock/a/1", &url), ("/mock/a/2", &url)], true); + let (store, _dir) = store().await; + // Byte budget: two 10-byte bodies against a 15-byte budget. + let caps = Caps { + max_bytes: 15, + ..DOWNLOAD_CAPS + }; + let err = store + .capture_with_caps(&mock, "/mock/a", "big", caps) + .await + .expect_err("over the byte budget"); + assert!(matches!(err, CaptureError::TooLarge(_)), "got {err:?}"); + // Track cap. + let caps = Caps { + max_tracks: 1, + ..DOWNLOAD_CAPS + }; + let err = store + .capture_with_caps(&mock, "/mock/a", "big", caps) + .await + .expect_err("over the track cap"); + assert!(matches!(err, CaptureError::TooLarge(_)), "got {err:?}"); + assert_eq!(fs::read_dir(store.dir()).expect("store dir").count(), 0); + } + + #[tokio::test] + async fn capture_validates_names_and_overwrites() { + let url = serve("200 OK", "audio/flac", b"x".to_vec()).await; + let mock = MockProvider::new(&[("/mock/a/1", &url), ("/mock/a/2", &url)], true); + let (store, _dir) = store().await; + assert!(matches!( + store.capture(&mock, "/mock/a", "a/b").await, + Err(CaptureError::InvalidName(_)) + )); + store + .capture(&mock, "/mock/a", "faves") + .await + .expect("first capture"); + store + .capture(&mock, "/mock/a/1", "faves") + .await + .expect("overwrite"); + // The overwrite fully replaces the older, larger capture. + assert_eq!( + visible(&store.dir().join("faves")), + vec![ + "0001 one.cbd-track.toml".to_string(), + "0001 one.flac".into() + ] + ); + } +} diff --git a/crabidy-server/src/lib.rs b/crabidy-server/src/lib.rs index 911c0fe..7d9cd9f 100644 --- a/crabidy-server/src/lib.rs +++ b/crabidy-server/src/lib.rs @@ -1,4 +1,6 @@ pub mod bookmark_store; +pub mod capture; +pub mod capture_store; pub mod queue_store; use crabidy_core::proto::crabidy::{Queue, Track}; diff --git a/crabidy-server/src/main.rs b/crabidy-server/src/main.rs index 19ee676..082895d 100644 --- a/crabidy-server/src/main.rs +++ b/crabidy-server/src/main.rs @@ -205,12 +205,16 @@ pub enum ProviderCommand { result_tx: flume::Sender<Result<LibraryNode, ProviderError>>, }, /// Captures the queueable subtree at `path` as the bookmark `name` - /// (see `architecture/bookmarks.md` D1–D3). Handled on a spawned task — - /// a large walk must not block the orchestrator loop. + /// (see `architecture/bookmarks.md` D1–D3) — or, with `download`, as + /// the download capture `name` under `/captures`, fetching every + /// track's audio (see `architecture/captures.md`). Handled on a + /// spawned task — a large walk or download must not block the + /// orchestrator loop. CaptureLibraryNode { path: String, name: String, - result_tx: flume::Sender<Result<(), crabidy_server::bookmark_store::CaptureError>>, + download: bool, + result_tx: flume::Sender<Result<(), crabidy_server::capture::CaptureError>>, }, } diff --git a/crabidy-server/src/provider.rs b/crabidy-server/src/provider.rs index c17e8f7..1e0c9df 100644 --- a/crabidy-server/src/provider.rs +++ b/crabidy-server/src/provider.rs @@ -5,6 +5,7 @@ use crabidy_core::{ ProviderClient, ProviderError, }; use crabidy_server::bookmark_store::{BookmarkStore, BOOKMARKS_PROVIDER_ROOT}; +use crabidy_server::capture_store::{CaptureStore, CAPTURES_PROVIDER_ROOT}; use crabidy_server::queue_store::{CURRENT_QUEUE_NAME, QUEUES_PROVIDER_ROOT}; use std::{fs, path::PathBuf, sync::Arc}; use tracing::{debug, debug_span, error, instrument, warn, Instrument}; @@ -25,9 +26,16 @@ pub struct ProviderOrchestrator { /// `/bookmarks` (architecture/bookmarks.md D1). `None` without a /// config directory. bookmarks_client: Option<Arc<fsdy::Client>>, - /// The capture writer; `None` disables `CaptureLibraryNode` (and the + /// The bookmark writer; `None` disables link captures (and the /// `/bookmarks` mount goes with it). bookmark_store: Option<Arc<BookmarkStore>>, + /// Fourth `fsdy` instance over the captures folder, mounted at + /// `/captures` (architecture/captures.md D1). `None` without a config + /// directory. + captures_client: Option<Arc<fsdy::Client>>, + /// The download-capture writer; `None` disables download captures + /// (and the `/captures` mount goes with it). + capture_store: Option<Arc<CaptureStore>>, } /// Whether a path belongs to the filesystem provider. @@ -46,6 +54,11 @@ fn bookmarks_owns(path: &str) -> bool { path == BOOKMARKS_PROVIDER_ROOT || path.starts_with("/bookmarks/") } +/// Whether a path belongs to the captures provider instance. +fn captures_owns(path: &str) -> bool { + path == CAPTURES_PROVIDER_ROOT || path.starts_with("/captures/") +} + impl ProviderOrchestrator { /// The fs client, or `MalformedPath` (with a warning) when the /// provider is disabled — a `/fs` path then has no owner. @@ -73,6 +86,15 @@ impl ProviderOrchestrator { ProviderError::MalformedPath }) } + + /// The captures client, or `MalformedPath` (with a warning) when the + /// instance is disabled — a `/captures` path then has no owner. + fn captures_provider(&self) -> Result<&fsdy::Client, ProviderError> { + self.captures_client.as_deref().ok_or_else(|| { + warn!("captures library is disabled"); + ProviderError::MalformedPath + }) + } pub fn run(self) { tokio::spawn(async move { // Behind an Arc so long-running resolves can be spawned onto @@ -148,20 +170,29 @@ impl ProviderOrchestrator { ProviderCommand::CaptureLibraryNode { path, name, + download, result_tx, } => { // Spawned: capturing a large artist walks many provider - // nodes and must not block this loop (the walk itself - // calls back into `get_lib_node` via `this`). + // nodes (and, for downloads, streams audio) and must not + // block this loop (the walk itself calls back into + // `get_lib_node` via `this`). let this = Arc::clone(&self); tokio::spawn( async move { - let result = match &this.bookmark_store { - Some(store) => store.capture(&*this, &path, &name).await, - None => Err(crabidy_server::bookmark_store::CaptureError::Disabled), + let result = if download { + match &this.capture_store { + Some(store) => store.capture(&*this, &path, &name).await, + None => Err(crabidy_server::capture::CaptureError::Disabled), + } + } else { + match &this.bookmark_store { + Some(store) => store.capture(&*this, &path, &name).await, + None => Err(crabidy_server::capture::CaptureError::Disabled), + } }; if let Err(err) = &result { - warn!(path, name, "cannot capture subtree: {err}"); + warn!(path, name, download, "cannot capture subtree: {err}"); } if let Err(err) = result_tx.send_async(result).await { error!("failed to send capture_library_node result: {err}"); @@ -263,6 +294,30 @@ impl ProviderClient for ProviderOrchestrator { } } }); + // Captures: like bookmarks, but the store downloads every track's + // audio next to its toml (architecture/captures.md D1). Non-fatal. + let capture_store = match crabidy_server::capture_store::captures_dir() { + Some(dir) => match CaptureStore::open(dir).await { + Ok(store) => Some(Arc::new(store)), + Err(err) => { + warn!("captures disabled: {err}"); + None + } + }, + None => { + warn!("captures disabled: no config directory"); + None + } + }; + let captures_client = capture_store.as_ref().and_then(|store| { + match fsdy::Client::new(CAPTURES_PROVIDER_ROOT, store.dir().to_path_buf()) { + Ok(client) => Some(Arc::new(client.with_editable_top_level(&[]))), + Err(err) => { + warn!("captures library disabled: {err}"); + None + } + } + }); let (provider_tx, provider_rx) = flume::bounded(100); Ok(Self { provider_rx, @@ -272,6 +327,8 @@ impl ProviderClient for ProviderOrchestrator { queues_client, bookmarks_client, bookmark_store, + captures_client, + capture_store, }) } @@ -302,6 +359,12 @@ impl ProviderClient for ProviderOrchestrator { .as_ref() .is_some_and(|bookmarks| bookmarks.is_track_path(path)); } + if captures_owns(path) { + return self + .captures_client + .as_ref() + .is_some_and(|captures| captures.is_track_path(path)); + } false } @@ -322,6 +385,12 @@ impl ProviderClient for ProviderOrchestrator { .get_urls_for_track(track_path) .await; } + if captures_owns(track_path) { + return self + .captures_provider()? + .get_urls_for_track(track_path) + .await; + } warn!(path = track_path, "no provider owns this track path"); Err(ProviderError::MalformedPath) } @@ -346,6 +415,12 @@ impl ProviderClient for ProviderOrchestrator { .get_metadata_for_track(track_path) .await; } + if captures_owns(track_path) { + return self + .captures_provider()? + .get_metadata_for_track(track_path) + .await; + } warn!(path = track_path, "no provider owns this track path"); Err(ProviderError::MalformedPath) } @@ -373,6 +448,14 @@ impl ProviderClient for ProviderOrchestrator { ); root_node.children.push(child); } + if self.captures_client.is_some() { + let child = LibraryNodeChild::new( + CAPTURES_PROVIDER_ROOT.to_owned(), + "captures".to_owned(), + false, + ); + root_node.children.push(child); + } root_node } @@ -394,6 +477,9 @@ impl ProviderClient for ProviderOrchestrator { if bookmarks_owns(path) { return self.bookmarks_provider()?.get_lib_node(path).await; } + if captures_owns(path) { + return self.captures_provider()?.get_lib_node(path).await; + } warn!(path, "no provider owns this path"); Err(ProviderError::MalformedPath) } @@ -427,6 +513,12 @@ impl ProviderClient for ProviderOrchestrator { .create_lib_node(parent_path, title) .await; } + if captures_owns(parent_path) { + return self + .captures_provider()? + .create_lib_node(parent_path, title) + .await; + } warn!(parent_path, "no provider supports creating nodes here"); Err(ProviderError::NotSupported) } @@ -457,6 +549,12 @@ impl ProviderClient for ProviderOrchestrator { .rename_lib_node(path, new_title) .await; } + if captures_owns(path) { + return self + .captures_provider()? + .rename_lib_node(path, new_title) + .await; + } warn!(path, "no provider supports renaming this node"); Err(ProviderError::NotSupported) } @@ -490,6 +588,12 @@ impl ProviderClient for ProviderOrchestrator { .resolve_tracks_into(path, chunk_tx) .await; } + if captures_owns(path) { + return self + .captures_provider()? + .resolve_tracks_into(path, chunk_tx) + .await; + } warn!(path, "no provider owns this path"); Err(ProviderError::MalformedPath) } @@ -510,6 +614,9 @@ impl ProviderClient for ProviderOrchestrator { if bookmarks_owns(path) { return self.bookmarks_provider()?.delete_lib_node(path).await; } + if captures_owns(path) { + return self.captures_provider()?.delete_lib_node(path).await; + } warn!(path, "no provider supports deleting this node"); Err(ProviderError::NotSupported) } diff --git a/crabidy-server/src/rpc.rs b/crabidy-server/src/rpc.rs index e08a13d..6d3ed7e 100644 --- a/crabidy-server/src/rpc.rs +++ b/crabidy-server/src/rpc.rs @@ -377,26 +377,33 @@ impl CrabidyService for RpcService { } /// Captures a queueable subtree as a bookmark via the provider loop - /// (structure-preserving snapshot under `/bookmarks/<name>`). + /// (structure-preserving snapshot under `/bookmarks/<name>`), or — + /// with `download` — as a download capture under `/captures/<name>`. /// /// Error mapping is part of the contract: invalid name or - /// uncapturable source → `invalid_argument`; an over-cap subtree or - /// disabled bookmarks → `failed_precondition`; walk/write failures → - /// `internal`. - #[instrument(skip(self, request), fields(path, name))] + /// uncapturable source → `invalid_argument`; an over-cap subtree, + /// disabled store, or a source that does not allow downloads → + /// `failed_precondition`; walk/write/download failures → `internal`. + #[instrument(skip(self, request), fields(path, name, download))] async fn capture_library_node( &self, request: Request<CaptureLibraryNodeRequest>, ) -> Result<Response<CaptureLibraryNodeResponse>, Status> { - let CaptureLibraryNodeRequest { path, name } = request.into_inner(); + let CaptureLibraryNodeRequest { + path, + name, + download, + } = request.into_inner(); tracing::Span::current().record("path", path.as_str()); tracing::Span::current().record("name", name.as_str()); + tracing::Span::current().record("download", download); debug!("received capture_library_node request"); let (result_tx, result_rx) = flume::bounded(1); self.provider_tx .send_async(ProviderMessage::new(ProviderCommand::CaptureLibraryNode { path, name, + download, result_tx, })) .await @@ -413,9 +420,11 @@ impl CrabidyService for RpcService { Err(err @ (CaptureError::InvalidName(_) | CaptureError::BadSource(_))) => { Err(Status::invalid_argument(err.to_string())) } - Err(err @ (CaptureError::TooLarge(_) | CaptureError::Disabled)) => { - Err(Status::failed_precondition(err.to_string())) - } + Err( + err @ (CaptureError::TooLarge(_) + | CaptureError::Disabled + | CaptureError::Unsupported), + ) => Err(Status::failed_precondition(err.to_string())), Err(err) => { error!("capture_library_node failed: {err}"); Err(Status::internal("cannot capture the subtree")) diff --git a/fsdy/src/lib.rs b/fsdy/src/lib.rs index 345d1a9..4943886 100644 --- a/fsdy/src/lib.rs +++ b/fsdy/src/lib.rs @@ -201,6 +201,22 @@ impl TrackFile { } } + /// Like [`Self::from_track`], but the playable is a local + /// [`Playable::File`] at `file` instead of a link — used by download + /// captures, where the audio sits next to the toml and `file` is its + /// relative name (see `architecture/captures.md` D3). Relative paths + /// resolve against the track file's directory at play time, so the + /// capture folder stays relocatable. + pub fn from_track_with_file(track: &Track, file: &Path) -> Self { + let mut this = Self::from_track(track); + this.playable = PlayableSpec { + file: Some(file.to_path_buf()), + url: None, + link: None, + }; + this + } + /// Serializes this file to TOML text. /// /// Only fails when TOML cannot represent the value @@ -482,6 +498,7 @@ impl Client { tracks, is_queable: true, is_creatable: false, + is_downloadable: false, }) } @@ -629,6 +646,7 @@ impl ProviderClient for Client { tracks: Vec::new(), is_queable: true, is_creatable: false, + is_downloadable: false, } } @@ -1158,6 +1176,51 @@ mod tests { assert_eq!(restored, track); } + #[tokio::test] + async fn from_track_with_file_plays_the_relative_sibling() { + let track = Track { + path: "/tidal/artists/1/2/3".to_string(), + artist: "Queen".to_string(), + title: "One".to_string(), + duration: Some(90), + album: Some(Album { + title: "Greatest".to_string(), + release_date: None, + }), + }; + let file = TrackFile::from_track_with_file(&track, Path::new("0001 One.flac")); + let toml_text = file.to_toml().expect("serialize"); + let reparsed = TrackFile::parse(&toml_text).expect("reparse"); + assert_eq!( + reparsed.playable().expect("playable"), + Playable::File("0001 One.flac".into()) + ); + // File playables keep the *library* path (no link rewrite), and the + // metadata survives the round trip. + let lib_path = "/captures/faves/0001%20One.cbd-track.toml"; + let listed = reparsed.to_track(lib_path); + assert_eq!(listed.path, lib_path); + assert_eq!(listed.title, track.title); + assert_eq!(listed.album, track.album); + + // End to end: a capture folder replays through an instance and the + // audio resolves to the absolute sibling path. + let dir = TempDir::new().expect("tempdir"); + let faves = dir.path().join("faves"); + fs::create_dir_all(&faves).expect("mkdir"); + fs::write(faves.join("0001 One.flac"), b"flac").expect("audio"); + fs::write(faves.join(track_file_name(0, "One")), toml_text).expect("toml"); + let client = Client::new("/captures", dir.path().to_path_buf()).expect("instance"); + let urls = client + .get_urls_for_track("/captures/faves/0001%20One.cbd-track.toml") + .await + .expect("resolve"); + assert_eq!( + urls, + vec![faves.join("0001 One.flac").display().to_string()] + ); + } + #[test] fn from_track_serializes_sparse_metadata() { // No artist/duration/album: serialization must not fail on `None` diff --git a/plan/captures.md b/plan/captures.md new file mode 100644 index 0000000..dcff75f --- /dev/null +++ b/plan/captures.md @@ -0,0 +1,54 @@ +# Plan: captures + +Ordered tasks; each names its verification (tests in `fsdy/src/lib.rs`, +`crabidy-server/src/capture.rs`, `crabidy-server/src/capture_store.rs` +and/or gates in `quality/captures.md`). Stubs, the proto field, tidal's +flag rule, and the full orchestrator/TUI plumbing exist; the new tests +fail on `todo!()` at plan time. + +- [x] **T1 — fsdy file-playable constructor.** + `TrackFile::from_track_with_file`: metadata like `from_track`, playable + `file = <relative sibling>`. Verifies: + `from_track_with_file_plays_the_relative_sibling`. +- [x] **T2 — capture.rs naming + extension helpers.** `audio_file_name` + (shared `ordered_name` semantics via a public fsdy seam) and + `extension_for` (Content-Type map, URL-path fallback, `bin`). + Verifies: `extension_prefers_content_type_then_url_then_bin`, + `audio_files_pair_with_their_toml_names`. +- [x] **T3 — shared walk.** Move the bookmark walk into + `capture::capture_into`/`write_tree` parameterized by `Caps` and + `Sink`; per-track writes go through `write_track` (`Sink::Link` = + today's link file). Rewire `BookmarkStore` onto it (delete its copy; + keep its public API and `BOOKMARK_CAPS`-equivalent behavior). + Verifies: all existing `bookmark_store::tests` unchanged; gates + "Shared walk". +- [x] **T4 — download sink.** `Downloader::new` (connect timeout), + `Sink::Download` in `write_track`: per-track deadline around URL + fetch, GET, and the streamed body against `bytes_left`, + `error_for_status`, extension from the response, audio first then toml + (`from_track_with_file`). Verifies: + `download_capture_writes_audio_next_to_pointing_tomls`, + `download_capture_is_all_or_nothing`, + `download_capture_enforces_its_caps`; gates "Download sink". +- [x] **T5 — CaptureStore.** `open` (create dir + build downloader), + `capture` → blessing check (`source_allows_download`: node or track + parent) then `capture_with_caps(DOWNLOAD_CAPS)` → `capture_into`. + Verifies: `capture_requires_the_root_download_blessing`, + `capturing_a_single_blessed_track_writes_one_pair`, + `capture_validates_names_and_overwrites`; gates "Opt-in". +- [x] **T6 — TUI tests.** Binding test for `W` (Library scope, shift), + `selected_downloadable` gating (queueable-but-not-downloadable stays + closed; tracks inherit the node flag), overlay label `capture`, + submit carries `download: true`. Verifies: new `cbd-tui` tests; gates + "TUI". +- [x] **T7 — full verification.** Whole workspace suite green; + clippy/fmt/taplo/markdownlint clean; walk every gate in + `quality/captures.md` and tick it; no `todo!()` left. +- [x] **T8 — live smoke test.** Capture a real Tidal track with download + through the provider layer into a temp store (a single track keeps the + probe cheap; the multi-track walk is unit-covered), verify the audio + exists with a plausible size, replay through a `/captures` instance, + and play-resolve it — remove any temporary probe afterwards. +- [x] **T9 — docs.** `plan/summary.md` section incl. deviations; + reconcile `architecture/captures.md` (tidal flag rule, anything else + that moved). diff --git a/plan/summary.md b/plan/summary.md index 8c7bfd6..fad6a76 100644 --- a/plan/summary.md +++ b/plan/summary.md @@ -1,5 +1,52 @@ # Implementation summaries +## captures (2026-07-21) + +Built per `plan/captures.md`: `W` (shift) on a downloadable library +selection captures the subtree like a bookmark, but into +`<config>/crabidy/captures/<name>/` with every track's audio +**downloaded** next to its order-prefixed toml — the toml's playable is +the audio file's *relative* name (`TrackFile::from_track_with_file`), so +a capture plays with no provider round trip and the folder stays +relocatable. `/captures` is a fourth `fsdy` instance (editable top +level, nothing reserved): browse, queue, rename (`e`), delete (`d`), +and re-capture to refresh; the audio files are invisible to listings +(only dirs and `*.cbd-track.toml` count). + +The bookmark walk was extracted into `capture.rs` +(`capture_into`/`write_tree`, `Caps`, one `CaptureError` for both +stores) parameterized by a per-track `Sink` — `Link` is byte-identical +bookmark behavior, `Download` fetches the first `get_urls_for_track` +URL through one shared reqwest client (30 s connect timeout, 600 s +per-track deadline, no retries), streams the body to disk against a +capture-wide byte budget, picks the extension from `Content-Type` (URL +path, then `bin`, as fallbacks), and writes the toml only after the +audio succeeded. Download caps: 1 000 dirs, 500 tracks, 4 GiB. Still +all-or-nothing with temp cleanup; downloads are sequential inside the +one spawned capture task. Download error messages carry the track's +library path, never the stream URL (`reqwest::Error::without_url`). + +Nodes opt in via new additive proto flags +(`LibraryNode.is_downloadable = 8`, `LibraryNodeChild = 7`). Tidal sets +them centrally at the end of `get_lib_node`: downloadable = queueable +**or lists tracks** (so search-term track results are downloadable even +though the term node isn't queueable); children mirror `is_queable`; +tracks inherit their node's flag in the TUI. The server re-enforces at +the capture root (`Unsupported` → `failed_precondition`); the rpc +gained `CaptureLibraryNodeRequest.download = 3` (additive; old clients +keep bookmarking). + +Deviations from the plan/architecture: the naming helper ended up +`audio_file_name` (the path variant was clippy-dead); the tidal flag +rule grew the "or lists tracks" clause (architecture D4 reconciled); +the live probe downloaded a single real track (8.6 MB m4a, +Content-Type-derived extension, replayed through a `/captures` +instance) instead of a whole album — the multi-track walk is +unit-covered and a full album download is needlessly heavy for a smoke +test. 142 workspace tests green (1 new in `fsdy`, 8 in +`capture`/`capture_store`, 3 TUI + 1 extended); every gate in +`quality/captures.md` checked. + ## bookmarks (2026-07-21) Built per `plan/bookmarks.md`: `w` on a queueable library selection now diff --git a/quality/captures.md b/quality/captures.md new file mode 100644 index 0000000..9b69bad --- /dev/null +++ b/quality/captures.md @@ -0,0 +1,84 @@ +# Quality gates: captures + +Criteria the implementation must satisfy beyond the automatic tests +(`crabidy-server/src/capture.rs`, `crabidy-server/src/capture_store.rs`, +`fsdy/src/lib.rs`, plus the TUI tests added during implementation). Each +gate is pass/fail by reading the code. + +## Shared walk (refactor) + +- [x] Bookmarks and download captures run through **one** walk + (`capture::capture_into`/`write_tree`); `bookmark_store` keeps no copy + of the worklist, caps, tmp-and-swap, or cleanup logic. +- [x] The refactor is behavior-preserving for bookmarks: every existing + `bookmark_store` test passes unchanged (module path of `CaptureError` + aside). +- [x] The walk stays iterative (worklist), all-or-nothing, and removes the + temp folder on every failure path — including failed downloads and the + byte budget. + +## Download sink + +- [x] Every external call is bounded: connect timeout on the shared HTTP + client and one per-track deadline covering URL fetch, request, and the + whole body stream. No retries. +- [x] Bodies are **streamed** to disk (never buffered whole) and counted + against the capture's byte budget while streaming; exceeding it is + `TooLarge`, not partial data left behind. +- [x] Non-2xx responses and transport errors are typed + (`CaptureError::Download`) — no panic on any network condition. +- [x] Download log lines carry paths, names, and counts — never stream + URLs (they embed tokens) and never file contents. +- [x] The toml is written only after its audio file succeeded, with a + relative `file` playable naming the sibling + (`TrackFile::from_track_with_file`); the audio name shares the toml's + order prefix and sanitizer (`capture::audio_file_name`). +- [x] Downloads run sequentially inside the one spawned capture task; the + orchestrator loop keeps serving commands during a capture. + +## Opt-in (is_downloadable) + +- [x] Tidal sets the flag centrally: nodes are downloadable when queueable + or when they list tracks; children mirror `is_queable`. No per-arm + copies to drift. +- [x] Every other provider (fs, queues, bookmarks, captures, orchestrator + root) leaves the flag `false` — a capture can never be built from + another capture's or bookmark's links masquerading as sources. +- [x] The server enforces the blessing at the capture root (directory + source: its own node; track source: its parent node) with + `CaptureError::Unsupported`; an unreadable root is `BadSource`. + +## RPC and orchestrator + +- [x] `CaptureLibraryNodeRequest.download` is additive: old clients (no + flag) keep getting bookmarks, byte-for-byte. +- [x] Error mapping: `InvalidName`/`BadSource` → `invalid_argument`; + `TooLarge`/`Disabled`/`Unsupported` → `failed_precondition`; + download/walk/write failures → `internal`. +- [x] The orchestrator routes `/captures` in every `ProviderClient` method + (same completeness as `/bookmarks`), and `get_lib_root` lists the + `captures` child only when the instance exists. +- [x] Captures init is non-fatal: no config dir or an unopenable store + disables download captures and the `/captures` mount, never the server + (and never bookmarks). +- [x] `/captures` mounts with an editable top level (no reserved names): + captures are renamable (`e`) and deletable (`d`) like bookmarks. + +## TUI + +- [x] `W` (shift) is bound in `Scope::Library` with a help description and + passes the bindings-table invariant tests unchanged; `w` behavior is + untouched. +- [x] The capture overlay opens for `W` only when the bare selection is + queueable **and** downloadable; tracks inherit their node's flag; + marks are ignored. Label: `capture` (vs `bookmark`). +- [x] `MessageFromUi::CaptureNode` carries `download`; a failed capture is + logged and never tears down the poll loop. + +## Hygiene + +- [x] New public items are documented; docs state error/edge behavior. +- [x] `clippy -D warnings`, `fmt`, `taplo`, `markdownlint` clean on the + whole workspace; all tests green. +- [x] `architecture/captures.md` reconciled where the implementation + diverged (e.g. the exact tidal flag rule). diff --git a/tidaldy/src/lib.rs b/tidaldy/src/lib.rs index c3e6211..d4b4dca 100644 --- a/tidaldy/src/lib.rs +++ b/tidaldy/src/lib.rs @@ -128,6 +128,7 @@ impl crabidy_core::ProviderClient for Client { ], is_queable: false, is_creatable: false, + is_downloadable: false, } } @@ -155,6 +156,7 @@ impl crabidy_core::ProviderClient for Client { children: Vec::new(), is_queable: false, is_creatable: false, + is_downloadable: false, }; let user_id = user_id.ok_or(crabidy_core::ProviderError::UnknownUser)?; for playlist in self @@ -186,6 +188,7 @@ impl crabidy_core::ProviderClient for Client { children: Vec::new(), is_queable: true, is_creatable: false, + is_downloadable: false, } } TidalPath::Artists => { @@ -197,6 +200,7 @@ impl crabidy_core::ProviderClient for Client { children: Vec::new(), is_queable: false, is_creatable: false, + is_downloadable: false, }; let user_id = user_id.ok_or(crabidy_core::ProviderError::UnknownUser)?; for artist in self.get_users_artists(&user_id).await? { @@ -231,6 +235,7 @@ impl crabidy_core::ProviderClient for Client { children, is_queable: true, is_creatable: false, + is_downloadable: false, } } TidalPath::Album { album, .. } => { @@ -249,6 +254,7 @@ impl crabidy_core::ProviderClient for Client { children: Vec::new(), is_queable: true, is_creatable: false, + is_downloadable: false, } } TidalPath::Search => crabidy_core::proto::crabidy::LibraryNode { @@ -275,6 +281,7 @@ impl crabidy_core::ProviderClient for Client { .collect(), is_queable: false, is_creatable: true, + is_downloadable: false, }, TidalPath::SearchTerm(encoded) => { let term = crabidy_core::decode_segment(encoded); @@ -290,6 +297,16 @@ impl crabidy_core::ProviderClient for Client { return Err(crabidy_core::ProviderError::MalformedPath); } }; + // Tidal implements download captures (architecture/captures.md D4): + // every queueable subtree may be captured with `W`, and every node + // that lists tracks blesses them for download (listed tracks + // inherit their node's flag — this covers search-term results, + // whose node is not queueable as a whole). + let mut node = node; + node.is_downloadable = node.is_queable || !node.tracks.is_empty(); + for child in &mut node.children { + child.is_downloadable = child.is_queable; + } Ok(node) } @@ -897,6 +914,7 @@ impl Client { children, is_queable: false, is_creatable: false, + is_downloadable: false, }) }