From 956a4906067ac9a33b227528f6fbbf3f49220274 Mon Sep 17 00:00:00 2001 From: Test User Date: Sun, 26 Jul 2026 14:16:25 +0200 Subject: [PATCH] rss: don't list image enclosures as episodes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reported from a real subscription: an episode failed to play with "the format of the data has not been recognized" on a cdn.netzpolitik.org/….jpg URL, 272428 bytes — exactly the length the feed declared for a JPEG. audio_url preferred an audio-typed enclosure but then fell back to *any* media url, and a WordPress blog feed attaches each post's featured image as an , structurally identical to a podcast enclosure apart from type="image/jpeg". https://netzpolitik.org/feed/ is 25 items, 25 JPEG enclosures, and no audio reference of any kind, so every post became an episode that could not play. An enclosure is now accepted when its type says audio, when the feed omits the type (plenty of hand-rolled feeds do), or when a generic application/octet-stream is backed by an audio file extension — and rejected otherwise, so images and video are skipped. Audio-typed still wins, so a show that publishes both plays as audio. A feed with entries and no audio now says so ("this looks like a blog feed rather than a podcast feed"): an empty listing explains nothing on its own. No URL in the message — a feed URL is a credential. Also raises DEFAULT_MAX_FEED_BYTES from 8 to 32 MiB. Logbuch:Netzpolitik, 559 episodes in, is a healthy 6.8 MiB: feeds carry their whole back catalogue with full show notes, so the first cap would have started refusing real feeds within a year or two. Still bounded, still enforced while reading rather than after, still lowerable via max_feed_bytes. Verified: 30 rssdy tests (5 new, over feed-rs's real entry shapes: image-only, mixed image+audio, untyped, generic-with-extension, generic-without, video), and both live feeds — netzpolitik.org/feed/ now yields 0 episodes with the warning, logbuch-netzpolitik.de/feed/mp3 (-> feeds.metaebene.me/lnp/mp3) yields 559 audio/mpeg episodes whose HH:MM:SS durations parse. Co-Authored-By: Claude Opus 5 (1M context) --- docs/src/providers/rss.md | 27 ++++- plan/summary.md | 86 +++++++++++++++ rssdy/README.md | 2 +- rssdy/src/api.rs | 215 +++++++++++++++++++++++++++++++++----- rssdy/src/lib.rs | 12 ++- 5 files changed, 313 insertions(+), 29 deletions(-) diff --git a/docs/src/providers/rss.md b/docs/src/providers/rss.md index b20c8c5..d19458d 100644 --- a/docs/src/providers/rss.md +++ b/docs/src/providers/rss.md @@ -102,7 +102,7 @@ url = "https://feeds.example.org/cautionary-tales" # Optional, defaults shown. episodes_per_feed = 200 # episodes listed per feed call_timeout_secs = 30 # per-request timeout -max_feed_bytes = 8388608 # 8 MiB cap on a feed body +max_feed_bytes = 33554432 # 32 MiB cap on a feed body ``` `%` appends to this file, so subscriptions made from a client persist. An @@ -111,3 +111,28 @@ entry with no url is skipped with a warning, and no feeds at all is fine — Feeds are read as RSS 2.0/1.0/0.x, Atom, or JSON Feed, and a malformed episode is skipped rather than failing the listing. + +## Subscribe to the podcast feed, not the site feed + +A blog and its podcast usually have **different** feeds, and subscribing to +the wrong one gives you a subscription that lists nothing. That is by +design: an entry only becomes an episode if it carries a playable audio +enclosure, and a blog feed's enclosures are the posts' featured *images*. +WordPress feeds in particular look exactly like podcast feeds apart from +that one attribute — so `https://netzpolitik.org/feed/` yields no +episodes, while `https://logbuch-netzpolitik.de/feed/mp3` yields the +podcast. + +When a feed has entries but none of them carry audio, the server says so: + +```text +WARN feed has entries but none carry playable audio; this looks like a blog + feed rather than a podcast feed entries=25 +``` + +Look for a "Podcast" or "Subscribe" link on the show's page, or a +`/feed/mp3`-style URL. An enclosure is accepted when its type says audio, +when the feed omits the type (many hand-rolled feeds do), or when a generic +`application/octet-stream` is backed up by an audio file extension. Video +enclosures are left alone; a show that publishes both video and audio plays +as audio. diff --git a/plan/summary.md b/plan/summary.md index 09d72c0..dc2c65c 100644 --- a/plan/summary.md +++ b/plan/summary.md @@ -1393,3 +1393,89 @@ whole `check-features` matrix (now including `rss` alone) clippy-clean under `-D warnings`, fmt clean, book builds, and a live fetch of the user's own premium Economist feed. Not exercised: playing an episode through an audio device. + +## seek within a track (2026-07-26) + +`architecture/seek.md` → `quality/seek.md` → `plan/seek.md`. `Ctrl-b` / +`Ctrl-f` move the playing position 15 seconds, from the TUI, the browser, and +`cbd global seek `. + +Almost all of it was wiring: `PlayerEngine::seek_to` and its command already +existed and **nothing called them** — no RPC, no playback command, no binding. + +*The one real decision was where the arithmetic lives.* A seek is relative +("15 seconds back") but the engine seeks to an absolute position, so either the +client computes the target from the last `TrackPosition` it received, or it +sends a delta and the engine adds it to the live position. The delta wins on +the ordinary case of pressing the key twice: positions are broadcast on a +250 ms tick and then cross the network, so three quick presses all read the +*same* stale base and jump 15 s instead of 45. It also keeps the clamping +policy in one place instead of three clients, and matters more while paused, +where no position updates arrive at all. So the wire carries +`sint32 delta_millis` and the step (15 s) is a client constant. + +*It also uncovered a live panic.* `seek_to` did +`time.clamp(Duration::from_secs(1), duration)`. `Ord::clamp` asserts +`min <= max`, and `duration()` returns **0** whenever the source reported no +length (HLS, some streams) — so that call panicked the engine thread, killing +audio, on any duration-less track. Unreachable only because nothing called it; +wiring seek made it reachable from user input, which the hard rules forbid. It +is now saturating arithmetic in a pure, exhaustively-tested function. + +Boundaries: backwards saturates at 0 and never enters the previous track; +forwards stops 1 s short of the end so the track finishes through the ordinary +end-of-stream path (which advances the queue) rather than relying on +seek-to-exact-end, which every decoder treats differently; an unknown duration +has no upper clamp and the decoder decides. The engine emits `Elapsed` from the +seek path itself, because `tick()` skips a paused sink and a paused seek would +otherwise show the old position until playback resumed. An unseekable source +(SoundCloud's HLS) warns server-side and changes nothing — clients never move +the position themselves, so there is nothing to correct. + +Bindings landed on `Ctrl-b`/`Ctrl-f` at the user's request (a first round used +`,`/`.`). They join the existing control-chord family (`Ctrl-n`/`Ctrl-p`, +`Ctrl-d`/`Ctrl-u`); plain `f` still toggles the spectrum because the TUI's +`lookup` compares every modifier but `SHIFT` exactly. In the browser `Ctrl-f` +would open the find bar, but the keydown handler already `prevent_default`s any +chord that resolves — binding it is enough to claim it. + +Deferred: absolute seek plus click-to-seek on the web progress gauge (a +compatible proto3 field addition; the gauge is already rendered), a +configurable step, and chapter-aware seek. + +Verified: 20 audio-player tests (5 new, covering `i64::MIN`/`MAX`, zero +duration, sub-second tracks, composition, near-end saturation), 120 cbd-tui, +21 cbd-web, crabidy-server, fmt clean, the wasm bundle builds. Not exercised: +an actual seek through an audio device. + +### Fixed alongside: image enclosures listed as episodes (`/rss`) + +Reported from a real subscription: an episode failed with "the format of the +data has not been recognized" on a `cdn.netzpolitik.org/…jpg` URL. + +`audio_url` preferred an audio-typed enclosure but then fell back to *any* +media URL — and a WordPress blog feed attaches each post's featured **image** +as an ``, structurally identical to a podcast enclosure apart from +`type="image/jpeg"`. `https://netzpolitik.org/feed/` is 25 items, 25 JPEG +enclosures, and no audio reference of any kind, so every post became an +episode that could not play. + +An enclosure is now accepted when its type says audio, when the feed omits the +type (many hand-rolled feeds do), or when a generic `application/octet-stream` +is backed by an audio file extension — and rejected otherwise, so images and +video are skipped. Audio-typed still wins, so a show publishing both plays as +audio. When a feed has entries but none carry audio, the server says so +("this looks like a blog feed rather than a podcast feed"): an empty listing +explains nothing on its own, and no URL goes in the message. + +Also raised `DEFAULT_MAX_FEED_BYTES` 8 MiB → 32 MiB. Logbuch:Netzpolitik, 559 +episodes in, is a healthy 6.8 MiB — feeds carry their whole back catalogue with +full show notes, so the first cap would have started refusing real feeds within +a year or two. Still bounded, still enforced while reading, still lowerable via +`max_feed_bytes`. + +Verified: 30 rssdy tests (5 new, over feed-rs's real entry shapes: image-only, +mixed image+audio, untyped, generic-with-extension, generic-without and video), +and the two live feeds — netzpolitik.org/feed/ now yields 0 episodes with the +warning, logbuch-netzpolitik.de/feed/mp3 (→ feeds.metaebene.me/lnp/mp3) yields +559 `audio/mpeg` episodes with `HH:MM:SS` durations. diff --git a/rssdy/README.md b/rssdy/README.md index eba8367..70ff1e9 100644 --- a/rssdy/README.md +++ b/rssdy/README.md @@ -74,7 +74,7 @@ url = "https://feeds.example.org/cautionary-tales" # Optional, defaults shown. # episodes_per_feed = 200 # episodes listed per feed # call_timeout_secs = 30 # per-request timeout -# max_feed_bytes = 8388608 # 8 MiB cap on a feed body +# max_feed_bytes = 33554432 # 32 MiB cap on a feed body ``` A feed entry with no url is skipped with a warning. No feeds at all is fine: diff --git a/rssdy/src/api.rs b/rssdy/src/api.rs index 1cf0dd6..55d13a3 100644 --- a/rssdy/src/api.rs +++ b/rssdy/src/api.rs @@ -17,6 +17,7 @@ use std::time::Duration; use async_trait::async_trait; use thiserror::Error; +use tracing::warn; /// A typed feed failure. Carries only non-secret context — never a feed URL, /// which would put a subscriber token in the logs. @@ -174,6 +175,7 @@ fn episodes_from( limit: usize, durations: &HashMap, ) -> Vec { + let entry_count = entries.len(); let mut episodes: Vec = entries .into_iter() .filter_map(|entry| { @@ -218,36 +220,106 @@ fn episodes_from( // sorting newest first. episodes.sort_by_key(|e| std::cmp::Reverse(e.published)); episodes.truncate(limit); + // A feed full of entries and empty of audio is almost always a blog feed + // subscribed by mistake (its enclosures are the posts' featured images). + // Say so: the listing itself can only show nothing, which explains nothing. + // No URL in the message — a feed URL is a credential. + if episodes.is_empty() && entry_count > 0 { + warn!( + entries = entry_count, + "feed has entries but none carry playable audio; this looks like a blog feed rather \ + than a podcast feed" + ); + } episodes } -/// The first playable audio URL of an entry: a media enclosure, else a link -/// that advertises audio (feed dialects disagree about where it goes). +/// Media types that some feeds use for audio files without saying so. +const GENERIC_TYPES: [&str; 2] = ["application/octet-stream", "binary/octet-stream"]; + +/// File extensions we treat as audio when the declared type is generic or +/// missing. Deliberately excludes `.mp4`, which is as often video as audio. +const AUDIO_EXTENSIONS: [&str; 13] = [ + "mp3", "m4a", "m4b", "aac", "ogg", "oga", "opus", "flac", "wav", "aiff", "aif", "mp2", "mpga", +]; + +/// Whether a declared media type names audio. +fn is_audio_type(content_type: &str) -> bool { + let normalized = content_type.trim().to_ascii_lowercase(); + normalized.starts_with("audio/") || normalized == "audio" +} + +/// Whether an enclosure is plausibly something we can decode. +/// +/// The permissive cases exist because real podcast feeds are sloppy: plenty +/// omit the `type` attribute entirely, and a few serve mp3s as +/// `application/octet-stream`. So a missing type is accepted, and a generic one +/// is accepted when the URL's extension backs it up. +/// +/// What is *not* accepted is a type that names something else. Blog feeds — +/// WordPress ones especially — attach the post's featured **image** as an +/// ``, which is structurally identical to a podcast enclosure and +/// distinguishable only by its type. Accepting those turned every blog post +/// into an episode that failed to decode at play time. +fn plausibly_audio(content_type: Option<&str>, url: &str) -> bool { + match content_type { + None => true, + Some(ct) if is_audio_type(ct) => true, + Some(ct) => { + let normalized = ct.trim().to_ascii_lowercase(); + GENERIC_TYPES.contains(&normalized.as_str()) && has_audio_extension(url) + } + } +} + +/// Whether the URL's path ends in a known audio extension. Query strings and +/// fragments are ignored — enclosure URLs routinely carry tracking parameters. +fn has_audio_extension(url: &str) -> bool { + let path = url + .split(['?', '#']) + .next() + .unwrap_or(url) + .trim_end_matches('/'); + let Some((_, extension)) = path.rsplit_once('.') else { + return false; + }; + let extension = extension.to_ascii_lowercase(); + AUDIO_EXTENSIONS.contains(&extension.as_str()) +} + +/// The playable audio URL of an entry: a media enclosure, else a link that +/// advertises audio (feed dialects disagree about where it goes). +/// +/// An entry whose only enclosures are images or video yields `None` and is +/// skipped — better an episode that never appears than one that appears and +/// cannot be played. fn audio_url(entry: &feed_rs::model::Entry) -> Option { - let media = entry.media.iter().flat_map(|m| m.content.iter()); - // Prefer something explicitly typed as audio, then any media url at all. - let typed = media.clone().find(|c| { - c.url.is_some() - && c.content_type - .as_ref() - .is_some_and(|ct| ct.to_string().starts_with("audio")) - }); - if let Some(url) = typed.and_then(|c| c.url.as_ref()) { - return Some(url.to_string()); - } - if let Some(url) = media.filter_map(|c| c.url.as_ref()).next() { - return Some(url.to_string()); - } - entry - .links + let enclosures: Vec<(Option, String)> = entry + .media .iter() - .find(|l| { - l.rel.as_deref() == Some("enclosure") - || l.media_type - .as_deref() - .is_some_and(|ct| ct.starts_with("audio")) + .flat_map(|m| m.content.iter()) + .filter_map(|content| { + let url = content.url.as_ref()?.to_string(); + Some((content.content_type.as_ref().map(ToString::to_string), url)) }) - .map(|l| l.href.clone()) + .chain(entry.links.iter().filter_map(|link| { + let candidate = link.rel.as_deref() == Some("enclosure") + || link.media_type.as_deref().is_some_and(is_audio_type); + candidate.then(|| (link.media_type.clone(), link.href.clone())) + })) + .collect(); + // An explicitly audio-typed enclosure wins; a video podcast that also + // ships an audio version therefore plays as audio. Only then do the + // permissive cases (no type, generic type) get a turn. + enclosures + .iter() + .find(|(content_type, _)| content_type.as_deref().is_some_and(is_audio_type)) + .or_else(|| { + enclosures + .iter() + .find(|(content_type, url)| plausibly_audio(content_type.as_deref(), url)) + }) + .map(|(_, url)| url.clone()) } /// `itunes:duration` (or a media duration) in whole seconds. @@ -380,6 +452,101 @@ mod tests { assert!(!found.contains_key("no-duration")); } + /// Parses an RSS body the way [`FeedFetcher::fetch`] does and returns the + /// episodes, so these tests exercise the real feed-rs entry shapes rather + /// than hand-built model structs. + fn episodes_of(body: &str) -> Vec { + let feed = feed_rs::parser::parse(body.as_bytes()).expect("parses"); + episodes_from(feed.entries, 100, &HashMap::new()) + } + + /// The netzpolitik.org case: a WordPress blog feed whose `` is + /// the post's featured **image**. Accepting it produced episodes that + /// failed to decode at play time ("format of the data has not been + /// recognized"), which is worse than not listing them at all. + #[test] + fn image_enclosures_are_not_episodes() { + let episodes = episodes_of( + r#" + A blog + + A post + post-1 + + + "#, + ); + assert!( + episodes.is_empty(), + "an image enclosure is not a playable episode" + ); + } + + /// A real podcast entry still resolves, and an entry that ships both an + /// image and audio picks the audio. + #[test] + fn audio_enclosures_win_over_other_media() { + let episodes = episodes_of( + r#" + A podcast + + Episode + ep-1 + + + + "#, + ); + assert_eq!(episodes.len(), 1); + assert_eq!(episodes[0].enclosure_url, "https://cdn.example/ep1.mp3"); + } + + /// Sloppy-but-real feeds: no `type` at all, or a generic one backed by the + /// extension. Both stay playable — the decoder is the final judge. + #[test] + fn untyped_and_generic_enclosures_are_accepted() { + let episodes = episodes_of( + r#" + A podcast + a + + b + + "#, + ); + assert_eq!(episodes.len(), 2); + } + + /// A generic type with no audible extension is not taken on faith, and + /// video enclosures are left alone. + #[test] + fn generic_and_video_enclosures_without_audio_are_skipped() { + let episodes = episodes_of( + r#" + Mixed + a + + b + + "#, + ); + assert!(episodes.is_empty()); + } + + #[test] + fn audio_extensions_ignore_query_strings() { + assert!(has_audio_extension("https://cdn.example/a.mp3")); + assert!(has_audio_extension("https://cdn.example/a.MP3?tk=X#f")); + assert!(has_audio_extension("https://cdn.example/a.opus?x=.jpg")); + assert!(!has_audio_extension("https://cdn.example/a.jpg")); + assert!(!has_audio_extension("https://cdn.example/a.mp4")); + assert!(!has_audio_extension("https://cdn.example/no-extension")); + assert!(!has_audio_extension("")); + } + #[test] fn tag_text_handles_cdata_and_attributes() { assert_eq!( diff --git a/rssdy/src/lib.rs b/rssdy/src/lib.rs index 2a89c3d..94a9df7 100644 --- a/rssdy/src/lib.rs +++ b/rssdy/src/lib.rs @@ -37,9 +37,15 @@ pub const PROVIDER_ROOT: &str = "/rss"; pub const DEFAULT_EPISODES_PER_FEED: usize = 200; /// Default per-request timeout in seconds. pub const DEFAULT_CALL_TIMEOUT_SECS: u64 = 30; -/// Default cap on a feed body, in bytes (8 MiB). A feed larger than this is a -/// publishing bug, not something to load into memory (D6). -pub const DEFAULT_MAX_FEED_BYTES: u64 = 8 * 1024 * 1024; +/// Default cap on a feed body, in bytes (32 MiB) — the bound that keeps a +/// runaway response from being read into memory (D6). +/// +/// Generous on purpose: podcast feeds routinely carry their *whole* back +/// catalogue with full show notes in every item. Logbuch:Netzpolitik, 559 +/// episodes in, is 6.8 MiB — so the first cap of 8 MiB would have started +/// refusing real, healthy feeds within a year or two of publishing. Lower it in +/// `rss.toml` (`max_feed_bytes`) if a tighter bound is wanted. +pub const DEFAULT_MAX_FEED_BYTES: u64 = 32 * 1024 * 1024; /// How many feeds' episode lists the memo keeps (D3). pub const MEMO_CAPACITY: usize = 8;