//! The feed HTTP seam. //! //! All network access goes through the [`Feeds`] trait so the provider's //! subscription, slug, identity, and ordering logic is unit-tested with a fake //! and no network (architecture/rss-provider.md). [`FeedFetcher`] is the //! production `reqwest` + `feed-rs` implementation; tests supply their own //! [`Feeds`]. //! //! **Every feed URL here is a credential.** A premium podcast URL embeds a //! per-subscriber token, so URLs are redacted from `Debug`, never logged, and //! never carried in an error message. Enclosure URLs get the same treatment — //! they can be signed too. use std::collections::HashMap; use std::fmt::{self, Debug}; use std::time::Duration; use async_trait::async_trait; use thiserror::Error; use tracing::warn; /// A typed feed failure. Carries only non-secret context — never a feed URL, /// which would put a subscriber token in the logs. #[derive(Debug, Error)] pub enum FetchError { /// Transport, timeout, or a non-success status. #[error("feed request failed: {0}")] Http(String), /// The body was not a feed we could parse. #[error("feed could not be parsed: {0}")] Parse(String), /// The body exceeded `max_feed_bytes` (D6). #[error("feed is larger than the {limit} byte limit")] TooLarge { limit: u64 }, } /// One episode, normalized out of whatever feed dialect produced it. #[derive(Clone, Debug, PartialEq, Eq)] pub struct Episode { /// The publisher's ``, or the enclosure URL when the feed omits one. /// Keys [`Self::key`] and becomes `Track.provider_item_id` so the content /// store de-duplicates captures of the same episode. pub guid: String, /// Stable short path segment: `blake3(guid)[..16]` (D4). pub key: String, pub title: String, /// Show/author, used as the track artist. Empty when the feed omits it. pub author: String, /// Publication instant as a unix timestamp, when the feed has a parseable /// date. `None` keeps the entry in feed order (D9). pub published: Option, pub duration_secs: Option, /// The audio to play. **Secret-ish**: never logged. pub enclosure_url: String, } impl Episode { /// The path segment for `guid` — `blake3` truncated to 16 hex chars, which /// is short, readable, and stable across builds and restarts (D4). pub fn key_for(guid: &str) -> String { blake3::hash(guid.as_bytes()).to_hex()[..KEY_HEX_LEN].to_string() } } /// Hex characters of the guid hash kept in a path segment. 64 bits is far more /// than enough to keep one feed's episodes apart, and stays readable. const KEY_HEX_LEN: usize = 16; /// A parsed feed: the publisher's own title (used to name a new subscription, /// D5) plus its episodes. #[derive(Clone, Debug, PartialEq, Eq)] pub struct Feed { pub title: String, pub episodes: Vec, } /// Everything the provider does over the network. #[async_trait] pub trait Feeds: Debug + Send + Sync { /// Fetches and parses `url`, returning at most `limit` episodes, newest /// first. Bounded by a per-request timeout and a byte cap; a malformed /// entry is skipped rather than failing the feed. async fn fetch(&self, url: &str, limit: usize) -> Result; } /// The production implementation: `reqwest` for transport, `feed-rs` for /// parsing. pub struct FeedFetcher { http: reqwest::Client, max_bytes: u64, } impl Debug for FeedFetcher { /// No URLs are held here, but the impl is explicit so it stays that way. fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("FeedFetcher") .field("max_bytes", &self.max_bytes) .finish_non_exhaustive() } } impl FeedFetcher { /// `timeout` bounds every request; `max_bytes` caps a response body so a /// broken or hostile feed cannot exhaust memory (D6). pub fn new(timeout: Duration, max_bytes: u64) -> Result { let http = reqwest::Client::builder() .timeout(timeout) .user_agent(USER_AGENT) .build() // `without_url` so a builder error cannot echo a URL. .map_err(|err| FetchError::Http(err.without_url().to_string()))?; Ok(Self { http, max_bytes }) } /// Reads the body in chunks, refusing to buffer more than `max_bytes` — /// the cap has to bite *before* the allocation, not after (D6). async fn bounded_body(&self, mut response: reqwest::Response) -> Result, FetchError> { let mut body: Vec = Vec::new(); while let Some(chunk) = response .chunk() .await .map_err(|err| FetchError::Http(err.without_url().to_string()))? { if body.len() as u64 + chunk.len() as u64 > self.max_bytes { return Err(FetchError::TooLarge { limit: self.max_bytes, }); } body.extend_from_slice(&chunk); } Ok(body) } } /// Sent on every feed request. Some publishers reject an empty agent. const USER_AGENT: &str = concat!("crabidy/", env!("CARGO_PKG_VERSION")); #[async_trait] impl Feeds for FeedFetcher { async fn fetch(&self, url: &str, limit: usize) -> Result { // `without_url` on every error: a premium feed URL is a credential and // reqwest puts the URL in its Display output by default (G1). let response = self .http .get(url) .send() .await .map_err(|err| FetchError::Http(err.without_url().to_string()))?; let status = response.status(); if !status.is_success() { return Err(FetchError::Http(status.to_string())); } let body = self.bounded_body(response).await?; // Recovered before parsing, because feed-rs loses this field (see // `itunes_durations`). let durations = itunes_durations(&body); let parsed = feed_rs::parser::parse(body.as_slice()) .map_err(|err| FetchError::Parse(err.to_string()))?; Ok(Feed { title: parsed .title .map(|t| t.content) .unwrap_or_default() .trim() .to_string(), episodes: episodes_from(parsed.entries, limit, &durations), }) } } /// Normalizes feed entries into episodes: newest first, capped, and skipping /// anything unplayable. A single bad entry never fails the feed (G15). fn episodes_from( entries: Vec, limit: usize, durations: &HashMap, ) -> Vec { let entry_count = entries.len(); let mut episodes: Vec = entries .into_iter() .filter_map(|entry| { let enclosure_url = audio_url(&entry)?; // RSS `` normalizes into `id`; a feed without one keys on // the enclosure URL instead (D4). The URL stays out of the path — // only its hash is used. let guid = if entry.id.trim().is_empty() { enclosure_url.clone() } else { entry.id.clone() }; let title = entry .title .as_ref() .map(|t| t.content.trim().to_string()) .filter(|t| !t.is_empty()) .unwrap_or_else(|| "(untitled)".to_string()); let duration_secs = durations .get(&guid) .copied() .or_else(|| duration_of(&entry)); Some(Episode { key: Episode::key_for(&guid), guid, title, author: entry .authors .first() .map(|p| p.name.trim().to_string()) .unwrap_or_default(), published: entry.published.or(entry.updated).map(|d| d.timestamp()), // Our own `itunes:duration` wins over feed-rs's NPT parse. duration_secs, enclosure_url, }) }) .collect(); // Newest first where dates parse; a stable sort keeps dateless feeds in // publication order (D9). // Descending: `Reverse` keeps clippy's sort_by_key form while // sorting newest first. episodes.sort_by_key(|e| std::cmp::Reverse(e.published)); episodes.truncate(limit); // A feed full of entries and empty of audio is almost always a blog feed // subscribed by mistake (its enclosures are the posts' featured images). // Say so: the listing itself can only show nothing, which explains nothing. // No URL in the message — a feed URL is a credential. if episodes.is_empty() && entry_count > 0 { warn!( entries = entry_count, "feed has entries but none carry playable audio; this looks like a blog feed rather \ than a podcast feed" ); } episodes } /// Media types that some feeds use for audio files without saying so. const GENERIC_TYPES: [&str; 2] = ["application/octet-stream", "binary/octet-stream"]; /// File extensions we treat as audio when the declared type is generic or /// missing. Deliberately excludes `.mp4`, which is as often video as audio. const AUDIO_EXTENSIONS: [&str; 13] = [ "mp3", "m4a", "m4b", "aac", "ogg", "oga", "opus", "flac", "wav", "aiff", "aif", "mp2", "mpga", ]; /// Whether a declared media type names audio. fn is_audio_type(content_type: &str) -> bool { let normalized = content_type.trim().to_ascii_lowercase(); normalized.starts_with("audio/") || normalized == "audio" } /// Whether an enclosure is plausibly something we can decode. /// /// The permissive cases exist because real podcast feeds are sloppy: plenty /// omit the `type` attribute entirely, and a few serve mp3s as /// `application/octet-stream`. So a missing type is accepted, and a generic one /// is accepted when the URL's extension backs it up. /// /// What is *not* accepted is a type that names something else. Blog feeds — /// WordPress ones especially — attach the post's featured **image** as an /// ``, which is structurally identical to a podcast enclosure and /// distinguishable only by its type. Accepting those turned every blog post /// into an episode that failed to decode at play time. fn plausibly_audio(content_type: Option<&str>, url: &str) -> bool { match content_type { None => true, Some(ct) if is_audio_type(ct) => true, Some(ct) => { let normalized = ct.trim().to_ascii_lowercase(); GENERIC_TYPES.contains(&normalized.as_str()) && has_audio_extension(url) } } } /// Whether the URL's path ends in a known audio extension. Query strings and /// fragments are ignored — enclosure URLs routinely carry tracking parameters. fn has_audio_extension(url: &str) -> bool { let path = url .split(['?', '#']) .next() .unwrap_or(url) .trim_end_matches('/'); let Some((_, extension)) = path.rsplit_once('.') else { return false; }; let extension = extension.to_ascii_lowercase(); AUDIO_EXTENSIONS.contains(&extension.as_str()) } /// The playable audio URL of an entry: a media enclosure, else a link that /// advertises audio (feed dialects disagree about where it goes). /// /// An entry whose only enclosures are images or video yields `None` and is /// skipped — better an episode that never appears than one that appears and /// cannot be played. fn audio_url(entry: &feed_rs::model::Entry) -> Option { let enclosures: Vec<(Option, String)> = entry .media .iter() .flat_map(|m| m.content.iter()) .filter_map(|content| { let url = content.url.as_ref()?.to_string(); Some((content.content_type.as_ref().map(ToString::to_string), url)) }) .chain(entry.links.iter().filter_map(|link| { let candidate = link.rel.as_deref() == Some("enclosure") || link.media_type.as_deref().is_some_and(is_audio_type); candidate.then(|| (link.media_type.clone(), link.href.clone())) })) .collect(); // An explicitly audio-typed enclosure wins; a video podcast that also // ships an audio version therefore plays as audio. Only then do the // permissive cases (no type, generic type) get a turn. enclosures .iter() .find(|(content_type, _)| content_type.as_deref().is_some_and(is_audio_type)) .or_else(|| { enclosures .iter() .find(|(content_type, url)| plausibly_audio(content_type.as_deref(), url)) }) .map(|(_, url)| url.clone()) } /// `itunes:duration` (or a media duration) in whole seconds. fn duration_of(entry: &feed_rs::model::Entry) -> Option { entry .media .iter() .find_map(|m| { m.duration .or_else(|| m.content.iter().find_map(|c| c.duration)) }) .map(|d| d.as_secs().min(u32::MAX as u64) as u32) } /// `itunes:duration` values recovered from the raw feed, keyed by both the /// item's guid and its enclosure URL (the two things an [`Episode`] can key /// on). /// /// Why this exists: feed-rs parses `` with an **NPT** parser, /// and NPT has no `MM:SS` form. For `53:25` /// its fallback regex matches the leading number, so a 53-minute episode comes /// back as 53 *seconds*. `1:20:40` happens to parse correctly. The iTunes spec /// allows `S`, `MM:SS` and `HH:MM:SS`, so the field is recovered here and /// overrides what feed-rs produced. /// /// This is a deliberately shallow scan, not a second feed parser: it walks /// item chunks and pulls three optional strings. Anything it cannot find falls /// back to feed-rs's value. fn itunes_durations(body: &[u8]) -> HashMap { let text = String::from_utf8_lossy(body); let mut found = HashMap::new(); // `text` in `chunk`, trimmed. fn tag_text(chunk: &str, name: &str) -> Option { let open = chunk.find(&format!("<{name}"))?; let after_open = chunk[open..].find('>')? + open + 1; let close = chunk[after_open..].find(&format!(""))? + after_open; let inner = chunk[after_open..close] .trim() .trim_start_matches("") .trim(); (!inner.is_empty()).then(|| inner.to_string()) } /// The `url="…"` of the chunk's ``, XML-unescaped for `&` — the /// one entity that routinely appears in a query string. fn enclosure_url_attr(chunk: &str) -> Option { let at = chunk.find(" Option { let mut parts: Vec = Vec::new(); for field in text.split(':') { // Tolerate "53:25.5" and stray whitespace. let field = field.trim(); let field = field.split('.').next().unwrap_or(field); parts.push(field.parse::().ok()?); } let seconds = match parts.as_slice() { [s] => *s, [m, s] => m * 60 + s, [h, m, s] => h * 3600 + m * 60 + s, _ => return None, }; Some(seconds.min(u32::MAX as u64) as u32) } #[cfg(test)] mod tests { use super::*; /// The bug this module works around: `MM:SS` is not NPT, and a 53-minute /// episode must not come back as 53 seconds. #[test] fn itunes_durations_accept_every_documented_form() { assert_eq!(parse_itunes_duration("3205".to_string()), Some(3205)); assert_eq!(parse_itunes_duration("53:25".to_string()), Some(3205)); assert_eq!(parse_itunes_duration("1:20:40".to_string()), Some(4840)); assert_eq!(parse_itunes_duration("53:25.5".to_string()), Some(3205)); assert_eq!(parse_itunes_duration("".to_string()), None); assert_eq!(parse_itunes_duration("about an hour".to_string()), None); } /// Scanned straight out of a feed body shaped like the real one, keyed by /// guid *and* enclosure url so either identity resolves it. #[test] fn durations_are_recovered_from_the_raw_body() { let body = br#" 6a6479f2a51cbd54 53:25 second 1:20:40 no-duration "#; let found = itunes_durations(body); assert_eq!(found.get("6a6479f2a51cbd54"), Some(&3205)); // The enclosure url is a key too, unescaped, for feeds without a guid. assert_eq!( found.get("https://cdn.example/a.mp3?tk=X&sig=Y"), Some(&3205) ); assert_eq!(found.get("second"), Some(&4840)); assert!(!found.contains_key("no-duration")); } /// Parses an RSS body the way [`FeedFetcher::fetch`] does and returns the /// episodes, so these tests exercise the real feed-rs entry shapes rather /// than hand-built model structs. fn episodes_of(body: &str) -> Vec { let feed = feed_rs::parser::parse(body.as_bytes()).expect("parses"); episodes_from(feed.entries, 100, &HashMap::new()) } /// The netzpolitik.org case: a WordPress blog feed whose `` is /// the post's featured **image**. Accepting it produced episodes that /// failed to decode at play time ("format of the data has not been /// recognized"), which is worse than not listing them at all. #[test] fn image_enclosures_are_not_episodes() { let episodes = episodes_of( r#" A blog A post post-1 "#, ); assert!( episodes.is_empty(), "an image enclosure is not a playable episode" ); } /// A real podcast entry still resolves, and an entry that ships both an /// image and audio picks the audio. #[test] fn audio_enclosures_win_over_other_media() { let episodes = episodes_of( r#" A podcast Episode ep-1 "#, ); assert_eq!(episodes.len(), 1); assert_eq!(episodes[0].enclosure_url, "https://cdn.example/ep1.mp3"); } /// Sloppy-but-real feeds: no `type` at all, or a generic one backed by the /// extension. Both stay playable — the decoder is the final judge. #[test] fn untyped_and_generic_enclosures_are_accepted() { let episodes = episodes_of( r#" A podcast a b "#, ); assert_eq!(episodes.len(), 2); } /// A generic type with no audible extension is not taken on faith, and /// video enclosures are left alone. #[test] fn generic_and_video_enclosures_without_audio_are_skipped() { let episodes = episodes_of( r#" Mixed a b "#, ); assert!(episodes.is_empty()); } #[test] fn audio_extensions_ignore_query_strings() { assert!(has_audio_extension("https://cdn.example/a.mp3")); assert!(has_audio_extension("https://cdn.example/a.MP3?tk=X#f")); assert!(has_audio_extension("https://cdn.example/a.opus?x=.jpg")); assert!(!has_audio_extension("https://cdn.example/a.jpg")); assert!(!has_audio_extension("https://cdn.example/a.mp4")); assert!(!has_audio_extension("https://cdn.example/no-extension")); assert!(!has_audio_extension("")); } #[test] fn tag_text_handles_cdata_and_attributes() { assert_eq!( tag_text("abc", "guid").as_deref(), Some("abc") ); assert_eq!( tag_text("<![CDATA[ Hello ]]>", "title").as_deref(), Some("Hello") ); assert_eq!(tag_text("", "guid"), None); assert_eq!(tag_text("nothing here", "guid"), None); } }