//! The feed HTTP seam. //! //! All network access goes through the [`Feeds`] trait so the provider's //! subscription, slug, identity, and ordering logic is unit-tested with a fake //! and no network (architecture/rss-provider.md). [`FeedFetcher`] is the //! production `reqwest` + `feed-rs` implementation; tests supply their own //! [`Feeds`]. //! //! **Every feed URL here is a credential.** A premium podcast URL embeds a //! per-subscriber token, so URLs are redacted from `Debug`, never logged, and //! never carried in an error message. Enclosure URLs get the same treatment — //! they can be signed too. use std::collections::HashMap; use std::fmt::{self, Debug}; use std::time::Duration; use async_trait::async_trait; use thiserror::Error; /// A typed feed failure. Carries only non-secret context — never a feed URL, /// which would put a subscriber token in the logs. #[derive(Debug, Error)] pub enum FetchError { /// Transport, timeout, or a non-success status. #[error("feed request failed: {0}")] Http(String), /// The body was not a feed we could parse. #[error("feed could not be parsed: {0}")] Parse(String), /// The body exceeded `max_feed_bytes` (D6). #[error("feed is larger than the {limit} byte limit")] TooLarge { limit: u64 }, } /// One episode, normalized out of whatever feed dialect produced it. #[derive(Clone, Debug, PartialEq, Eq)] pub struct Episode { /// The publisher's ``, or the enclosure URL when the feed omits one. /// Keys [`Self::key`] and becomes `Track.provider_item_id` so the content /// store de-duplicates captures of the same episode. pub guid: String, /// Stable short path segment: `blake3(guid)[..16]` (D4). pub key: String, pub title: String, /// Show/author, used as the track artist. Empty when the feed omits it. pub author: String, /// Publication instant as a unix timestamp, when the feed has a parseable /// date. `None` keeps the entry in feed order (D9). pub published: Option, pub duration_secs: Option, /// The audio to play. **Secret-ish**: never logged. pub enclosure_url: String, } impl Episode { /// The path segment for `guid` — `blake3` truncated to 16 hex chars, which /// is short, readable, and stable across builds and restarts (D4). pub fn key_for(guid: &str) -> String { blake3::hash(guid.as_bytes()).to_hex()[..KEY_HEX_LEN].to_string() } } /// Hex characters of the guid hash kept in a path segment. 64 bits is far more /// than enough to keep one feed's episodes apart, and stays readable. const KEY_HEX_LEN: usize = 16; /// A parsed feed: the publisher's own title (used to name a new subscription, /// D5) plus its episodes. #[derive(Clone, Debug, PartialEq, Eq)] pub struct Feed { pub title: String, pub episodes: Vec, } /// Everything the provider does over the network. #[async_trait] pub trait Feeds: Debug + Send + Sync { /// Fetches and parses `url`, returning at most `limit` episodes, newest /// first. Bounded by a per-request timeout and a byte cap; a malformed /// entry is skipped rather than failing the feed. async fn fetch(&self, url: &str, limit: usize) -> Result; } /// The production implementation: `reqwest` for transport, `feed-rs` for /// parsing. pub struct FeedFetcher { http: reqwest::Client, max_bytes: u64, } impl Debug for FeedFetcher { /// No URLs are held here, but the impl is explicit so it stays that way. fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("FeedFetcher") .field("max_bytes", &self.max_bytes) .finish_non_exhaustive() } } impl FeedFetcher { /// `timeout` bounds every request; `max_bytes` caps a response body so a /// broken or hostile feed cannot exhaust memory (D6). pub fn new(timeout: Duration, max_bytes: u64) -> Result { let http = reqwest::Client::builder() .timeout(timeout) .user_agent(USER_AGENT) .build() // `without_url` so a builder error cannot echo a URL. .map_err(|err| FetchError::Http(err.without_url().to_string()))?; Ok(Self { http, max_bytes }) } /// Reads the body in chunks, refusing to buffer more than `max_bytes` — /// the cap has to bite *before* the allocation, not after (D6). async fn bounded_body(&self, mut response: reqwest::Response) -> Result, FetchError> { let mut body: Vec = Vec::new(); while let Some(chunk) = response .chunk() .await .map_err(|err| FetchError::Http(err.without_url().to_string()))? { if body.len() as u64 + chunk.len() as u64 > self.max_bytes { return Err(FetchError::TooLarge { limit: self.max_bytes, }); } body.extend_from_slice(&chunk); } Ok(body) } } /// Sent on every feed request. Some publishers reject an empty agent. const USER_AGENT: &str = concat!("crabidy/", env!("CARGO_PKG_VERSION")); #[async_trait] impl Feeds for FeedFetcher { async fn fetch(&self, url: &str, limit: usize) -> Result { // `without_url` on every error: a premium feed URL is a credential and // reqwest puts the URL in its Display output by default (G1). let response = self .http .get(url) .send() .await .map_err(|err| FetchError::Http(err.without_url().to_string()))?; let status = response.status(); if !status.is_success() { return Err(FetchError::Http(status.to_string())); } let body = self.bounded_body(response).await?; // Recovered before parsing, because feed-rs loses this field (see // `itunes_durations`). let durations = itunes_durations(&body); let parsed = feed_rs::parser::parse(body.as_slice()) .map_err(|err| FetchError::Parse(err.to_string()))?; Ok(Feed { title: parsed .title .map(|t| t.content) .unwrap_or_default() .trim() .to_string(), episodes: episodes_from(parsed.entries, limit, &durations), }) } } /// Normalizes feed entries into episodes: newest first, capped, and skipping /// anything unplayable. A single bad entry never fails the feed (G15). fn episodes_from( entries: Vec, limit: usize, durations: &HashMap, ) -> Vec { let mut episodes: Vec = entries .into_iter() .filter_map(|entry| { let enclosure_url = audio_url(&entry)?; // RSS `` normalizes into `id`; a feed without one keys on // the enclosure URL instead (D4). The URL stays out of the path — // only its hash is used. let guid = if entry.id.trim().is_empty() { enclosure_url.clone() } else { entry.id.clone() }; let title = entry .title .as_ref() .map(|t| t.content.trim().to_string()) .filter(|t| !t.is_empty()) .unwrap_or_else(|| "(untitled)".to_string()); let duration_secs = durations .get(&guid) .copied() .or_else(|| duration_of(&entry)); Some(Episode { key: Episode::key_for(&guid), guid, title, author: entry .authors .first() .map(|p| p.name.trim().to_string()) .unwrap_or_default(), published: entry.published.or(entry.updated).map(|d| d.timestamp()), // Our own `itunes:duration` wins over feed-rs's NPT parse. duration_secs, enclosure_url, }) }) .collect(); // Newest first where dates parse; a stable sort keeps dateless feeds in // publication order (D9). // Descending: `Reverse` keeps clippy's sort_by_key form while // sorting newest first. episodes.sort_by_key(|e| std::cmp::Reverse(e.published)); episodes.truncate(limit); episodes } /// The first playable audio URL of an entry: a media enclosure, else a link /// that advertises audio (feed dialects disagree about where it goes). fn audio_url(entry: &feed_rs::model::Entry) -> Option { let media = entry.media.iter().flat_map(|m| m.content.iter()); // Prefer something explicitly typed as audio, then any media url at all. let typed = media.clone().find(|c| { c.url.is_some() && c.content_type .as_ref() .is_some_and(|ct| ct.to_string().starts_with("audio")) }); if let Some(url) = typed.and_then(|c| c.url.as_ref()) { return Some(url.to_string()); } if let Some(url) = media.filter_map(|c| c.url.as_ref()).next() { return Some(url.to_string()); } entry .links .iter() .find(|l| { l.rel.as_deref() == Some("enclosure") || l.media_type .as_deref() .is_some_and(|ct| ct.starts_with("audio")) }) .map(|l| l.href.clone()) } /// `itunes:duration` (or a media duration) in whole seconds. fn duration_of(entry: &feed_rs::model::Entry) -> Option { entry .media .iter() .find_map(|m| { m.duration .or_else(|| m.content.iter().find_map(|c| c.duration)) }) .map(|d| d.as_secs().min(u32::MAX as u64) as u32) } /// `itunes:duration` values recovered from the raw feed, keyed by both the /// item's guid and its enclosure URL (the two things an [`Episode`] can key /// on). /// /// Why this exists: feed-rs parses `` with an **NPT** parser, /// and NPT has no `MM:SS` form. For `53:25` /// its fallback regex matches the leading number, so a 53-minute episode comes /// back as 53 *seconds*. `1:20:40` happens to parse correctly. The iTunes spec /// allows `S`, `MM:SS` and `HH:MM:SS`, so the field is recovered here and /// overrides what feed-rs produced. /// /// This is a deliberately shallow scan, not a second feed parser: it walks /// item chunks and pulls three optional strings. Anything it cannot find falls /// back to feed-rs's value. fn itunes_durations(body: &[u8]) -> HashMap { let text = String::from_utf8_lossy(body); let mut found = HashMap::new(); // `text` in `chunk`, trimmed. fn tag_text(chunk: &str, name: &str) -> Option { let open = chunk.find(&format!("<{name}"))?; let after_open = chunk[open..].find('>')? + open + 1; let close = chunk[after_open..].find(&format!(""))? + after_open; let inner = chunk[after_open..close] .trim() .trim_start_matches("") .trim(); (!inner.is_empty()).then(|| inner.to_string()) } /// The `url="…"` of the chunk's ``, XML-unescaped for `&` — the /// one entity that routinely appears in a query string. fn enclosure_url_attr(chunk: &str) -> Option { let at = chunk.find(" Option { let mut parts: Vec = Vec::new(); for field in text.split(':') { // Tolerate "53:25.5" and stray whitespace. let field = field.trim(); let field = field.split('.').next().unwrap_or(field); parts.push(field.parse::().ok()?); } let seconds = match parts.as_slice() { [s] => *s, [m, s] => m * 60 + s, [h, m, s] => h * 3600 + m * 60 + s, _ => return None, }; Some(seconds.min(u32::MAX as u64) as u32) } #[cfg(test)] mod tests { use super::*; /// The bug this module works around: `MM:SS` is not NPT, and a 53-minute /// episode must not come back as 53 seconds. #[test] fn itunes_durations_accept_every_documented_form() { assert_eq!(parse_itunes_duration("3205".to_string()), Some(3205)); assert_eq!(parse_itunes_duration("53:25".to_string()), Some(3205)); assert_eq!(parse_itunes_duration("1:20:40".to_string()), Some(4840)); assert_eq!(parse_itunes_duration("53:25.5".to_string()), Some(3205)); assert_eq!(parse_itunes_duration("".to_string()), None); assert_eq!(parse_itunes_duration("about an hour".to_string()), None); } /// Scanned straight out of a feed body shaped like the real one, keyed by /// guid *and* enclosure url so either identity resolves it. #[test] fn durations_are_recovered_from_the_raw_body() { let body = br#" 6a6479f2a51cbd54 53:25 second 1:20:40 no-duration "#; let found = itunes_durations(body); assert_eq!(found.get("6a6479f2a51cbd54"), Some(&3205)); // The enclosure url is a key too, unescaped, for feeds without a guid. assert_eq!( found.get("https://cdn.example/a.mp3?tk=X&sig=Y"), Some(&3205) ); assert_eq!(found.get("second"), Some(&4840)); assert!(!found.contains_key("no-duration")); } #[test] fn tag_text_handles_cdata_and_attributes() { assert_eq!( tag_text("abc", "guid").as_deref(), Some("abc") ); assert_eq!( tag_text("<![CDATA[ Hello ]]>", "title").as_deref(), Some("Hello") ); assert_eq!(tag_text("", "guid"), None); assert_eq!(tag_text("nothing here", "guid"), None); } }