397 lines
15 KiB
Rust
397 lines
15 KiB
Rust
//! The feed HTTP seam.
|
|
//!
|
|
//! All network access goes through the [`Feeds`] trait so the provider's
|
|
//! subscription, slug, identity, and ordering logic is unit-tested with a fake
|
|
//! and no network (architecture/rss-provider.md). [`FeedFetcher`] is the
|
|
//! production `reqwest` + `feed-rs` implementation; tests supply their own
|
|
//! [`Feeds`].
|
|
//!
|
|
//! **Every feed URL here is a credential.** A premium podcast URL embeds a
|
|
//! per-subscriber token, so URLs are redacted from `Debug`, never logged, and
|
|
//! never carried in an error message. Enclosure URLs get the same treatment —
|
|
//! they can be signed too.
|
|
|
|
use std::collections::HashMap;
|
|
use std::fmt::{self, Debug};
|
|
use std::time::Duration;
|
|
|
|
use async_trait::async_trait;
|
|
use thiserror::Error;
|
|
|
|
/// A typed feed failure. Carries only non-secret context — never a feed URL,
|
|
/// which would put a subscriber token in the logs.
|
|
#[derive(Debug, Error)]
|
|
pub enum FetchError {
|
|
/// Transport, timeout, or a non-success status.
|
|
#[error("feed request failed: {0}")]
|
|
Http(String),
|
|
/// The body was not a feed we could parse.
|
|
#[error("feed could not be parsed: {0}")]
|
|
Parse(String),
|
|
/// The body exceeded `max_feed_bytes` (D6).
|
|
#[error("feed is larger than the {limit} byte limit")]
|
|
TooLarge { limit: u64 },
|
|
}
|
|
|
|
/// One episode, normalized out of whatever feed dialect produced it.
|
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
|
pub struct Episode {
|
|
/// The publisher's `<guid>`, or the enclosure URL when the feed omits one.
|
|
/// Keys [`Self::key`] and becomes `Track.provider_item_id` so the content
|
|
/// store de-duplicates captures of the same episode.
|
|
pub guid: String,
|
|
/// Stable short path segment: `blake3(guid)[..16]` (D4).
|
|
pub key: String,
|
|
pub title: String,
|
|
/// Show/author, used as the track artist. Empty when the feed omits it.
|
|
pub author: String,
|
|
/// Publication instant as a unix timestamp, when the feed has a parseable
|
|
/// date. `None` keeps the entry in feed order (D9).
|
|
pub published: Option<i64>,
|
|
pub duration_secs: Option<u32>,
|
|
/// The audio to play. **Secret-ish**: never logged.
|
|
pub enclosure_url: String,
|
|
}
|
|
|
|
impl Episode {
|
|
/// The path segment for `guid` — `blake3` truncated to 16 hex chars, which
|
|
/// is short, readable, and stable across builds and restarts (D4).
|
|
pub fn key_for(guid: &str) -> String {
|
|
blake3::hash(guid.as_bytes()).to_hex()[..KEY_HEX_LEN].to_string()
|
|
}
|
|
}
|
|
|
|
/// Hex characters of the guid hash kept in a path segment. 64 bits is far more
|
|
/// than enough to keep one feed's episodes apart, and stays readable.
|
|
const KEY_HEX_LEN: usize = 16;
|
|
|
|
/// A parsed feed: the publisher's own title (used to name a new subscription,
|
|
/// D5) plus its episodes.
|
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
|
pub struct Feed {
|
|
pub title: String,
|
|
pub episodes: Vec<Episode>,
|
|
}
|
|
|
|
/// Everything the provider does over the network.
|
|
#[async_trait]
|
|
pub trait Feeds: Debug + Send + Sync {
|
|
/// Fetches and parses `url`, returning at most `limit` episodes, newest
|
|
/// first. Bounded by a per-request timeout and a byte cap; a malformed
|
|
/// entry is skipped rather than failing the feed.
|
|
async fn fetch(&self, url: &str, limit: usize) -> Result<Feed, FetchError>;
|
|
}
|
|
|
|
/// The production implementation: `reqwest` for transport, `feed-rs` for
|
|
/// parsing.
|
|
pub struct FeedFetcher {
|
|
http: reqwest::Client,
|
|
max_bytes: u64,
|
|
}
|
|
|
|
impl Debug for FeedFetcher {
|
|
/// No URLs are held here, but the impl is explicit so it stays that way.
|
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
|
f.debug_struct("FeedFetcher")
|
|
.field("max_bytes", &self.max_bytes)
|
|
.finish_non_exhaustive()
|
|
}
|
|
}
|
|
|
|
impl FeedFetcher {
|
|
/// `timeout` bounds every request; `max_bytes` caps a response body so a
|
|
/// broken or hostile feed cannot exhaust memory (D6).
|
|
pub fn new(timeout: Duration, max_bytes: u64) -> Result<Self, FetchError> {
|
|
let http = reqwest::Client::builder()
|
|
.timeout(timeout)
|
|
.user_agent(USER_AGENT)
|
|
.build()
|
|
// `without_url` so a builder error cannot echo a URL.
|
|
.map_err(|err| FetchError::Http(err.without_url().to_string()))?;
|
|
Ok(Self { http, max_bytes })
|
|
}
|
|
|
|
/// Reads the body in chunks, refusing to buffer more than `max_bytes` —
|
|
/// the cap has to bite *before* the allocation, not after (D6).
|
|
async fn bounded_body(&self, mut response: reqwest::Response) -> Result<Vec<u8>, FetchError> {
|
|
let mut body: Vec<u8> = Vec::new();
|
|
while let Some(chunk) = response
|
|
.chunk()
|
|
.await
|
|
.map_err(|err| FetchError::Http(err.without_url().to_string()))?
|
|
{
|
|
if body.len() as u64 + chunk.len() as u64 > self.max_bytes {
|
|
return Err(FetchError::TooLarge {
|
|
limit: self.max_bytes,
|
|
});
|
|
}
|
|
body.extend_from_slice(&chunk);
|
|
}
|
|
Ok(body)
|
|
}
|
|
}
|
|
|
|
/// Sent on every feed request. Some publishers reject an empty agent.
|
|
const USER_AGENT: &str = concat!("crabidy/", env!("CARGO_PKG_VERSION"));
|
|
|
|
#[async_trait]
|
|
impl Feeds for FeedFetcher {
|
|
async fn fetch(&self, url: &str, limit: usize) -> Result<Feed, FetchError> {
|
|
// `without_url` on every error: a premium feed URL is a credential and
|
|
// reqwest puts the URL in its Display output by default (G1).
|
|
let response = self
|
|
.http
|
|
.get(url)
|
|
.send()
|
|
.await
|
|
.map_err(|err| FetchError::Http(err.without_url().to_string()))?;
|
|
let status = response.status();
|
|
if !status.is_success() {
|
|
return Err(FetchError::Http(status.to_string()));
|
|
}
|
|
let body = self.bounded_body(response).await?;
|
|
// Recovered before parsing, because feed-rs loses this field (see
|
|
// `itunes_durations`).
|
|
let durations = itunes_durations(&body);
|
|
let parsed = feed_rs::parser::parse(body.as_slice())
|
|
.map_err(|err| FetchError::Parse(err.to_string()))?;
|
|
Ok(Feed {
|
|
title: parsed
|
|
.title
|
|
.map(|t| t.content)
|
|
.unwrap_or_default()
|
|
.trim()
|
|
.to_string(),
|
|
episodes: episodes_from(parsed.entries, limit, &durations),
|
|
})
|
|
}
|
|
}
|
|
|
|
/// Normalizes feed entries into episodes: newest first, capped, and skipping
|
|
/// anything unplayable. A single bad entry never fails the feed (G15).
|
|
fn episodes_from(
|
|
entries: Vec<feed_rs::model::Entry>,
|
|
limit: usize,
|
|
durations: &HashMap<String, u32>,
|
|
) -> Vec<Episode> {
|
|
let mut episodes: Vec<Episode> = entries
|
|
.into_iter()
|
|
.filter_map(|entry| {
|
|
let enclosure_url = audio_url(&entry)?;
|
|
// RSS `<guid>` normalizes into `id`; a feed without one keys on
|
|
// the enclosure URL instead (D4). The URL stays out of the path —
|
|
// only its hash is used.
|
|
let guid = if entry.id.trim().is_empty() {
|
|
enclosure_url.clone()
|
|
} else {
|
|
entry.id.clone()
|
|
};
|
|
let title = entry
|
|
.title
|
|
.as_ref()
|
|
.map(|t| t.content.trim().to_string())
|
|
.filter(|t| !t.is_empty())
|
|
.unwrap_or_else(|| "(untitled)".to_string());
|
|
let duration_secs = durations
|
|
.get(&guid)
|
|
.copied()
|
|
.or_else(|| duration_of(&entry));
|
|
Some(Episode {
|
|
key: Episode::key_for(&guid),
|
|
guid,
|
|
title,
|
|
author: entry
|
|
.authors
|
|
.first()
|
|
.map(|p| p.name.trim().to_string())
|
|
.unwrap_or_default(),
|
|
published: entry.published.or(entry.updated).map(|d| d.timestamp()),
|
|
// Our own `itunes:duration` wins over feed-rs's NPT parse.
|
|
duration_secs,
|
|
enclosure_url,
|
|
})
|
|
})
|
|
.collect();
|
|
// Newest first where dates parse; a stable sort keeps dateless feeds in
|
|
// publication order (D9).
|
|
// Descending: `Reverse` keeps clippy's sort_by_key form while
|
|
// sorting newest first.
|
|
episodes.sort_by_key(|e| std::cmp::Reverse(e.published));
|
|
episodes.truncate(limit);
|
|
episodes
|
|
}
|
|
|
|
/// The first playable audio URL of an entry: a media enclosure, else a link
|
|
/// that advertises audio (feed dialects disagree about where it goes).
|
|
fn audio_url(entry: &feed_rs::model::Entry) -> Option<String> {
|
|
let media = entry.media.iter().flat_map(|m| m.content.iter());
|
|
// Prefer something explicitly typed as audio, then any media url at all.
|
|
let typed = media.clone().find(|c| {
|
|
c.url.is_some()
|
|
&& c.content_type
|
|
.as_ref()
|
|
.is_some_and(|ct| ct.to_string().starts_with("audio"))
|
|
});
|
|
if let Some(url) = typed.and_then(|c| c.url.as_ref()) {
|
|
return Some(url.to_string());
|
|
}
|
|
if let Some(url) = media.filter_map(|c| c.url.as_ref()).next() {
|
|
return Some(url.to_string());
|
|
}
|
|
entry
|
|
.links
|
|
.iter()
|
|
.find(|l| {
|
|
l.rel.as_deref() == Some("enclosure")
|
|
|| l.media_type
|
|
.as_deref()
|
|
.is_some_and(|ct| ct.starts_with("audio"))
|
|
})
|
|
.map(|l| l.href.clone())
|
|
}
|
|
|
|
/// `itunes:duration` (or a media duration) in whole seconds.
|
|
fn duration_of(entry: &feed_rs::model::Entry) -> Option<u32> {
|
|
entry
|
|
.media
|
|
.iter()
|
|
.find_map(|m| {
|
|
m.duration
|
|
.or_else(|| m.content.iter().find_map(|c| c.duration))
|
|
})
|
|
.map(|d| d.as_secs().min(u32::MAX as u64) as u32)
|
|
}
|
|
|
|
/// `itunes:duration` values recovered from the raw feed, keyed by both the
|
|
/// item's guid and its enclosure URL (the two things an [`Episode`] can key
|
|
/// on).
|
|
///
|
|
/// Why this exists: feed-rs parses `<itunes:duration>` with an **NPT** parser,
|
|
/// and NPT has no `MM:SS` form. For `<itunes:duration>53:25</itunes:duration>`
|
|
/// its fallback regex matches the leading number, so a 53-minute episode comes
|
|
/// back as 53 *seconds*. `1:20:40` happens to parse correctly. The iTunes spec
|
|
/// allows `S`, `MM:SS` and `HH:MM:SS`, so the field is recovered here and
|
|
/// overrides what feed-rs produced.
|
|
///
|
|
/// This is a deliberately shallow scan, not a second feed parser: it walks
|
|
/// item chunks and pulls three optional strings. Anything it cannot find falls
|
|
/// back to feed-rs's value.
|
|
fn itunes_durations(body: &[u8]) -> HashMap<String, u32> {
|
|
let text = String::from_utf8_lossy(body);
|
|
let mut found = HashMap::new();
|
|
// `<item` for RSS, `<entry` for Atom; the first chunk is the feed header.
|
|
for chunk in text.split("<item").flat_map(|c| c.split("<entry")).skip(1) {
|
|
let Some(duration) = tag_text(chunk, "itunes:duration").and_then(parse_itunes_duration)
|
|
else {
|
|
continue;
|
|
};
|
|
if let Some(guid) = tag_text(chunk, "guid") {
|
|
found.insert(guid, duration);
|
|
}
|
|
if let Some(url) = enclosure_url_attr(chunk) {
|
|
found.insert(url, duration);
|
|
}
|
|
}
|
|
found
|
|
}
|
|
|
|
/// The text of the first `<name …>text</name>` in `chunk`, trimmed.
|
|
fn tag_text(chunk: &str, name: &str) -> Option<String> {
|
|
let open = chunk.find(&format!("<{name}"))?;
|
|
let after_open = chunk[open..].find('>')? + open + 1;
|
|
let close = chunk[after_open..].find(&format!("</{name}>"))? + after_open;
|
|
let inner = chunk[after_open..close]
|
|
.trim()
|
|
.trim_start_matches("<![CDATA[")
|
|
.trim_end_matches("]]>")
|
|
.trim();
|
|
(!inner.is_empty()).then(|| inner.to_string())
|
|
}
|
|
|
|
/// The `url="…"` of the chunk's `<enclosure>`, XML-unescaped for `&` — the
|
|
/// one entity that routinely appears in a query string.
|
|
fn enclosure_url_attr(chunk: &str) -> Option<String> {
|
|
let at = chunk.find("<enclosure")?;
|
|
let rest = &chunk[at..];
|
|
let start = rest.find("url=\"")? + 5;
|
|
let end = rest[start..].find('"')? + start;
|
|
Some(rest[start..end].replace("&", "&"))
|
|
}
|
|
|
|
/// `itunes:duration` in whole seconds: `S`, `MM:SS`, or `HH:MM:SS`. Fractional
|
|
/// seconds are truncated; anything unparseable is `None` rather than a guess.
|
|
fn parse_itunes_duration(text: String) -> Option<u32> {
|
|
let mut parts: Vec<u64> = Vec::new();
|
|
for field in text.split(':') {
|
|
// Tolerate "53:25.5" and stray whitespace.
|
|
let field = field.trim();
|
|
let field = field.split('.').next().unwrap_or(field);
|
|
parts.push(field.parse::<u64>().ok()?);
|
|
}
|
|
let seconds = match parts.as_slice() {
|
|
[s] => *s,
|
|
[m, s] => m * 60 + s,
|
|
[h, m, s] => h * 3600 + m * 60 + s,
|
|
_ => return None,
|
|
};
|
|
Some(seconds.min(u32::MAX as u64) as u32)
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
|
|
/// The bug this module works around: `MM:SS` is not NPT, and a 53-minute
|
|
/// episode must not come back as 53 seconds.
|
|
#[test]
|
|
fn itunes_durations_accept_every_documented_form() {
|
|
assert_eq!(parse_itunes_duration("3205".to_string()), Some(3205));
|
|
assert_eq!(parse_itunes_duration("53:25".to_string()), Some(3205));
|
|
assert_eq!(parse_itunes_duration("1:20:40".to_string()), Some(4840));
|
|
assert_eq!(parse_itunes_duration("53:25.5".to_string()), Some(3205));
|
|
assert_eq!(parse_itunes_duration("".to_string()), None);
|
|
assert_eq!(parse_itunes_duration("about an hour".to_string()), None);
|
|
}
|
|
|
|
/// Scanned straight out of a feed body shaped like the real one, keyed by
|
|
/// guid *and* enclosure url so either identity resolves it.
|
|
#[test]
|
|
fn durations_are_recovered_from_the_raw_body() {
|
|
let body = br#"<rss><channel>
|
|
<item>
|
|
<guid isPermaLink="false">6a6479f2a51cbd54</guid>
|
|
<itunes:duration>53:25</itunes:duration>
|
|
<enclosure url="https://cdn.example/a.mp3?tk=X&sig=Y" type="audio/mpeg"/>
|
|
</item>
|
|
<item>
|
|
<guid>second</guid>
|
|
<itunes:duration>1:20:40</itunes:duration>
|
|
</item>
|
|
<item><guid>no-duration</guid></item>
|
|
</channel></rss>"#;
|
|
let found = itunes_durations(body);
|
|
assert_eq!(found.get("6a6479f2a51cbd54"), Some(&3205));
|
|
// The enclosure url is a key too, unescaped, for feeds without a guid.
|
|
assert_eq!(
|
|
found.get("https://cdn.example/a.mp3?tk=X&sig=Y"),
|
|
Some(&3205)
|
|
);
|
|
assert_eq!(found.get("second"), Some(&4840));
|
|
assert!(!found.contains_key("no-duration"));
|
|
}
|
|
|
|
#[test]
|
|
fn tag_text_handles_cdata_and_attributes() {
|
|
assert_eq!(
|
|
tag_text("<guid isPermaLink=\"false\">abc</guid>", "guid").as_deref(),
|
|
Some("abc")
|
|
);
|
|
assert_eq!(
|
|
tag_text("<title><![CDATA[ Hello ]]></title>", "title").as_deref(),
|
|
Some("Hello")
|
|
);
|
|
assert_eq!(tag_text("<guid></guid>", "guid"), None);
|
|
assert_eq!(tag_text("nothing here", "guid"), None);
|
|
}
|
|
}
|