clusterfuck
/
crabidy
4
1
Fork 0
Code Issues 15 Pull Requests Actions Packages Projects 1 Releases 5 Wiki Activity
crabidy/rssdy/src/api.rs

564 lines
22 KiB
Rust
Raw Blame History

//! The feed HTTP seam.
//!
//! All network access goes through the [`Feeds`] trait so the provider's
//! subscription, slug, identity, and ordering logic is unit-tested with a fake
//! and no network (architecture/rss-provider.md). [`FeedFetcher`] is the
//! production `reqwest` + `feed-rs` implementation; tests supply their own
//! [`Feeds`].
//!
//! **Every feed URL here is a credential.** A premium podcast URL embeds a
//! per-subscriber token, so URLs are redacted from `Debug`, never logged, and
//! never carried in an error message. Enclosure URLs get the same treatment —
//! they can be signed too.
use std::collections::HashMap;
use std::fmt::{self, Debug};
use std::time::Duration;
use async_trait::async_trait;
use thiserror::Error;
use tracing::warn;
/// A typed feed failure. Carries only non-secret context — never a feed URL,
/// which would put a subscriber token in the logs.
#[derive(Debug, Error)]
pub enum FetchError {
/// Transport, timeout, or a non-success status.
#[error("feed request failed: {0}")]
Http(String),
/// The body was not a feed we could parse.
#[error("feed could not be parsed: {0}")]
Parse(String),
/// The body exceeded `max_feed_bytes` (D6).
#[error("feed is larger than the {limit} byte limit")]
TooLarge { limit: u64 },
}
/// One episode, normalized out of whatever feed dialect produced it.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Episode {
/// The publisher's `<guid>`, or the enclosure URL when the feed omits one.
/// Keys [`Self::key`] and becomes `Track.provider_item_id` so the content
/// store de-duplicates captures of the same episode.
pub guid: String,
/// Stable short path segment: `blake3(guid)[..16]` (D4).
pub key: String,
pub title: String,
/// Show/author, used as the track artist. Empty when the feed omits it.
pub author: String,
/// Publication instant as a unix timestamp, when the feed has a parseable
/// date. `None` keeps the entry in feed order (D9).
pub published: Option<i64>,
pub duration_secs: Option<u32>,
/// The audio to play. **Secret-ish**: never logged.
pub enclosure_url: String,
}
impl Episode {
/// The path segment for `guid` — `blake3` truncated to 16 hex chars, which
/// is short, readable, and stable across builds and restarts (D4).
pub fn key_for(guid: &str) -> String {
blake3::hash(guid.as_bytes()).to_hex()[..KEY_HEX_LEN].to_string()
}
}
/// Hex characters of the guid hash kept in a path segment. 64 bits is far more
/// than enough to keep one feed's episodes apart, and stays readable.
const KEY_HEX_LEN: usize = 16;
/// A parsed feed: the publisher's own title (used to name a new subscription,
/// D5) plus its episodes.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Feed {
pub title: String,
pub episodes: Vec<Episode>,
}
/// Everything the provider does over the network.
#[async_trait]
pub trait Feeds: Debug + Send + Sync {
/// Fetches and parses `url`, returning at most `limit` episodes, newest
/// first. Bounded by a per-request timeout and a byte cap; a malformed
/// entry is skipped rather than failing the feed.
async fn fetch(&self, url: &str, limit: usize) -> Result<Feed, FetchError>;
}
/// The production implementation: `reqwest` for transport, `feed-rs` for
/// parsing.
pub struct FeedFetcher {
http: reqwest::Client,
max_bytes: u64,
}
impl Debug for FeedFetcher {
/// No URLs are held here, but the impl is explicit so it stays that way.
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("FeedFetcher")
.field("max_bytes", &self.max_bytes)
.finish_non_exhaustive()
}
}
impl FeedFetcher {
/// `timeout` bounds every request; `max_bytes` caps a response body so a
/// broken or hostile feed cannot exhaust memory (D6).
pub fn new(timeout: Duration, max_bytes: u64) -> Result<Self, FetchError> {
let http = reqwest::Client::builder()
.timeout(timeout)
.user_agent(USER_AGENT)
.build()
// `without_url` so a builder error cannot echo a URL.
.map_err(|err| FetchError::Http(err.without_url().to_string()))?;
Ok(Self { http, max_bytes })
}
/// Reads the body in chunks, refusing to buffer more than `max_bytes` —
/// the cap has to bite *before* the allocation, not after (D6).
async fn bounded_body(&self, mut response: reqwest::Response) -> Result<Vec<u8>, FetchError> {
let mut body: Vec<u8> = Vec::new();
while let Some(chunk) = response
.chunk()
.await
.map_err(|err| FetchError::Http(err.without_url().to_string()))?
{
if body.len() as u64 + chunk.len() as u64 > self.max_bytes {
return Err(FetchError::TooLarge {
limit: self.max_bytes,
});
}
body.extend_from_slice(&chunk);
}
Ok(body)
}
}
/// Sent on every feed request. Some publishers reject an empty agent.
const USER_AGENT: &str = concat!("crabidy/", env!("CARGO_PKG_VERSION"));
#[async_trait]
impl Feeds for FeedFetcher {
async fn fetch(&self, url: &str, limit: usize) -> Result<Feed, FetchError> {
// `without_url` on every error: a premium feed URL is a credential and
// reqwest puts the URL in its Display output by default (G1).
let response = self
.http
.get(url)
.send()
.await
.map_err(|err| FetchError::Http(err.without_url().to_string()))?;
let status = response.status();
if !status.is_success() {
return Err(FetchError::Http(status.to_string()));
}
let body = self.bounded_body(response).await?;
// Recovered before parsing, because feed-rs loses this field (see
// `itunes_durations`).
let durations = itunes_durations(&body);
let parsed = feed_rs::parser::parse(body.as_slice())
.map_err(|err| FetchError::Parse(err.to_string()))?;
Ok(Feed {
title: parsed
.title
.map(|t| t.content)
.unwrap_or_default()
.trim()
.to_string(),
episodes: episodes_from(parsed.entries, limit, &durations),
})
}
}
/// Normalizes feed entries into episodes: newest first, capped, and skipping
/// anything unplayable. A single bad entry never fails the feed (G15).
fn episodes_from(
entries: Vec<feed_rs::model::Entry>,
limit: usize,
durations: &HashMap<String, u32>,
) -> Vec<Episode> {
let entry_count = entries.len();
let mut episodes: Vec<Episode> = entries
.into_iter()
.filter_map(|entry| {
let enclosure_url = audio_url(&entry)?;
// RSS `<guid>` normalizes into `id`; a feed without one keys on
// the enclosure URL instead (D4). The URL stays out of the path —
// only its hash is used.
let guid = if entry.id.trim().is_empty() {
enclosure_url.clone()
} else {
entry.id.clone()
};
let title = entry
.title
.as_ref()
.map(|t| t.content.trim().to_string())
.filter(|t| !t.is_empty())
.unwrap_or_else(|| "(untitled)".to_string());
let duration_secs = durations
.get(&guid)
.copied()
.or_else(|| duration_of(&entry));
Some(Episode {
key: Episode::key_for(&guid),
guid,
title,
author: entry
.authors
.first()
.map(|p| p.name.trim().to_string())
.unwrap_or_default(),
published: entry.published.or(entry.updated).map(|d| d.timestamp()),
// Our own `itunes:duration` wins over feed-rs's NPT parse.
duration_secs,
enclosure_url,
})
})
.collect();
// Newest first where dates parse; a stable sort keeps dateless feeds in
// publication order (D9).
// Descending: `Reverse` keeps clippy's sort_by_key form while
// sorting newest first.
episodes.sort_by_key(|e| std::cmp::Reverse(e.published));
episodes.truncate(limit);
// A feed full of entries and empty of audio is almost always a blog feed
// subscribed by mistake (its enclosures are the posts' featured images).
// Say so: the listing itself can only show nothing, which explains nothing.
// No URL in the message — a feed URL is a credential.
if episodes.is_empty() && entry_count > 0 {
warn!(
entries = entry_count,
"feed has entries but none carry playable audio; this looks like a blog feed rather \
than a podcast feed"
);
}
episodes
}
/// Media types that some feeds use for audio files without saying so.
const GENERIC_TYPES: [&str; 2] = ["application/octet-stream", "binary/octet-stream"];
/// File extensions we treat as audio when the declared type is generic or
/// missing. Deliberately excludes `.mp4`, which is as often video as audio.
const AUDIO_EXTENSIONS: [&str; 13] = [
"mp3", "m4a", "m4b", "aac", "ogg", "oga", "opus", "flac", "wav", "aiff", "aif", "mp2", "mpga",
];
/// Whether a declared media type names audio.
fn is_audio_type(content_type: &str) -> bool {
let normalized = content_type.trim().to_ascii_lowercase();
normalized.starts_with("audio/") || normalized == "audio"
}
/// Whether an enclosure is plausibly something we can decode.
///
/// The permissive cases exist because real podcast feeds are sloppy: plenty
/// omit the `type` attribute entirely, and a few serve mp3s as
/// `application/octet-stream`. So a missing type is accepted, and a generic one
/// is accepted when the URL's extension backs it up.
///
/// What is *not* accepted is a type that names something else. Blog feeds —
/// WordPress ones especially — attach the post's featured **image** as an
/// `<enclosure>`, which is structurally identical to a podcast enclosure and
/// distinguishable only by its type. Accepting those turned every blog post
/// into an episode that failed to decode at play time.
fn plausibly_audio(content_type: Option<&str>, url: &str) -> bool {
match content_type {
None => true,
Some(ct) if is_audio_type(ct) => true,
Some(ct) => {
let normalized = ct.trim().to_ascii_lowercase();
GENERIC_TYPES.contains(&normalized.as_str()) && has_audio_extension(url)
}
}
}
/// Whether the URL's path ends in a known audio extension. Query strings and
/// fragments are ignored — enclosure URLs routinely carry tracking parameters.
fn has_audio_extension(url: &str) -> bool {
let path = url
.split(['?', '#'])
.next()
.unwrap_or(url)
.trim_end_matches('/');
let Some((_, extension)) = path.rsplit_once('.') else {
return false;
};
let extension = extension.to_ascii_lowercase();
AUDIO_EXTENSIONS.contains(&extension.as_str())
}
/// The playable audio URL of an entry: a media enclosure, else a link that
/// advertises audio (feed dialects disagree about where it goes).
///
/// An entry whose only enclosures are images or video yields `None` and is
/// skipped — better an episode that never appears than one that appears and
/// cannot be played.
fn audio_url(entry: &feed_rs::model::Entry) -> Option<String> {
let enclosures: Vec<(Option<String>, String)> = entry
.media
.iter()
.flat_map(|m| m.content.iter())
.filter_map(|content| {
let url = content.url.as_ref()?.to_string();
Some((content.content_type.as_ref().map(ToString::to_string), url))
})
.chain(entry.links.iter().filter_map(|link| {
let candidate = link.rel.as_deref() == Some("enclosure")
|| link.media_type.as_deref().is_some_and(is_audio_type);
candidate.then(|| (link.media_type.clone(), link.href.clone()))
}))
.collect();
// An explicitly audio-typed enclosure wins; a video podcast that also
// ships an audio version therefore plays as audio. Only then do the
// permissive cases (no type, generic type) get a turn.
enclosures
.iter()
.find(|(content_type, _)| content_type.as_deref().is_some_and(is_audio_type))
.or_else(|| {
enclosures
.iter()
.find(|(content_type, url)| plausibly_audio(content_type.as_deref(), url))
})
.map(|(_, url)| url.clone())
}
/// `itunes:duration` (or a media duration) in whole seconds.
fn duration_of(entry: &feed_rs::model::Entry) -> Option<u32> {
entry
.media
.iter()
.find_map(|m| {
m.duration
.or_else(|| m.content.iter().find_map(|c| c.duration))
})
.map(|d| d.as_secs().min(u32::MAX as u64) as u32)
}
/// `itunes:duration` values recovered from the raw feed, keyed by both the
/// item's guid and its enclosure URL (the two things an [`Episode`] can key
/// on).
///
/// Why this exists: feed-rs parses `<itunes:duration>` with an **NPT** parser,
/// and NPT has no `MM:SS` form. For `<itunes:duration>53:25</itunes:duration>`
/// its fallback regex matches the leading number, so a 53-minute episode comes
/// back as 53 *seconds*. `1:20:40` happens to parse correctly. The iTunes spec
/// allows `S`, `MM:SS` and `HH:MM:SS`, so the field is recovered here and
/// overrides what feed-rs produced.
///
/// This is a deliberately shallow scan, not a second feed parser: it walks
/// item chunks and pulls three optional strings. Anything it cannot find falls
/// back to feed-rs's value.
fn itunes_durations(body: &[u8]) -> HashMap<String, u32> {
let text = String::from_utf8_lossy(body);
let mut found = HashMap::new();
// `<item` for RSS, `<entry` for Atom; the first chunk is the feed header.
for chunk in text.split("<item").flat_map(|c| c.split("<entry")).skip(1) {
let Some(duration) = tag_text(chunk, "itunes:duration").and_then(parse_itunes_duration)
else {
continue;
};
if let Some(guid) = tag_text(chunk, "guid") {
found.insert(guid, duration);
}
if let Some(url) = enclosure_url_attr(chunk) {
found.insert(url, duration);
}
}
found
}
/// The text of the first `<name …>text</name>` in `chunk`, trimmed.
fn tag_text(chunk: &str, name: &str) -> Option<String> {
let open = chunk.find(&format!("<{name}"))?;
let after_open = chunk[open..].find('>')? + open + 1;
let close = chunk[after_open..].find(&format!("</{name}>"))? + after_open;
let inner = chunk[after_open..close]
.trim()
.trim_start_matches("<![CDATA[")
.trim_end_matches("]]>")
.trim();
(!inner.is_empty()).then(|| inner.to_string())
}
/// The `url="…"` of the chunk's `<enclosure>`, XML-unescaped for `&amp;` — the
/// one entity that routinely appears in a query string.
fn enclosure_url_attr(chunk: &str) -> Option<String> {
let at = chunk.find("<enclosure")?;
let rest = &chunk[at..];
let start = rest.find("url=\"")? + 5;
let end = rest[start..].find('"')? + start;
Some(rest[start..end].replace("&amp;", "&"))
}
/// `itunes:duration` in whole seconds: `S`, `MM:SS`, or `HH:MM:SS`. Fractional
/// seconds are truncated; anything unparseable is `None` rather than a guess.
fn parse_itunes_duration(text: String) -> Option<u32> {
let mut parts: Vec<u64> = Vec::new();
for field in text.split(':') {
// Tolerate "53:25.5" and stray whitespace.
let field = field.trim();
let field = field.split('.').next().unwrap_or(field);
parts.push(field.parse::<u64>().ok()?);
}
let seconds = match parts.as_slice() {
[s] => *s,
[m, s] => m * 60 + s,
[h, m, s] => h * 3600 + m * 60 + s,
_ => return None,
};
Some(seconds.min(u32::MAX as u64) as u32)
}
#[cfg(test)]
mod tests {
use super::*;
/// The bug this module works around: `MM:SS` is not NPT, and a 53-minute
/// episode must not come back as 53 seconds.
#[test]
fn itunes_durations_accept_every_documented_form() {
assert_eq!(parse_itunes_duration("3205".to_string()), Some(3205));
assert_eq!(parse_itunes_duration("53:25".to_string()), Some(3205));
assert_eq!(parse_itunes_duration("1:20:40".to_string()), Some(4840));
assert_eq!(parse_itunes_duration("53:25.5".to_string()), Some(3205));
assert_eq!(parse_itunes_duration("".to_string()), None);
assert_eq!(parse_itunes_duration("about an hour".to_string()), None);
}
/// Scanned straight out of a feed body shaped like the real one, keyed by
/// guid *and* enclosure url so either identity resolves it.
#[test]
fn durations_are_recovered_from_the_raw_body() {
let body = br#"<rss><channel>
<item>
<guid isPermaLink="false">6a6479f2a51cbd54</guid>
<itunes:duration>53:25</itunes:duration>
<enclosure url="https://cdn.example/a.mp3?tk=X&amp;sig=Y" type="audio/mpeg"/>
</item>
<item>
<guid>second</guid>
<itunes:duration>1:20:40</itunes:duration>
</item>
<item><guid>no-duration</guid></item>
</channel></rss>"#;
let found = itunes_durations(body);
assert_eq!(found.get("6a6479f2a51cbd54"), Some(&3205));
// The enclosure url is a key too, unescaped, for feeds without a guid.
assert_eq!(
found.get("https://cdn.example/a.mp3?tk=X&sig=Y"),
Some(&3205)
);
assert_eq!(found.get("second"), Some(&4840));
assert!(!found.contains_key("no-duration"));
}
/// Parses an RSS body the way [`FeedFetcher::fetch`] does and returns the
/// episodes, so these tests exercise the real feed-rs entry shapes rather
/// than hand-built model structs.
fn episodes_of(body: &str) -> Vec<Episode> {
let feed = feed_rs::parser::parse(body.as_bytes()).expect("parses");
episodes_from(feed.entries, 100, &HashMap::new())
}
/// The netzpolitik.org case: a WordPress blog feed whose `<enclosure>` is
/// the post's featured **image**. Accepting it produced episodes that
/// failed to decode at play time ("format of the data has not been
/// recognized"), which is worse than not listing them at all.
#[test]
fn image_enclosures_are_not_episodes() {
let episodes = episodes_of(
r#"<?xml version="1.0"?><rss version="2.0"><channel>
<title>A blog</title>
<item>
<title>A post</title>
<guid>post-1</guid>
<enclosure url="https://cdn.example/featured-scaled.jpg"
length="272428" type="image/jpeg" />
</item>
</channel></rss>"#,
);
assert!(
episodes.is_empty(),
"an image enclosure is not a playable episode"
);
}
/// A real podcast entry still resolves, and an entry that ships both an
/// image and audio picks the audio.
#[test]
fn audio_enclosures_win_over_other_media() {
let episodes = episodes_of(
r#"<?xml version="1.0"?><rss version="2.0"><channel>
<title>A podcast</title>
<item>
<title>Episode</title>
<guid>ep-1</guid>
<enclosure url="https://cdn.example/cover.jpg" type="image/jpeg" />
<enclosure url="https://cdn.example/ep1.mp3" type="audio/mpeg" />
</item>
</channel></rss>"#,
);
assert_eq!(episodes.len(), 1);
assert_eq!(episodes[0].enclosure_url, "https://cdn.example/ep1.mp3");
}
/// Sloppy-but-real feeds: no `type` at all, or a generic one backed by the
/// extension. Both stay playable — the decoder is the final judge.
#[test]
fn untyped_and_generic_enclosures_are_accepted() {
let episodes = episodes_of(
r#"<?xml version="1.0"?><rss version="2.0"><channel>
<title>A podcast</title>
<item><guid>a</guid>
<enclosure url="https://cdn.example/a.mp3" /></item>
<item><guid>b</guid>
<enclosure url="https://cdn.example/b.mp3?tk=X"
type="application/octet-stream" /></item>
</channel></rss>"#,
);
assert_eq!(episodes.len(), 2);
}
/// A generic type with no audible extension is not taken on faith, and
/// video enclosures are left alone.
#[test]
fn generic_and_video_enclosures_without_audio_are_skipped() {
let episodes = episodes_of(
r#"<?xml version="1.0"?><rss version="2.0"><channel>
<title>Mixed</title>
<item><guid>a</guid>
<enclosure url="https://cdn.example/a.bin"
type="application/octet-stream" /></item>
<item><guid>b</guid>
<enclosure url="https://cdn.example/b.mp4" type="video/mp4" /></item>
</channel></rss>"#,
);
assert!(episodes.is_empty());
}
#[test]
fn audio_extensions_ignore_query_strings() {
assert!(has_audio_extension("https://cdn.example/a.mp3"));
assert!(has_audio_extension("https://cdn.example/a.MP3?tk=X#f"));
assert!(has_audio_extension("https://cdn.example/a.opus?x=.jpg"));
assert!(!has_audio_extension("https://cdn.example/a.jpg"));
assert!(!has_audio_extension("https://cdn.example/a.mp4"));
assert!(!has_audio_extension("https://cdn.example/no-extension"));
assert!(!has_audio_extension(""));
}
#[test]
fn tag_text_handles_cdata_and_attributes() {
assert_eq!(
tag_text("<guid isPermaLink=\"false\">abc</guid>", "guid").as_deref(),
Some("abc")
);
assert_eq!(
tag_text("<title><![CDATA[ Hello ]]></title>", "title").as_deref(),
Some("Hello")
);
assert_eq!(tag_text("<guid></guid>", "guid"), None);
assert_eq!(tag_text("nothing here", "guid"), None);
}
}
Reference in New Issue View Git Blame Copy Permalink