From c13ce71c6998b4814092f14aeefb798f85b93ab2 Mon Sep 17 00:00:00 2001 From: Paul van Tilburg Date: Fri, 27 May 2022 22:31:17 +0200 Subject: [PATCH] Add feed item limit support * The feed item limit defaults to the default page size (50) if not provided * Move caching from response to URL fetch results; add helper functions * Add a helper function to set the paging query of an URL * Modify paging so we don't retrieve more than the feed item limit --- src/lib.rs | 11 ++++-- src/mixcloud.rs | 89 ++++++++++++++++++++++++++++++++++++------------- 2 files changed, 73 insertions(+), 27 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 0551f80..43a7d1a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -92,10 +92,15 @@ pub(crate) async fn download(file: PathBuf, backend: &str) -> Result { } /// Handler for retrieving the RSS feed of user on a certain back-end. -#[get("/feed//")] -async fn feed(backend: &str, username: &str, config: &State) -> Result { +#[get("/feed//?")] +async fn feed( + backend: &str, + username: &str, + limit: Option, + config: &State, +) -> Result { let user = mixcloud::user(username).await?; - let cloudcasts = mixcloud::cloudcasts(username).await?; + let cloudcasts = mixcloud::cloudcasts(username, limit).await?; let mut last_build = DateTime::::from_utc(NaiveDateTime::from_timestamp(0, 0), Utc); let category = CategoryBuilder::default() diff --git a/src/mixcloud.rs b/src/mixcloud.rs index 86a60b7..d719c45 100644 --- a/src/mixcloud.rs +++ b/src/mixcloud.rs @@ -37,7 +37,7 @@ pub(crate) struct Pictures { } /// The Mixcloud cloudcasts response. -#[derive(Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize)] #[serde(crate = "rocket::serde")] pub(crate) struct CloudcastsResponse { /// The contained cloudcast items. @@ -49,7 +49,7 @@ pub(crate) struct CloudcastsResponse { } /// The Mixcloud paging info. -#[derive(Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize)] #[serde(crate = "rocket::serde")] pub(crate) struct CloudcastsPaging { /// The API URL of the next page. @@ -109,7 +109,7 @@ const DEFAULT_BITRATE: u32 = 64 * 1024; const DEFAULT_FILE_TYPE: &str = "audio/mpeg"; /// The default page size. -const DEFAULT_PAGE_SIZE: &str = "50"; +const DEFAULT_PAGE_SIZE: usize = 50; /// Returns the default file type used by Mixcloud. pub(crate) const fn default_file_type() -> &'static str { @@ -124,54 +124,95 @@ pub(crate) fn estimated_file_size(duration: u32) -> u32 { } /// Retrieves the user data using the Mixcloud API. -#[cached( - key = "String", - convert = r#"{ username.to_owned() }"#, - time = 3600, - result = true -)] pub(crate) async fn user(username: &str) -> Result { let mut url = Url::parse(API_BASE_URL).expect("URL can always be parsed"); url.set_path(username); println!("⏬ Retrieving user {username} from {url}..."); + fetch_user(url).await +} + +/// Fetches the user from the URL. +#[cached( + key = "String", + convert = r#"{ url.to_string() }"#, + time = 86400, + result = true +)] +/// +/// If the result is [`Ok`], the user will be cached for 24 hours for the given username. +async fn fetch_user(url: Url) -> Result { let response = reqwest::get(url).await?.error_for_status()?; let user = response.json().await?; Ok(user) } -/// Retrieves the cloudcasts of the user using the Mixcloud API. -#[cached( - key = "String", - convert = r#"{ username.to_owned() }"#, - time = 3600, - result = true -)] -pub(crate) async fn cloudcasts(username: &str) -> Result> { +/// Retrieves the cloudcasts data of the user using the Mixcloud API. +pub(crate) async fn cloudcasts(username: &str, limit: Option) -> Result> { + let mut limit = limit.unwrap_or(DEFAULT_PAGE_SIZE); + let mut offset = 0; let mut url = Url::parse(API_BASE_URL).expect("URL can always be parsed"); url.set_path(&format!("{username}/cloudcasts/")); - url.query_pairs_mut() - .append_pair("limit", DEFAULT_PAGE_SIZE) - .append_pair("offset", "0"); - println!("⏬ Retrieving cloudcasts of user {username} from {url}..."); + + set_paging_query(&mut url, limit, offset); let mut cloudcasts = Vec::with_capacity(50); // The initial limit loop { - let response = reqwest::get(url).await?.error_for_status()?; - let cloudcasts_res: CloudcastsResponse = response.json().await?; + let cloudcasts_res: CloudcastsResponse = fetch_cloudcasts(url).await?; + let count = cloudcasts_res.items.len(); cloudcasts.extend(cloudcasts_res.items); // Continue onto the next URL in the paging, if there is one. + limit = limit.saturating_sub(count); + offset += count; match cloudcasts_res.paging.next { - Some(next_url) => url = Url::parse(&next_url)?, + Some(next_url) => { + url = Url::parse(&next_url)?; + set_paging_query(&mut url, limit, offset); + } None => break, } + + // We have reached the limit. + if limit == 0 { + break; + } } Ok(cloudcasts) } +/// Fetches cloudcasts from the URL. +/// +/// If the result is [`Ok`], the cloudcasts will be cached for 24 hours for the given username. +#[cached( + key = "String", + convert = r#"{ url.to_string() }"#, + time = 86400, + result = true +)] +async fn fetch_cloudcasts(url: Url) -> Result { + let response = reqwest::get(url).await?.error_for_status()?; + let cloudcasts_res = response.json().await?; + + Ok(cloudcasts_res) +} + +/// Set paging query pairs for URL. +/// +/// The limit is capped to the default page size. Another request will be necessary to retrieve +/// more. +fn set_paging_query(url: &mut Url, limit: usize, offset: usize) { + url.query_pairs_mut() + .clear() + .append_pair( + "limit", + &format!("{}", std::cmp::min(limit, DEFAULT_PAGE_SIZE)), + ) + .append_pair("offset", &format!("{}", offset)); +} + /// Retrieves the redirect URL for the provided Mixcloud cloudcast key. #[cached( key = "String",