Compare commits

...

4 commits

Author SHA1 Message Date
Paul van Tilburg 0701088fbc
Update the documentation 2022-05-27 22:47:52 +02:00
Paul van Tilburg c13ce71c69
Add feed item limit support
* The feed item limit defaults to the default page size (50) if not
  provided
* Move caching from response to URL fetch results; add helper functions
* Add a helper function to set the paging query of an URL
* Modify paging so we don't retrieve more than the feed item limit
2022-05-27 22:47:52 +02:00
Paul van Tilburg 78fc93fedf
Retrieve all pages by following the next URL
* Derserialize the paging information
* Parse each next URL; handle URL parse errors
* Use a default page size of 50; pass offset 0 to count by item index
2022-05-27 22:47:52 +02:00
Paul van Tilburg 09ee0b9ba9
Make default_file_type functions const 2022-05-27 22:39:53 +02:00
5 changed files with 118 additions and 28 deletions

1
Cargo.lock generated
View file

@ -1598,6 +1598,7 @@ dependencies = [
"rocket_dyn_templates",
"rss",
"thiserror",
"url",
"youtube_dl",
]

View file

@ -15,6 +15,7 @@ rocket = { version = "0.5.0-rc.2", features = ["json"] }
rocket_dyn_templates = { version = "0.1.0-rc.2", features = ["tera"] }
rss = "2.0.1"
thiserror = "1.0.31"
url = "2.2.2"
youtube_dl = { version = "0.7.0", features = ["tokio"] }
[package.metadata.deb]

View file

@ -55,6 +55,15 @@ need to use for Podbringer is comprised of the following parts:
The Podbringer location URL Service User @ service
```
### Feed item limit
To prevent feeds with a very large number of items, any feed that is returned
contains at most 50 items by default. If you want to have more (or less) items,
provide the limit in the URL by setting the `limit` parameter.
For example, to get up until 1000 items the URL becomes:
`https://my.domain.tld/podbringer/feed/mixcloud/myfavouriteband?limit=1000`
## License
Podbringer is licensed under the MIT license (see the `LICENSE` file or

View file

@ -43,6 +43,9 @@ pub(crate) enum Error {
#[error("Unknown supported back-end: {0}")]
UnsupportedBackend(String),
#[error("URL parse error: {0}")]
UrlParse(#[from] url::ParseError),
#[error("Youtube_dl failed: {0}")]
YoutubeDl(#[from] youtube_dl::Error),
}
@ -89,10 +92,17 @@ pub(crate) async fn download(file: PathBuf, backend: &str) -> Result<Redirect> {
}
/// Handler for retrieving the RSS feed of user on a certain back-end.
#[get("/feed/<backend>/<username>")]
async fn feed(backend: &str, username: &str, config: &State<Config>) -> Result<RssFeed> {
///
/// The limit parameter determines the maximum of items that can be in the feed.
#[get("/feed/<backend>/<username>?<limit>")]
async fn feed(
backend: &str,
username: &str,
limit: Option<usize>,
config: &State<Config>,
) -> Result<RssFeed> {
let user = mixcloud::user(username).await?;
let cloudcasts = mixcloud::cloudcasts(username).await?;
let cloudcasts = mixcloud::cloudcasts(username, limit).await?;
let mut last_build = DateTime::<Utc>::from_utc(NaiveDateTime::from_timestamp(0, 0), Utc);
let category = CategoryBuilder::default()

View file

@ -11,7 +11,7 @@ use youtube_dl::{YoutubeDl, YoutubeDlOutput};
use super::{Error, Result};
/// A Mixcloud user.
/// A Mixcloud user (response).
#[derive(Clone, Debug, Deserialize)]
#[serde(crate = "rocket::serde")]
pub(crate) struct User {
@ -36,12 +36,24 @@ pub(crate) struct Pictures {
pub(crate) large: String,
}
/// The Mixcloud cloudcasts container.
#[derive(Debug, Deserialize)]
/// The Mixcloud cloudcasts response.
#[derive(Clone, Debug, Deserialize)]
#[serde(crate = "rocket::serde")]
pub(crate) struct CloudcastData {
/// The contained cloudcasts.
data: Vec<Cloudcast>,
pub(crate) struct CloudcastsResponse {
/// The contained cloudcast items.
#[serde(rename = "data")]
items: Vec<Cloudcast>,
/// The paging information.
paging: CloudcastsPaging,
}
/// The Mixcloud paging info.
#[derive(Clone, Debug, Deserialize)]
#[serde(crate = "rocket::serde")]
pub(crate) struct CloudcastsPaging {
/// The API URL of the next page.
next: Option<String>,
}
/// A Mixcloud cloudcast.
@ -96,8 +108,11 @@ const DEFAULT_BITRATE: u32 = 64 * 1024;
/// The default file (MIME) type used by Mixcloud.
const DEFAULT_FILE_TYPE: &str = "audio/mpeg";
/// The default page size.
const DEFAULT_PAGE_SIZE: usize = 50;
/// Returns the default file type used by Mixcloud.
pub(crate) fn default_file_type() -> &'static str {
pub(crate) const fn default_file_type() -> &'static str {
DEFAULT_FILE_TYPE
}
@ -109,39 +124,93 @@ pub(crate) fn estimated_file_size(duration: u32) -> u32 {
}
/// Retrieves the user data using the Mixcloud API.
#[cached(
key = "String",
convert = r#"{ username.to_owned() }"#,
time = 3600,
result = true
)]
pub(crate) async fn user(username: &str) -> Result<User> {
let mut url = Url::parse(API_BASE_URL).expect("URL can always be parsed");
url.set_path(username);
println!("⏬ Retrieving user {username} from {url}...");
fetch_user(url).await
}
/// Fetches the user from the URL.
#[cached(
key = "String",
convert = r#"{ url.to_string() }"#,
time = 86400,
result = true
)]
///
/// If the result is [`Ok`], the user will be cached for 24 hours for the given username.
async fn fetch_user(url: Url) -> Result<User> {
let response = reqwest::get(url).await?.error_for_status()?;
let user = response.json().await?;
Ok(user)
}
/// Retrieves the cloudcasts of the user using the Mixcloud API.
#[cached(
key = "String",
convert = r#"{ username.to_owned() }"#,
time = 3600,
result = true
)]
pub(crate) async fn cloudcasts(username: &str) -> Result<Vec<Cloudcast>> {
/// Retrieves the cloudcasts data of the user using the Mixcloud API.
pub(crate) async fn cloudcasts(username: &str, limit: Option<usize>) -> Result<Vec<Cloudcast>> {
let mut limit = limit.unwrap_or(DEFAULT_PAGE_SIZE);
let mut offset = 0;
let mut url = Url::parse(API_BASE_URL).expect("URL can always be parsed");
url.set_path(&format!("{username}/cloudcasts/"));
println!("⏬ Retrieving cloudcasts of user {username} from {url}...");
let response = reqwest::get(url).await?.error_for_status()?;
let cloudcasts: CloudcastData = response.json().await?;
Ok(cloudcasts.data)
set_paging_query(&mut url, limit, offset);
let mut cloudcasts = Vec::with_capacity(50); // The initial limit
loop {
let cloudcasts_res: CloudcastsResponse = fetch_cloudcasts(url).await?;
let count = cloudcasts_res.items.len();
cloudcasts.extend(cloudcasts_res.items);
// Continue onto the next URL in the paging, if there is one.
limit = limit.saturating_sub(count);
offset += count;
match cloudcasts_res.paging.next {
Some(next_url) => {
url = Url::parse(&next_url)?;
set_paging_query(&mut url, limit, offset);
}
None => break,
}
// We have reached the limit.
if limit == 0 {
break;
}
}
Ok(cloudcasts)
}
/// Fetches cloudcasts from the URL.
///
/// If the result is [`Ok`], the cloudcasts will be cached for 24 hours for the given username.
#[cached(
key = "String",
convert = r#"{ url.to_string() }"#,
time = 86400,
result = true
)]
async fn fetch_cloudcasts(url: Url) -> Result<CloudcastsResponse> {
let response = reqwest::get(url).await?.error_for_status()?;
let cloudcasts_res = response.json().await?;
Ok(cloudcasts_res)
}
/// Set paging query pairs for URL.
///
/// The limit is capped to the default page size. Another request will be necessary to retrieve
/// more.
fn set_paging_query(url: &mut Url, limit: usize, offset: usize) {
url.query_pairs_mut()
.clear()
.append_pair(
"limit",
&format!("{}", std::cmp::min(limit, DEFAULT_PAGE_SIZE)),
)
.append_pair("offset", &format!("{}", offset));
}
/// Retrieves the redirect URL for the provided Mixcloud cloudcast key.