From 1bcc83bbfaf1a7b8cb125899614e94d9edf2042a Mon Sep 17 00:00:00 2001
From: Paul van Tilburg <paul@luon.net>
Date: Thu, 26 May 2022 22:06:22 +0200
Subject: [PATCH] Retrieve all pages by following the next URL

* Derserialize the paging information
* Parse each next URL; handle URL parse errors
* Use a default page size of 50; pass offset 0 to count by item index
---
 Cargo.lock      |  1 +
 Cargo.toml      |  1 +
 src/lib.rs      |  3 +++
 src/mixcloud.rs | 44 ++++++++++++++++++++++++++++++++++++--------
 4 files changed, 41 insertions(+), 8 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 5f9d164..fa3d428 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1598,6 +1598,7 @@ dependencies = [
  "rocket_dyn_templates",
  "rss",
  "thiserror",
+ "url",
  "youtube_dl",
 ]
 
diff --git a/Cargo.toml b/Cargo.toml
index 37c39d9..7949ce1 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -15,6 +15,7 @@ rocket = { version = "0.5.0-rc.2", features = ["json"] }
 rocket_dyn_templates = { version = "0.1.0-rc.2", features = ["tera"] }
 rss = "2.0.1"
 thiserror = "1.0.31"
+url = "2.2.2"
 youtube_dl = { version = "0.7.0", features = ["tokio"] }
 
 [package.metadata.deb]
diff --git a/src/lib.rs b/src/lib.rs
index c6e3ea7..0551f80 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -43,6 +43,9 @@ pub(crate) enum Error {
     #[error("Unknown supported back-end: {0}")]
     UnsupportedBackend(String),
 
+    #[error("URL parse error: {0}")]
+    UrlParse(#[from] url::ParseError),
+
     #[error("Youtube_dl failed: {0}")]
     YoutubeDl(#[from] youtube_dl::Error),
 }
diff --git a/src/mixcloud.rs b/src/mixcloud.rs
index d3d405c..ceb938b 100644
--- a/src/mixcloud.rs
+++ b/src/mixcloud.rs
@@ -11,7 +11,7 @@ use youtube_dl::{YoutubeDl, YoutubeDlOutput};
 
 use super::{Error, Result};
 
-/// A Mixcloud user.
+/// A Mixcloud user (response).
 #[derive(Clone, Debug, Deserialize)]
 #[serde(crate = "rocket::serde")]
 pub(crate) struct User {
@@ -36,12 +36,24 @@ pub(crate) struct Pictures {
     pub(crate) large: String,
 }
 
-/// The Mixcloud cloudcasts container.
+/// The Mixcloud cloudcasts response.
 #[derive(Debug, Deserialize)]
 #[serde(crate = "rocket::serde")]
-pub(crate) struct CloudcastData {
-    /// The contained cloudcasts.
-    data: Vec<Cloudcast>,
+pub(crate) struct CloudcastsResponse {
+    /// The contained cloudcast items.
+    #[serde(rename = "data")]
+    items: Vec<Cloudcast>,
+
+    /// The paging information.
+    paging: CloudcastsPaging,
+}
+
+/// The Mixcloud paging info.
+#[derive(Debug, Deserialize)]
+#[serde(crate = "rocket::serde")]
+pub(crate) struct CloudcastsPaging {
+    /// The API URL of the next page.
+    next: Option<String>,
 }
 
 /// A Mixcloud cloudcast.
@@ -96,6 +108,9 @@ const DEFAULT_BITRATE: u32 = 64 * 1024;
 /// The default file (MIME) type used by Mixcloud.
 const DEFAULT_FILE_TYPE: &str = "audio/mpeg";
 
+/// The default page size.
+const DEFAULT_PAGE_SIZE: &str = "50";
+
 /// Returns the default file type used by Mixcloud.
 pub(crate) fn default_file_type() -> &'static str {
     DEFAULT_FILE_TYPE
@@ -136,12 +151,25 @@ pub(crate) async fn user(username: &str) -> Result<User> {
 pub(crate) async fn cloudcasts(username: &str) -> Result<Vec<Cloudcast>> {
     let mut url = Url::parse(API_BASE_URL).expect("URL can always be parsed");
     url.set_path(&format!("{username}/cloudcasts/"));
+    url.query_pairs_mut()
+        .append_pair("limit", DEFAULT_PAGE_SIZE)
+        .append_pair("offset", "0");
 
     println!("⏬ Retrieving cloudcasts of user {username} from {url}...");
-    let response = reqwest::get(url).await?.error_for_status()?;
-    let cloudcasts: CloudcastData = response.json().await?;
+    let mut cloudcasts = Vec::with_capacity(50); // The initial limit
+    loop {
+        let response = reqwest::get(url).await?.error_for_status()?;
+        let cloudcasts_res: CloudcastsResponse = response.json().await?;
+        cloudcasts.extend(cloudcasts_res.items);
 
-    Ok(cloudcasts.data)
+        // Continue onto the next URL in the paging, if there is one.
+        match cloudcasts_res.paging.next {
+            Some(next_url) => url = Url::parse(&next_url)?,
+            None => break,
+        }
+    }
+
+    Ok(cloudcasts)
 }
 
 /// Retrieves the redirect URL for the provided Mixcloud cloudcast key.