Update the changelog

Bump the version to 0.2.0
Increase TTL for redirect URI to 24 hours
2022-05-27 22:59:29 +02:00 · 2022-05-27 22:59:29 +02:00 · 2022-05-27 22:59:28 +02:00 · 2022-05-27 22:50:51 +02:00 · 2022-05-27 22:47:52 +02:00 · 2022-05-27 22:47:52 +02:00
7 changed files with 287 additions and 227 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -7,9 +7,30 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

 ## [Unreleased]

+## [0.2.0] - 2022-05-27
+
+### Added
+
+* Add support for paging, i.e. retrieving more that 50 past items (#9)
+* Introduce the `limit` parameter to get more/less than 50 feed items
+* Add caching; all Mixcloud user, cloudcasts and download URL requests are
+  cached for 24 hours (#3)
+
+### Changed
+
+* Implemented proper error logging and handling (#6)
+* Replaces own youtube-dl command running implementation by `youtub_dl`
+  crate (#8)
+* Several code and documentation improvements & fixes
+
+### Removed
+
+* Drop dependencies on some unnecessary/unused crates
+
 ## [0.1.0] - 2022-05-24

 Initial release.

-[Unreleased]: https://git.luon.net/paul/podbringer/compare/v0.1.0...HEAD
+[Unreleased]: https://git.luon.net/paul/podbringer/compare/v0.2.0...HEAD
+[0.2.0]: https://git.luon.net/paul/podbringer/compare/tag/v0.1.0..v0.2.0
 [0.1.0]: https://git.luon.net/paul/podbringer/commits/tag/v0.1.0
--- a/Cargo.lock
+++ b/Cargo.lock
@ -2,21 +2,6 @@
 # It is not intended for manual editing.
 version = 3

-[[package]]
-name = "addr2line"
-version = "0.17.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b9ecd88a8c8378ca913a680cd98f0f13ac67383d35993f86c90a70e3f137816b"
-dependencies = [
- "gimli",
-]
-
-[[package]]
-name = "adler"
-version = "1.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
-
 [[package]]
 name = "aead"
 version = "0.4.3"
@ -147,21 +132,6 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"

-[[package]]
-name = "backtrace"
-version = "0.3.65"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "11a17d453482a265fd5f8479f2a3f405566e6ca627837aaddb85af8b1ab8ef61"
-dependencies = [
- "addr2line",
- "cc",
- "cfg-if 1.0.0",
- "libc",
- "miniz_oxide",
- "object",
- "rustc-demangle",
-]
-
 [[package]]
 name = "base64"
 version = "0.13.0"
@ -307,7 +277,7 @@ dependencies = [
 "num-integer",
 "num-traits",
 "serde",
- "time 0.1.43",
+ "time 0.1.44",
 "winapi 0.3.9",
 ]

@ -342,33 +312,6 @@ dependencies = [
 "generic-array 0.14.5",
 ]

-[[package]]
-name = "color-eyre"
-version = "0.6.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8ebf286c900a6d5867aeff75cfee3192857bb7f24b547d4f0df2ed6baa812c90"
-dependencies = [
- "backtrace",
- "color-spantrace",
- "eyre",
- "indenter",
- "once_cell",
- "owo-colors",
- "tracing-error",
-]
-
-[[package]]
-name = "color-spantrace"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ba75b3d9449ecdccb27ecbc479fdc0b87fa2dd43d2f8298f9bf0e59aacc8dce"
-dependencies = [
- "once_cell",
- "owo-colors",
- "tracing-core",
- "tracing-error",
-]
-
 [[package]]
 name = "cookie"
 version = "0.16.0"
@ -625,16 +568,6 @@ dependencies = [
 "cfg-if 1.0.0",
 ]

-[[package]]
-name = "eyre"
-version = "0.6.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c2b6b5a29c02cdc822728b7d7b8ae1bab3e3b05d44522770ddd49722eeac7eb"
-dependencies = [
- "indenter",
- "once_cell",
-]
-
 [[package]]
 name = "fake-simd"
 version = "0.1.2"
@ -871,7 +804,7 @@ checksum = "9be70c98951c83b8d2f8f60d7065fa6d5146873094452a1008da8c2f1e4205ad"
 dependencies = [
 "cfg-if 1.0.0",
 "libc",
- "wasi 0.10.2+wasi-snapshot-preview1",
+ "wasi 0.10.0+wasi-snapshot-preview1",
 ]

 [[package]]
@ -884,12 +817,6 @@ dependencies = [
 "polyval",
 ]

-[[package]]
-name = "gimli"
-version = "0.26.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "78cc372d058dcf6d5ecd98510e7fbc9e5aec4d21de70f65fea8fecebcd881bd4"
-
 [[package]]
 name = "glob"
 version = "0.3.0"
@ -991,9 +918,9 @@ dependencies = [

 [[package]]
 name = "http-body"
-version = "0.4.4"
+version = "0.4.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ff4f84919677303da5f147645dbea6b1881f368d03ac84e1dc09031ebd7b2c6"
+checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1"
 dependencies = [
 "bytes",
 "http",
@ -1090,12 +1017,6 @@ dependencies = [
 "winapi-util",
 ]

-[[package]]
-name = "indenter"
-version = "0.3.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ce23b50ad8242c51a442f3ff322d56b02f08852c77e4c0b4d3fd684abc89c683"
-
 [[package]]
 name = "indexmap"
 version = "1.8.1"
@ -1196,9 +1117,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"

 [[package]]
 name = "libc"
-version = "0.2.125"
+version = "0.2.126"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5916d2ae698f6de9bfb891ad7a8d65c09d232dc58cc4ac433c7da3b2fd84bc2b"
+checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836"

 [[package]]
 name = "lock_api"
@ -1221,9 +1142,9 @@ dependencies = [

 [[package]]
 name = "loom"
-version = "0.5.5"
+version = "0.5.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "85eb735cf3c8ebac6cc3655c5da2f4a088b6a19133aa482471a21ba0eb5d83ab"
+checksum = "ff50ecb28bb86013e935fb6683ab1f6d3a20016f123c76fd4c27470076ac30f5"
 dependencies = [
 "cfg-if 1.0.0",
 "generator",
@ -1267,15 +1188,6 @@ version = "0.3.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d"

-[[package]]
-name = "miniz_oxide"
-version = "0.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d2b29bd4bc3f33391105ebee3589c19197c4271e3e5a9ec9bfe8127eeff8f082"
-dependencies = [
- "adler",
-]
-
 [[package]]
 name = "mio"
 version = "0.6.23"
@ -1451,20 +1363,11 @@ dependencies = [
 "libc",
 ]

-[[package]]
-name = "object"
-version = "0.28.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e42c982f2d955fac81dd7e1d0e1426a7d702acd9c98d19ab01083a6a0328c424"
-dependencies = [
- "memchr",
-]
-
 [[package]]
 name = "once_cell"
-version = "1.10.0"
+version = "1.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9"
+checksum = "7709cef83f0c1f58f666e746a08b21e0085f7440fa6a29cc194d68aac97a4225"

 [[package]]
 name = "opaque-debug"
@ -1523,12 +1426,6 @@ dependencies = [
 "vcpkg",
 ]

-[[package]]
-name = "owo-colors"
-version = "3.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "decf7381921fea4dcb2549c5667eda59b3ec297ab7e2b5fc33eac69d2e7da87b"
-
 [[package]]
 name = "parking_lot"
 version = "0.12.0"
@ -1692,17 +1589,17 @@ checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae"

 [[package]]
 name = "podbringer"
-version = "0.1.0"
+version = "0.2.0"
 dependencies = [
 "cached",
 "chrono",
- "color-eyre",
 "reqwest",
 "rocket",
 "rocket_dyn_templates",
 "rss",
- "tempfile",
- "tokio",
+ "thiserror",
+ "url",
+ "youtube_dl",
 ]

 [[package]]
@ -1725,11 +1622,11 @@ checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872"

 [[package]]
 name = "proc-macro2"
-version = "1.0.38"
+version = "1.0.39"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9027b48e9d4c9175fa2218adf3557f91c1137021739951d4932f5f8268ac48aa"
+checksum = "c54b25569025b7fc9651de43004ae593a75ad88543b17178aa5e1b9c4f15f56f"
 dependencies = [
- "unicode-xid",
+ "unicode-ident",
 ]

 [[package]]
@ -1825,9 +1722,9 @@ dependencies = [

 [[package]]
 name = "regex"
-version = "1.5.5"
+version = "1.5.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1a11647b6b25ff05a515cb92c365cec08801e83423a235b51e231e1808747286"
+checksum = "d83f127d94bdbcda4c8cc2e50f6f84f4b611f69c902699ca385a39c3a75f9ff1"
 dependencies = [
 "aho-corasick",
 "memchr",
@ -1845,9 +1742,9 @@ dependencies = [

 [[package]]
 name = "regex-syntax"
-version = "0.6.25"
+version = "0.6.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b"
+checksum = "49b3de9ec5dc0a3417da371aab17d729997c15010e7fd24ff707773a33bddb64"

 [[package]]
 name = "remove_dir_all"
@ -2001,12 +1898,6 @@ dependencies = [
 "quick-xml",
 ]

-[[package]]
-name = "rustc-demangle"
-version = "0.1.21"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342"
-
 [[package]]
 name = "rustversion"
 version = "1.0.6"
@ -2030,12 +1921,12 @@ dependencies = [

 [[package]]
 name = "schannel"
-version = "0.1.19"
+version = "0.1.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75"
+checksum = "88d6731146462ea25d9244b2ed5fd1d716d25c52e4d54aa4fb0f3c4e9854dbe2"
 dependencies = [
 "lazy_static",
- "winapi 0.3.9",
+ "windows-sys",
 ]

 [[package]]
@ -2232,13 +2123,13 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"

 [[package]]
 name = "syn"
-version = "1.0.94"
+version = "1.0.95"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a07e33e919ebcd69113d5be0e4d70c5707004ff45188910106854f38b960df4a"
+checksum = "fbaf6116ab8924f39d52792136fb74fd60a80194cf1b1c6ffa6453eef1c3f942"
 dependencies = [
 "proc-macro2",
 "quote",
- "unicode-xid",
+ "unicode-ident",
 ]

 [[package]]
@ -2308,11 +2199,12 @@ dependencies = [

 [[package]]
 name = "time"
-version = "0.1.43"
+version = "0.1.44"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438"
+checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255"
 dependencies = [
 "libc",
+ "wasi 0.10.0+wasi-snapshot-preview1",
 "winapi 0.3.9",
 ]

@ -2476,16 +2368,6 @@ dependencies = [
 "valuable",
 ]

-[[package]]
-name = "tracing-error"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d686ec1c0f384b1277f097b2f279a2ecc11afe8c133c1aabf036a27cb4cd206e"
-dependencies = [
- "tracing",
- "tracing-subscriber",
-]
-
 [[package]]
 name = "tracing-log"
 version = "0.1.3"
@ -2529,9 +2411,9 @@ checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987"

 [[package]]
 name = "ubyte"
-version = "0.10.1"
+version = "0.10.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "42756bb9e708855de2f8a98195643dff31a97f0485d90d8467b39dc24be9e8fe"
+checksum = "a58e29f263341a29bb79e14ad7fda5f63b1c7e48929bad4c685d7876b1d04e94"
 dependencies = [
 "serde",
 ]
@ -2544,9 +2426,9 @@ checksum = "56dee185309b50d1f11bfedef0fe6d036842e3fb77413abef29f8f8d1c5d4c1c"

 [[package]]
 name = "uncased"
-version = "0.9.6"
+version = "0.9.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5baeed7327e25054889b9bd4f975f32e5f4c5d434042d59ab6cd4142c0a76ed0"
+checksum = "09b01702b0fd0b3fadcf98e098780badda8742d4f4a7676615cad90e8ac73622"
 dependencies = [
 "serde",
 "version_check",
@ -2608,6 +2490,12 @@ version = "0.3.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992"

+[[package]]
+name = "unicode-ident"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d22af068fba1eb5edcb4aea19d382b2a3deb4c8f9d475c589b6ada9e0fd493ee"
+
 [[package]]
 name = "unicode-normalization"
 version = "0.1.19"
@ -2663,6 +2551,15 @@ version = "0.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"

+[[package]]
+name = "wait-timeout"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "walkdir"
 version = "2.3.2"
@ -2686,9 +2583,9 @@ dependencies = [

 [[package]]
 name = "wasi"
-version = "0.10.2+wasi-snapshot-preview1"
+version = "0.10.0+wasi-snapshot-preview1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"
+checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f"

 [[package]]
 name = "wasi"
@ -2882,3 +2779,16 @@ name = "yansi"
 version = "0.5.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec"
+
+[[package]]
+name = "youtube_dl"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e3e3d041033acf677f28d7d79dc1f9207dfadf86ec05b82c5492126254d90a5d"
+dependencies = [
+ "log",
+ "serde",
+ "serde_json",
+ "tokio",
+ "wait-timeout",
+]
--- a/Cargo.toml
+++ b/Cargo.toml
@ -1,6 +1,6 @@
 [package]
 name = "podbringer"
-version = "0.1.0"
+version = "0.2.0"
 authors = ["Paul van Tilburg <paul@luon.net>"]
 edition = "2021"
 description = "Web service that provides podcasts for services that don't offer them (anymore)"
@ -8,15 +8,15 @@ readme = "README.md"
 license = "MIT"

 [dependencies]
-cached = "0.34.0"
+cached = { version = "0.34.0", features = ["async"] }
 chrono = { version = "0.4.19", features = ["serde"] }
-color-eyre = "0.6.1"
 reqwest = { version = "0.11.10", features = ["json"] }
 rocket = { version = "0.5.0-rc.2", features = ["json"] }
 rocket_dyn_templates = { version = "0.1.0-rc.2", features = ["tera"] }
 rss = "2.0.1"
-tempfile = "3"
-tokio = { version = "1.6.1", features = ["process"] }
+thiserror = "1.0.31"
+url = "2.2.2"
+youtube_dl = { version = "0.7.0", features = ["tokio"] }

 [package.metadata.deb]
 maintainer = "Paul van Tilburg <paul@luon.net>"
@ -26,7 +26,7 @@ extended-description = """\
 Podbringer is a web service that provides podcasts for services that don't
 offer them (anymore). It provides a way to get the RSS feed for your podcast
 client and it facilites the downloads of the pods (enclosures).
- 
+
 It currently only supports [Mixcloud](https://mixcloud.com).
 Other back-ends might be added in the future.
 """
--- a/README.md
+++ b/README.md
@ -55,6 +55,15 @@ need to use for Podbringer is comprised of the following parts:
   The Podbringer location URL          Service  User @ service
 ```

+### Feed item limit
+
+To prevent feeds with a very large number of items, any feed that is returned
+contains at most 50 items by default. If you want to have more (or less) items,
+provide the limit in the URL by setting the `limit` parameter.
+
+For example, to get up until 1000 items the URL becomes:
+`https://my.domain.tld/podbringer/feed/mixcloud/myfavouriteband?limit=1000`
+
 ## License

 Podbringer is licensed under the MIT license (see the `LICENSE` file or
--- a/src/lib.rs
+++ b/src/lib.rs
@ -1,9 +1,11 @@
 #![doc = include_str!("../README.md")]
 #![warn(
    clippy::all,
+    missing_copy_implementations,
    missing_debug_implementations,
    rust_2018_idioms,
-    rustdoc::broken_intra_doc_links
+    rustdoc::broken_intra_doc_links,
+    trivial_numeric_casts
 )]
 #![deny(missing_docs)]

@ -12,9 +14,10 @@ use std::path::PathBuf;
 use chrono::{DateTime, NaiveDateTime, Utc};
 use rocket::fairing::AdHoc;
 use rocket::http::uri::Absolute;
+use rocket::http::Status;
 use rocket::response::Redirect;
 use rocket::serde::{Deserialize, Serialize};
-use rocket::{get, routes, uri, Build, Responder, Rocket, State};
+use rocket::{get, routes, uri, Build, Request, Responder, Rocket, State};
 use rocket_dyn_templates::{context, Template};
 use rss::extension::itunes::{
    ITunesCategoryBuilder, ITunesChannelExtensionBuilder, ITunesItemExtensionBuilder,
@ -25,6 +28,42 @@ use rss::{

 pub(crate) mod mixcloud;

+/// The possible errors that can occur.
+#[derive(Debug, thiserror::Error)]
+pub(crate) enum Error {
+    #[error("IO error: {0}")]
+    Io(#[from] std::io::Error),
+
+    #[error("No redirect URL found")]
+    NoRedirectUrlFound,
+
+    #[error("HTTP error: {0}")]
+    Request(#[from] reqwest::Error),
+
+    #[error("Unknown supported back-end: {0}")]
+    UnsupportedBackend(String),
+
+    #[error("URL parse error: {0}")]
+    UrlParse(#[from] url::ParseError),
+
+    #[error("Youtube_dl failed: {0}")]
+    YoutubeDl(#[from] youtube_dl::Error),
+}
+
+impl<'r, 'o: 'r> rocket::response::Responder<'r, 'o> for Error {
+    fn respond_to(self, _request: &'r Request<'_>) -> rocket::response::Result<'o> {
+        eprintln!("💥 Encountered error: {}", self);
+
+        match self {
+            Error::NoRedirectUrlFound => Err(Status::NotFound),
+            _ => Err(Status::InternalServerError),
+        }
+    }
+}
+
+/// Result type that defaults to [`Error`] as the default error type.
+pub(crate) type Result<T, E = Error> = std::result::Result<T, E>;
+
 /// The extra application specific configuration.
 #[derive(Debug, Deserialize, Serialize)]
 #[serde(crate = "rocket::serde")]
@ -39,24 +78,31 @@ pub(crate) struct Config {
 #[response(content_type = "application/xml")]
 struct RssFeed(String);

-/// Retrieves a download using youtube-dl and redirection.
+/// Retrieves a download by redirecting to the URL resolved by the selected back-end.
 #[get("/download/<backend>/<file..>")]
-pub(crate) async fn download(file: PathBuf, backend: &str) -> Option<Redirect> {
+pub(crate) async fn download(file: PathBuf, backend: &str) -> Result<Redirect> {
    match backend {
        "mixcloud" => {
            let key = format!("/{}/", file.with_extension("").to_string_lossy());

            mixcloud::redirect_url(&key).await.map(Redirect::to)
        }
-        _ => None,
+        _ => Err(Error::UnsupportedBackend(backend.to_string())),
    }
 }

-/// Handler for retrieving the RSS feed of an Mixcloud user.
-#[get("/feed/<backend>/<username>")]
-async fn feed(backend: &str, username: &str, config: &State<Config>) -> Option<RssFeed> {
-    let user = mixcloud::get_user(username).await?;
-    let cloudcasts = mixcloud::get_cloudcasts(username).await?;
+/// Handler for retrieving the RSS feed of user on a certain back-end.
+///
+/// The limit parameter determines the maximum of items that can be in the feed.
+#[get("/feed/<backend>/<username>?<limit>")]
+async fn feed(
+    backend: &str,
+    username: &str,
+    limit: Option<usize>,
+    config: &State<Config>,
+) -> Result<RssFeed> {
+    let user = mixcloud::user(username).await?;
+    let cloudcasts = mixcloud::cloudcasts(username, limit).await?;
    let mut last_build = DateTime::<Utc>::from_utc(NaiveDateTime::from_timestamp(0, 0), Utc);

    let category = CategoryBuilder::default()
@ -156,7 +202,7 @@ async fn feed(backend: &str, username: &str, config: &State<Config>) -> Option<R
        .build();
    let feed = RssFeed(channel.to_string());

-    Some(feed)
+    Ok(feed)
 }

 /// Returns a simple index page that explains the usage.
--- a/src/main.rs
+++ b/src/main.rs
@ -1,19 +1,17 @@
 #![doc = include_str!("../README.md")]
 #![warn(
    clippy::all,
+    missing_copy_implementations,
    missing_debug_implementations,
    rust_2018_idioms,
-    rustdoc::broken_intra_doc_links
+    rustdoc::broken_intra_doc_links,
+    trivial_numeric_casts
 )]
 #![deny(missing_docs)]

-use color_eyre::Result;
-
 /// Sets up and launches Rocket.
 #[rocket::main]
-async fn main() -> Result<()> {
-    color_eyre::install()?;
-
+async fn main() -> Result<(), rocket::Error> {
    let rocket = podbringer::setup();
    let _ = rocket.ignite().await?.launch().await?;

--- a/src/mixcloud.rs
+++ b/src/mixcloud.rs
@ -3,15 +3,16 @@
 //! It uses the Mixcloud API to retrieve the feed (user) and items (cloudcasts)).
 //! See also: <https://www.mixcloud.com/developers/>

-use std::process::Stdio;
-
+use cached::proc_macro::cached;
 use chrono::{DateTime, Utc};
 use reqwest::Url;
 use rocket::serde::Deserialize;
-use tokio::process::Command;
+use youtube_dl::{YoutubeDl, YoutubeDlOutput};

-/// A Mixcloud user.
-#[derive(Debug, Deserialize)]
+use super::{Error, Result};
+
+/// A Mixcloud user (response).
+#[derive(Clone, Debug, Deserialize)]
 #[serde(crate = "rocket::serde")]
 pub(crate) struct User {
    /// The name of the user.
@ -28,23 +29,35 @@ pub(crate) struct User {
 }

 /// A collection of different sizes/variants of a picture.
-#[derive(Debug, Deserialize)]
+#[derive(Clone, Debug, Deserialize)]
 #[serde(crate = "rocket::serde")]
 pub(crate) struct Pictures {
    /// The large picture of the user.
    pub(crate) large: String,
 }

-/// The Mixcloud cloudcasts container.
-#[derive(Debug, Deserialize)]
+/// The Mixcloud cloudcasts response.
+#[derive(Clone, Debug, Deserialize)]
 #[serde(crate = "rocket::serde")]
-pub(crate) struct CloudcastData {
-    /// The contained cloudcasts.
-    data: Vec<Cloudcast>,
+pub(crate) struct CloudcastsResponse {
+    /// The contained cloudcast items.
+    #[serde(rename = "data")]
+    items: Vec<Cloudcast>,
+
+    /// The paging information.
+    paging: CloudcastsPaging,
+}
+
+/// The Mixcloud paging info.
+#[derive(Clone, Debug, Deserialize)]
+#[serde(crate = "rocket::serde")]
+pub(crate) struct CloudcastsPaging {
+    /// The API URL of the next page.
+    next: Option<String>,
 }

 /// A Mixcloud cloudcast.
-#[derive(Debug, Deserialize)]
+#[derive(Clone, Debug, Deserialize)]
 #[serde(crate = "rocket::serde")]
 pub(crate) struct Cloudcast {
    /// The key of the cloudcast.
@ -73,7 +86,7 @@ pub(crate) struct Cloudcast {
 }

 /// A Mixcloud cloudcast tag.
-#[derive(Debug, Deserialize)]
+#[derive(Clone, Debug, Deserialize)]
 #[serde(crate = "rocket::serde")]
 pub(crate) struct Tag {
    /// The name of the tag.
@ -89,14 +102,17 @@ const API_BASE_URL: &str = "https://api.mixcloud.com";
 /// The base URL for downloading Mixcloud files.
 const FILES_BASE_URL: &str = "https://www.mixcloud.com";

-/// The default bitrate used by
+/// The default bitrate used by Mixcloud.
 const DEFAULT_BITRATE: u32 = 64 * 1024;

-/// The default file (MIME) type.
+/// The default file (MIME) type used by Mixcloud.
 const DEFAULT_FILE_TYPE: &str = "audio/mpeg";

+/// The default page size.
+const DEFAULT_PAGE_SIZE: usize = 50;
+
 /// Returns the default file type used by Mixcloud.
-pub(crate) fn default_file_type() -> &'static str {
+pub(crate) const fn default_file_type() -> &'static str {
    DEFAULT_FILE_TYPE
 }

@ -108,52 +124,112 @@ pub(crate) fn estimated_file_size(duration: u32) -> u32 {
 }

 /// Retrieves the user data using the Mixcloud API.
-pub(crate) async fn get_user(username: &str) -> Option<User> {
-    let mut url = Url::parse(API_BASE_URL).unwrap();
+pub(crate) async fn user(username: &str) -> Result<User> {
+    let mut url = Url::parse(API_BASE_URL).expect("URL can always be parsed");
    url.set_path(username);

    println!("⏬ Retrieving user {username} from {url}...");
-    let response = reqwest::get(url).await.ok()?;
-    let user = match response.error_for_status() {
-        Ok(res) => res.json().await.ok()?,
-        Err(_err) => return None,
-    };
-
-    Some(user)
+    fetch_user(url).await
 }

-/// Retrieves the cloudcasts of the user using the Mixcloud API.
-pub(crate) async fn get_cloudcasts(username: &str) -> Option<Vec<Cloudcast>> {
-    let mut url = Url::parse(API_BASE_URL).unwrap();
+/// Fetches the user from the URL.
+#[cached(
+    key = "String",
+    convert = r#"{ url.to_string() }"#,
+    time = 86400,
+    result = true
+)]
+///
+/// If the result is [`Ok`], the user will be cached for 24 hours for the given username.
+async fn fetch_user(url: Url) -> Result<User> {
+    let response = reqwest::get(url).await?.error_for_status()?;
+    let user = response.json().await?;
+
+    Ok(user)
+}
+
+/// Retrieves the cloudcasts data of the user using the Mixcloud API.
+pub(crate) async fn cloudcasts(username: &str, limit: Option<usize>) -> Result<Vec<Cloudcast>> {
+    let mut limit = limit.unwrap_or(DEFAULT_PAGE_SIZE);
+    let mut offset = 0;
+    let mut url = Url::parse(API_BASE_URL).expect("URL can always be parsed");
    url.set_path(&format!("{username}/cloudcasts/"));
-
    println!("⏬ Retrieving cloudcasts of user {username} from {url}...");
-    let response = reqwest::get(url).await.ok()?;
-    let cloudcasts: CloudcastData = match response.error_for_status() {
-        Ok(res) => res.json().await.ok()?,
-        Err(_err) => return None,
-    };

-    Some(cloudcasts.data)
+    set_paging_query(&mut url, limit, offset);
+    let mut cloudcasts = Vec::with_capacity(50); // The initial limit
+    loop {
+        let cloudcasts_res: CloudcastsResponse = fetch_cloudcasts(url).await?;
+        let count = cloudcasts_res.items.len();
+        cloudcasts.extend(cloudcasts_res.items);
+
+        // Continue onto the next URL in the paging, if there is one.
+        limit = limit.saturating_sub(count);
+        offset += count;
+        match cloudcasts_res.paging.next {
+            Some(next_url) => {
+                url = Url::parse(&next_url)?;
+                set_paging_query(&mut url, limit, offset);
+            }
+            None => break,
+        }
+
+        // We have reached the limit.
+        if limit == 0 {
+            break;
+        }
+    }
+
+    Ok(cloudcasts)
+}
+
+/// Fetches cloudcasts from the URL.
+///
+/// If the result is [`Ok`], the cloudcasts will be cached for 24 hours for the given username.
+#[cached(
+    key = "String",
+    convert = r#"{ url.to_string() }"#,
+    time = 86400,
+    result = true
+)]
+async fn fetch_cloudcasts(url: Url) -> Result<CloudcastsResponse> {
+    let response = reqwest::get(url).await?.error_for_status()?;
+    let cloudcasts_res = response.json().await?;
+
+    Ok(cloudcasts_res)
+}
+
+/// Set paging query pairs for URL.
+///
+/// The limit is capped to the default page size. Another request will be necessary to retrieve
+/// more.
+fn set_paging_query(url: &mut Url, limit: usize, offset: usize) {
+    url.query_pairs_mut()
+        .clear()
+        .append_pair(
+            "limit",
+            &format!("{}", std::cmp::min(limit, DEFAULT_PAGE_SIZE)),
+        )
+        .append_pair("offset", &format!("{}", offset));
 }

 /// Retrieves the redirect URL for the provided Mixcloud cloudcast key.
-pub(crate) async fn redirect_url(key: &str) -> Option<String> {
-    let mut cmd = Command::new("youtube-dl");
-    cmd.args(&["--format", "http"])
-        .arg("--get-url")
-        .arg(&format!("{FILES_BASE_URL}{key}"))
-        .stdout(Stdio::piped());
+#[cached(
+    key = "String",
+    convert = r#"{ download_key.to_owned() }"#,
+    time = 86400,
+    result = true
+)]
+pub(crate) async fn redirect_url(download_key: &str) -> Result<String> {
+    let mut url = Url::parse(FILES_BASE_URL).expect("URL can always be parsed");
+    url.set_path(download_key);

-    let output = cmd.output().await.ok()?;
-    if output.status.success() {
-        let direct_url = String::from_utf8_lossy(&output.stdout)
-            .trim_end()
-            .to_owned();
-        println!("🌍 Determined direct URL for {key}: {direct_url}...");
+    println!("🌍 Determining direct URL for {download_key}...");
+    let output = YoutubeDl::new(url).run_async().await?;

-        Some(direct_url)
+    if let YoutubeDlOutput::SingleVideo(yt_item) = output {
+        yt_item.url.ok_or(Error::NoRedirectUrlFound)
    } else {
-        None
+        Err(Error::NoRedirectUrlFound)
    }
 }
Author	SHA1	Message	Date
Paul van Tilburg	61db7c4b76	Update the changelog	2022-05-27 22:59:29 +02:00
Paul van Tilburg	8c0bfd766a	Bump the version to 0.2.0	2022-05-27 22:59:29 +02:00
Paul van Tilburg	27a40b1a91	Increase TTL for redirect URI to 24 hours	2022-05-27 22:59:28 +02:00
Paul van Tilburg	dafcdc009b	Merge pull request 'Implement paging' (#9 ) from 2-implement-paging into main Reviewed-on: #9	2022-05-27 22:50:51 +02:00
Paul van Tilburg	0701088fbc	Update the documentation	2022-05-27 22:47:52 +02:00
Paul van Tilburg	c13ce71c69	Add feed item limit support * The feed item limit defaults to the default page size (50) if not provided * Move caching from response to URL fetch results; add helper functions * Add a helper function to set the paging query of an URL * Modify paging so we don't retrieve more than the feed item limit	2022-05-27 22:47:52 +02:00
Paul van Tilburg	78fc93fedf	Retrieve all pages by following the next URL * Derserialize the paging information * Parse each next URL; handle URL parse errors * Use a default page size of 50; pass offset 0 to count by item index	2022-05-27 22:47:52 +02:00
Paul van Tilburg	09ee0b9ba9	Make default_file_type functions const	2022-05-27 22:39:53 +02:00
Paul van Tilburg	10bbd9b495	Cargo update	2022-05-26 22:12:31 +02:00
Paul van Tilburg	cde2a34e91	Fix documentation	2022-05-26 22:12:01 +02:00
Paul van Tilburg	11c78a6cc8	Drop the get_ prefix for functions	2022-05-26 21:25:37 +02:00
Paul van Tilburg	11b32acfb4	Use an URL parser for the URL passed to youtube-dl	2022-05-26 21:23:38 +02:00
Paul van Tilburg	451c07a09e	Implement caching (closes: #3 ) * Enable the `async` feature for the `cached` crate * Make the types that we cache implement `Clone` * Rename the argument of mixcloud::redirect_url` because of this issue: https://github.com/jaemk/cached/issues/114	2022-05-26 21:20:10 +02:00
Paul van Tilburg	b4c0188fba	Drop some unnecessary bloat/unused crates	2022-05-26 20:40:18 +02:00
Paul van Tilburg	3ec1879932	Replace own youtube-dl impl by youtube_dl crate (refs: #8 ) * Drop the depend on the `tokio` crate, because we don't need to run our own processes anymore. * Remove unnecessary error variant for command failure	2022-05-26 20:38:51 +02:00
Paul van Tilburg	a67df934bf	Enable some more lints; fix clippy issues	2022-05-26 20:03:44 +02:00
Paul van Tilburg	b53365a293	Implement proper error logging and handling (closes: #6 ) * Use the `thiserror` crate to make our own error type * Implement Rocket's `Responder` type for the error type * Adjust all methods to use the error type * Small documentation tweaks	2022-05-26 19:57:24 +02:00