Implement retrieving and caching maps

This commit is contained in:
Paul van Tilburg 2022-02-12 21:08:13 +01:00
parent d058ab4448
commit 72fe9577bd
Signed by: paul
GPG Key ID: C6DE073EDA9EEC4D
4 changed files with 524 additions and 14 deletions

389
Cargo.lock generated
View File

@ -17,6 +17,12 @@ version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
[[package]]
name = "adler32"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234"
[[package]]
name = "ansi_term"
version = "0.12.1"
@ -103,7 +109,7 @@ dependencies = [
"cc",
"cfg-if 1.0.0",
"libc",
"miniz_oxide",
"miniz_oxide 0.4.4",
"object",
"rustc-demangle",
]
@ -126,6 +132,12 @@ version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "383d29d513d8764dcdc42ea295d979eb99c3c9f00607b3692cf68a431f7dca72"
[[package]]
name = "bit_field"
version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dcb6dd1c2376d2e096796e234a70e17e94cc2d5d54ff8ce42b28cef1d0d359a4"
[[package]]
name = "bitflags"
version = "1.3.2"
@ -138,6 +150,18 @@ version = "3.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4a45a46ab1f2412e53d3a0ade76ffad2025804294569aae387231a0cd6e0899"
[[package]]
name = "bytemuck"
version = "1.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "439989e6b8c38d1b6570a384ef1e49c8848128f5a97f3914baef02920842712f"
[[package]]
name = "byteorder"
version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
[[package]]
name = "bytes"
version = "0.5.6"
@ -209,6 +233,12 @@ dependencies = [
"tracing-error",
]
[[package]]
name = "color_quant"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b"
[[package]]
name = "const_fn"
version = "0.4.9"
@ -242,6 +272,68 @@ version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc"
[[package]]
name = "crc32fast"
version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d"
dependencies = [
"cfg-if 1.0.0",
]
[[package]]
name = "crossbeam-channel"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e54ea8bc3fb1ee042f5aace6e3c6e025d3874866da222930f70ce62aceba0bfa"
dependencies = [
"cfg-if 1.0.0",
"crossbeam-utils",
]
[[package]]
name = "crossbeam-deque"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e"
dependencies = [
"cfg-if 1.0.0",
"crossbeam-epoch",
"crossbeam-utils",
]
[[package]]
name = "crossbeam-epoch"
version = "0.9.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c00d6d2ea26e8b151d99093005cb442fb9a37aeaca582a03ec70946f49ab5ed9"
dependencies = [
"cfg-if 1.0.0",
"crossbeam-utils",
"lazy_static",
"memoffset",
"scopeguard",
]
[[package]]
name = "crossbeam-utils"
version = "0.8.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b5e5bed1f1c269533fa816a0a5492b3545209a205ca1a54842be180eb63a16a6"
dependencies = [
"cfg-if 1.0.0",
"lazy_static",
]
[[package]]
name = "deflate"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c86f7e25f518f4b81808a2cf1c50996a61f5c2eb394b2393bd87f2a4780a432f"
dependencies = [
"adler32",
]
[[package]]
name = "devise"
version = "0.3.1"
@ -296,6 +388,22 @@ dependencies = [
"cfg-if 1.0.0",
]
[[package]]
name = "exr"
version = "1.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4badb9489a465cb2c555af1f00f0bfd8cecd6fc12ac11da9d5b40c5dd5f0200"
dependencies = [
"bit_field",
"deflate",
"flume",
"half",
"inflate",
"lebe",
"smallvec",
"threadpool",
]
[[package]]
name = "eyre"
version = "0.6.6"
@ -329,6 +437,31 @@ dependencies = [
"version_check",
]
[[package]]
name = "flate2"
version = "1.0.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e6988e897c1c9c485f43b47a529cef42fde0547f9d8d41a7062518f1d8fc53f"
dependencies = [
"cfg-if 1.0.0",
"crc32fast",
"libc",
"miniz_oxide 0.4.4",
]
[[package]]
name = "flume"
version = "0.10.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d04dafd11240188e146b6f6476a898004cace3be31d4ec5e08e216bf4947ac0"
dependencies = [
"futures-core",
"futures-sink",
"nanorand",
"pin-project",
"spin",
]
[[package]]
name = "fnv"
version = "1.0.7"
@ -498,7 +631,7 @@ dependencies = [
"geo-types",
"hyper 0.13.10",
"num-traits",
"reqwest",
"reqwest 0.10.10",
"serde",
"serde_json",
"thiserror",
@ -511,8 +644,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "418d37c8b1d42553c93648be529cb70f920d3baf8ef469b74b9638df426e0b4c"
dependencies = [
"cfg-if 1.0.0",
"js-sys",
"libc",
"wasi",
"wasm-bindgen",
]
[[package]]
name = "gif"
version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3a7187e78088aead22ceedeee99779455b23fc231fe13ec443f99bb71694e5b"
dependencies = [
"color_quant",
"weezl",
]
[[package]]
@ -566,6 +711,12 @@ dependencies = [
"tracing",
]
[[package]]
name = "half"
version = "1.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7"
[[package]]
name = "hashbrown"
version = "0.11.2"
@ -692,6 +843,19 @@ dependencies = [
"tokio-tls",
]
[[package]]
name = "hyper-tls"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
dependencies = [
"bytes 1.1.0",
"hyper 0.14.17",
"native-tls",
"tokio 1.16.1",
"tokio-native-tls",
]
[[package]]
name = "idna"
version = "0.2.3"
@ -703,6 +867,26 @@ dependencies = [
"unicode-normalization",
]
[[package]]
name = "image"
version = "0.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e94ac3d41f882c624a82d7945952032388488681f45f9d4077999a6c85688d61"
dependencies = [
"bytemuck",
"byteorder",
"color_quant",
"exr",
"gif",
"jpeg-decoder 0.2.1",
"num-iter",
"num-rational",
"num-traits",
"png",
"scoped_threadpool",
"tiff",
]
[[package]]
name = "indenter"
version = "0.3.3"
@ -720,6 +904,15 @@ dependencies = [
"serde",
]
[[package]]
name = "inflate"
version = "0.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1cdb29978cc5797bd8dcc8e5bf7de604891df2a8dc576973d71a281e916db2ff"
dependencies = [
"adler32",
]
[[package]]
name = "inlinable_string"
version = "0.1.15"
@ -762,6 +955,21 @@ version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35"
[[package]]
name = "jpeg-decoder"
version = "0.1.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "229d53d58899083193af11e15917b5640cd40b29ff475a1fe4ef725deb02d0f2"
[[package]]
name = "jpeg-decoder"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fbcf0244f6597be39ab8d9203f574cafb529ae8c698afa2182f7b3c3205a4a9c"
dependencies = [
"rayon",
]
[[package]]
name = "js-sys"
version = "0.3.56"
@ -787,6 +995,12 @@ version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "lebe"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7efd1d698db0759e6ef11a7cd44407407399a910c774dd804c64c032da7826ff"
[[package]]
name = "libc"
version = "0.2.117"
@ -847,6 +1061,15 @@ version = "2.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a"
[[package]]
name = "memoffset"
version = "0.6.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce"
dependencies = [
"autocfg",
]
[[package]]
name = "mime"
version = "0.3.16"
@ -873,6 +1096,15 @@ dependencies = [
"autocfg",
]
[[package]]
name = "miniz_oxide"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2b29bd4bc3f33391105ebee3589c19197c4271e3e5a9ec9bfe8127eeff8f082"
dependencies = [
"adler",
]
[[package]]
name = "mio"
version = "0.6.23"
@ -946,6 +1178,15 @@ dependencies = [
"version_check",
]
[[package]]
name = "nanorand"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "729eb334247daa1803e0a094d0a5c55711b85571179f5ec6e53eccfdf7008958"
dependencies = [
"getrandom",
]
[[package]]
name = "native-tls"
version = "0.2.8"
@ -994,6 +1235,28 @@ dependencies = [
"num-traits",
]
[[package]]
name = "num-iter"
version = "0.1.42"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b2021c8337a54d21aca0d59a92577a029af9431cb59b909b03252b9c164fad59"
dependencies = [
"autocfg",
"num-integer",
"num-traits",
]
[[package]]
name = "num-rational"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d41702bd167c2df5520b384281bc111a4b5efcf7fbc4c9c222c815b07e0a6a6a"
dependencies = [
"autocfg",
"num-integer",
"num-traits",
]
[[package]]
name = "num-traits"
version = "0.2.14"
@ -1165,6 +1428,18 @@ version = "0.3.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "58893f751c9b0412871a09abd62ecd2a00298c6c83befa223ef98c52aef40cbe"
[[package]]
name = "png"
version = "0.17.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e8f1882177b17c98ec33a51f5910ecbf4db92ca0def706781a1f8d0c661f393"
dependencies = [
"bitflags",
"crc32fast",
"deflate",
"miniz_oxide 0.5.1",
]
[[package]]
name = "ppv-lite86"
version = "0.2.16"
@ -1248,6 +1523,31 @@ dependencies = [
"rand_core",
]
[[package]]
name = "rayon"
version = "1.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c06aca804d41dbc8ba42dfd964f0d01334eceb64314b9ecf7c5fad5188a06d90"
dependencies = [
"autocfg",
"crossbeam-deque",
"either",
"rayon-core",
]
[[package]]
name = "rayon-core"
version = "1.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d78120e2c850279833f1dd3582f730c4ab53ed95aeaaaa862a2a5c71b1656d8e"
dependencies = [
"crossbeam-channel",
"crossbeam-deque",
"crossbeam-utils",
"lazy_static",
"num_cpus",
]
[[package]]
name = "redox_syscall"
version = "0.2.10"
@ -1324,7 +1624,7 @@ dependencies = [
"http",
"http-body 0.3.1",
"hyper 0.13.10",
"hyper-tls",
"hyper-tls 0.4.3",
"ipnet",
"js-sys",
"lazy_static",
@ -1346,6 +1646,42 @@ dependencies = [
"winreg",
]
[[package]]
name = "reqwest"
version = "0.11.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87f242f1488a539a79bac6dbe7c8609ae43b7914b7736210f239a37cccb32525"
dependencies = [
"base64",
"bytes 1.1.0",
"encoding_rs",
"futures-core",
"futures-util",
"h2 0.3.11",
"http",
"http-body 0.4.4",
"hyper 0.14.17",
"hyper-tls 0.5.0",
"ipnet",
"js-sys",
"lazy_static",
"log",
"mime",
"native-tls",
"percent-encoding",
"pin-project-lite 0.2.8",
"serde",
"serde_json",
"serde_urlencoded",
"tokio 1.16.1",
"tokio-native-tls",
"url",
"wasm-bindgen",
"wasm-bindgen-futures",
"web-sys",
"winreg",
]
[[package]]
name = "rocket"
version = "0.5.0-rc.1"
@ -1472,6 +1808,12 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea6a9290e3c9cf0f18145ef7ffa62d68ee0bf5fcd651017e586dc7fd5da448c2"
[[package]]
name = "scoped_threadpool"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d51f5df5af43ab3f1360b429fa5e0152ac5ce8c0bd6485cae490332e96846a8"
[[package]]
name = "scopeguard"
version = "1.1.0"
@ -1599,7 +1941,9 @@ dependencies = [
"chrono",
"color-eyre",
"geocoding",
"image",
"once_cell",
"reqwest 0.11.9",
"rocket",
]
@ -1641,6 +1985,9 @@ name = "spin"
version = "0.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "511254be0c5bcf062b019a6c89c01a664aa359ded62f78aa72c6fc137c0590e5"
dependencies = [
"lock_api",
]
[[package]]
name = "stable-pattern"
@ -1772,6 +2119,26 @@ dependencies = [
"once_cell",
]
[[package]]
name = "threadpool"
version = "1.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa"
dependencies = [
"num_cpus",
]
[[package]]
name = "tiff"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0247608e998cb6ce39dfc8f4a16c50361ce71e5b52e6d24ea1227ea8ea8ee0b2"
dependencies = [
"flate2",
"jpeg-decoder 0.1.22",
"weezl",
]
[[package]]
name = "time"
version = "0.1.44"
@ -1883,6 +2250,16 @@ dependencies = [
"syn",
]
[[package]]
name = "tokio-native-tls"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b"
dependencies = [
"native-tls",
"tokio 1.16.1",
]
[[package]]
name = "tokio-stream"
version = "0.1.8"
@ -2220,6 +2597,12 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "weezl"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d8b77fdfd5a253be4ab714e4ffa3c49caf146b4de743e97510c0656cf90f1e8e"
[[package]]
name = "winapi"
version = "0.2.8"

View File

@ -7,5 +7,7 @@ edition = "2021"
chrono = "0.4.19"
color-eyre = "0.5.6"
geocoding = "0.3.1"
image = "0.24.0"
once_cell = "1.9.0"
reqwest = "0.11.9"
rocket = { version = "0.5.0-rc.1", features = ["json"] }

View File

@ -24,7 +24,8 @@ use self::maps::Maps;
mod maps;
static MAPS: Lazy<Mutex<Maps>> = Lazy::new(|| Mutex::new(Maps::default()));
/// Global maps cache refreshed by a separate task.
static MAPS: Lazy<Mutex<Maps>> = Lazy::new(|| Mutex::new(Maps::new()));
/// The current for a specific location.
///
@ -198,7 +199,7 @@ async fn main() -> Result<()> {
.await?;
let shutdown = rocket.shutdown();
let maps_updater = tokio::spawn(Maps::run());
let maps_updater = tokio::spawn(maps::run());
select! {
result = rocket.launch() => {

View File

@ -1,21 +1,145 @@
//! Maps handling.
//! Maps retrieval and caching.
//!
//! This module provides a task that keeps maps up-to-date using a maps-specific refresh interval.
//! It stores all the maps as [`DynamicImage`]s in memory.
use rocket::tokio::time::{sleep, Duration};
use chrono::DurationRound;
use image::DynamicImage;
use rocket::tokio::time::{sleep, Duration, Instant};
use crate::MAPS;
/// The interval between map refreshes (in seconds).
const SLEEP_INTERVAL: u64 = 60;
const SLEEP_INTERVAL: Duration = Duration::from_secs(60);
#[derive(Debug, Default)]
pub(crate) struct Maps;
/// The base URL for retrieving the precipitation map.
const PRECIPITATION_BASE_URL: &str =
"https://cluster.api.meteoplaza.com/v3/nowcast/tiles/radarnl-forecast";
/// The interval for retrieving precipitation maps.
const PRECIPITATION_INTERVAL: Duration = Duration::from_secs(300);
/// The base URL for retrieving the pollen maps.
const POLLEN_BASE_URL: &str =
"https://image.buienradar.nl/2.0/image/sprite/WeatherMapPollenRadarHourlyNL\
?height=988&width=820&extension=png&renderBackground=False&renderBranding=False\
&renderText=False&history=0&forecast=24&skip=0&timestamp=";
/// The interval for retrieving pollen maps.
const POLLEN_INTERVAL: Duration = Duration::from_secs(600);
/// The base URL for retrieving the UV index maps.
const UVI_BASE_URL: &str = "https://image.buienradar.nl/2.0/image/sprite/WeatherMapUVIndexNL\
?extension=png&width=820&height=988&renderText=False&renderBranding=False\
&renderBackground=False&history=0&forecast=5&skip=0&timestamp=";
/// The interval for retrieving UV index maps.
const UVI_INTERVAL: Duration = Duration::from_secs(3600);
/// Runs a loop that keeps refreshing the maps when necessary.
pub(crate) async fn run() -> ! {
loop {
let mut maps = MAPS.lock().await;
println!("🕔 Refreshing the maps (if necessary)...");
maps.refresh_precipitation().await;
maps.refresh_pollen().await;
maps.refresh_uvi().await;
sleep(SLEEP_INTERVAL).await;
}
}
/// Container type for all in-memory cached maps.
#[derive(Debug)]
pub(crate) struct Maps {
/// The pollen maps (from Buienradar).
pub(crate) pollen: Option<DynamicImage>,
/// The timestamp the pollen maps were last refreshed.
pollen_stamp: Instant,
/// The precipitation maps (from Weerplaza).
// TODO: Make one large image instead of using an array? This is already the case for the
// other maps.
pub(crate) precipitation: [Option<DynamicImage>; 24],
/// The timestamp the precipitation maps were last refreshed.
precipitation_stamp: Instant,
/// The UV index maps (from Buienradar).
pub(crate) uvi: Option<DynamicImage>,
/// The timestamp the UV index maps were last refreshed.
uvi_stamp: Instant,
}
impl Maps {
pub(crate) async fn run() -> ! {
loop {
let _maps = MAPS.lock().await;
pub(crate) fn new() -> Self {
let now = Instant::now();
// Because `Option<DynamicImage>` does not implement `Copy`
let precipitation = [(); 24].map(|_| None);
sleep(Duration::from_secs(SLEEP_INTERVAL)).await;
Self {
pollen: None,
pollen_stamp: now,
precipitation,
precipitation_stamp: now,
uvi: None,
uvi_stamp: now,
}
}
async fn refresh_precipitation(&mut self) {
if self.precipitation.iter().any(|map| map.is_none())
|| Instant::now().duration_since(self.precipitation_stamp) > PRECIPITATION_INTERVAL
{
let just_before = (chrono::Utc::now() - chrono::Duration::minutes(10))
// This only fails if timestamps and durations exceed limits!
.duration_trunc(chrono::Duration::minutes(5))
.unwrap();
let timestamp = just_before.format("%Y%m%d%H%M");
for k in 0..24 {
let suffix = format!("{:03}", k * 5);
let url = format!("{PRECIPITATION_BASE_URL}/{timestamp}_{suffix}");
println!("🔽 Refreshing precipitation maps from: {}", url);
self.precipitation[k] = retrieve_image(&url).await;
}
self.precipitation_stamp = Instant::now();
}
}
async fn refresh_pollen(&mut self) {
if self.pollen.is_none()
|| Instant::now().duration_since(self.pollen_stamp) > POLLEN_INTERVAL
{
let timestamp = chrono::Local::now().format("%y%m%d%H%M");
let url = format!("{POLLEN_BASE_URL}{timestamp}");
println!("🔽 Refreshing pollen maps from: {}", url);
self.pollen = retrieve_image(&url).await;
self.pollen_stamp = Instant::now();
}
}
async fn refresh_uvi(&mut self) {
if self.uvi.is_none() || Instant::now().duration_since(self.uvi_stamp) > UVI_INTERVAL {
let timestamp = chrono::Local::now().format("%y%m%d%H%M");
let url = format!("{UVI_BASE_URL}{timestamp}");
println!("🔽 Refreshing UV index maps from: {}", url);
self.uvi = retrieve_image(&url).await;
self.uvi_stamp = Instant::now();
}
}
}
/// Retrieves an image from the provided URL.
async fn retrieve_image(url: &str) -> Option<DynamicImage> {
// TODO: Handle or log errors!
let response = reqwest::get(url).await.ok()?;
let bytes = response.bytes().await.ok()?;
image::load_from_memory(&bytes).ok()
}