From f34e8ba309060a98c236cee35dda5f29cc1f3b31 Mon Sep 17 00:00:00 2001 From: Frank Villaro-Dixon Date: Mon, 15 Apr 2024 23:17:14 +0200 Subject: [PATCH] refactor cache Signed-off-by: Frank Villaro-Dixon --- Cargo.lock | 62 ++++++++++++++++++++++++++++++++++++++ Cargo.toml | 2 ++ Dockerfile | 4 +-- src/dem.rs | 86 ++++++++++++++++++++++++++++++++++++----------------- src/main.rs | 49 ++++++++++++++++++++---------- 5 files changed, 157 insertions(+), 46 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ca7fbca..5f03c8e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -37,6 +37,8 @@ dependencies = [ "serde", "serde_json", "tokio", + "tower", + "tower-http", "tracing", "tracing-subscriber", ] @@ -385,6 +387,12 @@ dependencies = [ "syn", ] +[[package]] +name = "futures-sink" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" + [[package]] name = "futures-task" version = "0.3.30" @@ -522,6 +530,12 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "http-range-header" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ce4ef31cda248bbdb6e6820603b82dfcd9e833db65a43e997a0ccec777d11fe" + [[package]] name = "httparse" version = "1.8.0" @@ -658,6 +672,16 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "mime_guess" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4192263c238a5f0d0c6bfd21f336a313a4ce1c450542449ca191bb657b4642ef" +dependencies = [ + "mime", + "unicase", +] + [[package]] name = "minimal-lexical" version = "0.2.1" @@ -1231,6 +1255,19 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-util" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + [[package]] name = "tower" version = "0.4.13" @@ -1247,6 +1284,31 @@ dependencies = [ "tracing", ] +[[package]] +name = "tower-http" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9cd434a998747dd2c4276bc96ee2e0c7a2eadf3cae88e52be55a05fa9053f5" +dependencies = [ + "bitflags 2.5.0", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "http-range-header", + "httpdate", + "mime", + "mime_guess", + "percent-encoding", + "pin-project-lite", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "tower-layer" version = "0.3.2" diff --git a/Cargo.toml b/Cargo.toml index 57b1a05..19f3926 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,5 +14,7 @@ moka = { version = "0.12.5", features = ["future"] } serde = { version = "1.0.197", features = ["derive"] } serde_json = "1.0.115" tokio = { version = "1.37.0", features = ["full"] } +tower = "0.4.13" +tower-http = { version = "0.5.2", features = ["fs"]} tracing = "0.1.40" tracing-subscriber = "0.3.18" diff --git a/Dockerfile b/Dockerfile index 229880d..6abf7ad 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,7 @@ RUN cargo build --release --bin api-server # We do not need the Rust toolchain to run the binary! FROM debian:bookworm-slim AS runtime -RUN apt update && apt install -y libgdal-dev +RUN apt update && apt install -y gdal-bin WORKDIR /app COPY --from=builder /app/target/release/api-server /usr/local/bin -ENTRYPOINT ["/usr/local/bin/api-server"] \ No newline at end of file +ENTRYPOINT ["/usr/local/bin/api-server"] diff --git a/src/dem.rs b/src/dem.rs index 8410187..6e1d649 100644 --- a/src/dem.rs +++ b/src/dem.rs @@ -1,52 +1,85 @@ use std::sync::Arc; -use std::env; use gdal::errors::GdalError; use gdal::Dataset; -use tracing::{info, debug_span, debug}; +use tracing::{debug, debug_span, info}; use moka::future::Cache; -pub struct MyDataset{pub ds: Dataset} -unsafe impl Send for MyDataset{} -unsafe impl Sync for MyDataset{} +pub struct MyDataset { + pub ds: Dataset, +} +unsafe impl Send for MyDataset {} +unsafe impl Sync for MyDataset {} -pub type DSC = Cache>; -pub fn new_cache(max_elems: u64) -> DSC { - Cache::builder() - // Up to 10,000 entries. - .max_capacity(max_elems) - // Create the cache. - .build() +pub struct DatasetRepository { + cache: Cache>, + basedir: String, } -pub async fn elevation_from_coordinates(dsc: DSC, lat: f64, lon: f64) -> f64 { +unsafe impl Send for DatasetRepository {} +unsafe impl Sync for DatasetRepository {} +impl DatasetRepository { + pub fn new(basedir: String) -> Self { + let c = Cache::builder() + .max_capacity(100) + // Create the cache. + .build(); + + DatasetRepository { cache: c, basedir } + } + + async fn get(&self, filename: String) -> Arc { + let full_filename = format!("{}/{filename}", self.basedir); + + if !self.cache.contains_key(&full_filename) { + info!("Will open {full_filename} because not in cache!"); + let ds = Arc::new(MyDataset { + ds: Dataset::open(full_filename.clone()).unwrap(), + }); + self.cache.insert(full_filename.clone(), ds).await; + } + + match self.cache.get(&full_filename).await { + Some(dataset_arc) => dataset_arc, + None => panic!("foo") + } + + } + +} + +impl Clone for DatasetRepository { + fn clone(&self) -> Self { + Self { + basedir: self.basedir.clone(), + cache: self.cache.clone(), + } + } +} + + +pub async fn elevation_from_coordinates(dr: DatasetRepository, lat: f64, lon: f64) -> f64 { let span = debug_span!("req", lat=%lat, lon=%lon); let _guard = span.enter(); let filename = get_filename_from_latlon(lat, lon); debug!(filename, "filename"); + let ds = &dr.get(filename).await.ds; - if !dsc.contains_key(&filename) { - info!("Will open {filename} because not in cache!"); - let ds = Arc::new(MyDataset{ds:Dataset::open(filename.clone()).unwrap()}); - dsc.insert(filename.clone(), ds).await; - } - - let ds = &dsc.get(&filename).await.unwrap().ds; - let (px, py) = geo_to_pixel(ds, lat, lon).unwrap(); let raster_band = ds.rasterband(1).unwrap(); - let raster_value = raster_band.read_as::((px, py), (1, 1), (1, 1), None).unwrap(); + let raster_value = raster_band + .read_as::((px, py), (1, 1), (1, 1), None) + .unwrap(); raster_value.data[0] } - fn get_filename_from_latlon(lat: f64, lon: f64) -> String { - // Calculate the rounded values for latitude and longitude + // Calculate the rounded values for latitude and longitude let rounded_lat = lat.floor(); let rounded_lon = lon.floor(); @@ -58,16 +91,13 @@ fn get_filename_from_latlon(lat: f64, lon: f64) -> String { let lat_prefix = if rounded_lat >= 0.0 { "N" } else { "S" }; let lon_prefix = if rounded_lon >= 0.0 { "E" } else { "W" }; - - let filename_prefix = env::var("DEM_LOCATION").unwrap_or("/data".to_string()); // Construct the filename let filename = format!( - "{filename_prefix}/Copernicus_DSM_30_{}{}_00_{}{}_00_DEM.tif", + "Copernicus_DSM_30_{}{}_00_{}{}_00_DEM.tif", lat_prefix, lat_deg, lon_prefix, lon_deg ); filename - } fn geo_to_pixel(dataset: &Dataset, lat: f64, lon: f64) -> Result<(isize, isize), GdalError> { diff --git a/src/main.rs b/src/main.rs index f4e9e2e..ae212ac 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,47 +1,64 @@ - mod dem; - -use std::env; -use axum_macros::debug_handler; use axum::{ - extract::{Path, State}, routing::{get}, Router + extract::{Path, State}, + routing::get, + Router, }; -use dem::{DSC, new_cache}; +use axum_macros::debug_handler; +use std::env; + +use tower_http::services::ServeDir; + +use dem::DatasetRepository; use tracing::info; +const DEFAULT_DATA_DIR: &str = "/data"; +const DEFAULT_PORT: &str = "3000"; -#[tokio::main(flavor = "current_thread")] +//#[tokio::main(flavor = "current_thread")] +#[tokio::main] async fn main() { // initialize tracing tracing_subscriber::fmt::init(); - let cache = new_cache(1000); + let config = load_config().unwrap(); + let cache = DatasetRepository::new(config.basedir); + + let serve_dir = ServeDir::new("assets"); // build our application with a route let app = Router::new() .route("/elevation/:lat/:lon", get(get_elevation)) + .nest_service("/", serve_dir) .with_state(cache); - - let port = env::var("HTTP_PORT").unwrap_or("3000".to_string()); - let host = format!("[::]:{port}"); + let host = format!("[::]:{}", config.port); info!("Will start server on {host}"); - let listener = tokio::net::TcpListener::bind(host).await.unwrap(); axum::serve(listener, app).await.unwrap(); } - #[debug_handler] -async fn get_elevation(State(shared): State, Path((lat, lon)): Path<(f64, f64)>) -> String{ - - let ele = dem::elevation_from_coordinates(shared, lat, lon); +async fn get_elevation(State(dsr): State, Path((lat, lon)): Path<(f64, f64)>) -> String { + let ele = dem::elevation_from_coordinates(dsr, lat, lon); let myele = ele.await; format!("{myele}") } + +fn load_config() -> Result { + Ok(Config { + basedir: env::var("DEM_LOCATION").unwrap_or_else(|_| DEFAULT_DATA_DIR.to_string()), + port: env::var("HTTP_PORT").unwrap_or_else(|_| DEFAULT_PORT.to_string()), + }) +} + +struct Config { + basedir: String, + port: String, +} \ No newline at end of file