Add /refresh to update sources and some configurations

docs/src/using.md

@@ -14,6 +14,7 @@ Martin data is available via the HTTP `GET` endpoints:
 | `/font/{font}/{start}-{end}`            | [Font source](sources-fonts.md)                |
 | `/font/{font1},…,{fontN}/{start}-{end}` | [Composite Font source](sources-fonts.md)      |
 | `/health`                               | Martin server health check: returns 200 `OK`   |
+| `POST /refresh`                         | Refresh MBTiles/PMTiles/PostGIS sources        |
 
 ### Duplicate Source ID
 
@@ -40,24 +41,24 @@ curl localhost:3000/catalog | jq
 ```yaml
 {
   "tiles" {
-    "function_zxy_query": {
-      "name": "public.function_zxy_query",
-      "content_type": "application/x-protobuf"
-    },
-    "points1": {
-      "name": "public.points1.geom",
-      "content_type": "image/webp"
-    },
-    ...
+  "function_zxy_query": {
+    "name": "public.function_zxy_query",
+    "content_type": "application/x-protobuf"
+  },
+  "points1": {
+    "name": "public.points1.geom",
+    "content_type": "image/webp"
   },
+  ...
+},
   "sprites": {
     "cool_icons": {
       "images": [
         "bicycle",
         "bear",
       ]
     },
-    ...
+               ...
   },
   "fonts": {
     "Noto Mono Regular": {
@@ -67,7 +68,7 @@ curl localhost:3000/catalog | jq
       "start": 0,
       "end": 65533
     },
-    ...
+             ...
   }
 }
 ```

martin/Cargo.toml

@@ -2,7 +2,7 @@ lints.workspace = true
 
 [package]
 name = "martin"
-version = "0.14.2"
+version = "0.15.0"
 authors = ["Stepan Kuzmin <[email protected]>", "Yuri Astrakhan <[email protected]>", "MapLibre contributors"]
 description = "Blazing fast and lightweight tile server with PostGIS, MBTiles, and PMTiles support"
 keywords = ["maps", "tiles", "mbtiles", "pmtiles", "postgis"]

martin/src/args/environment.rs

@@ -35,7 +35,7 @@ pub trait Env<'a>: VariableMap<'a> {
 
 /// A map that gives strings from the environment,
 /// but also keeps track of which variables were requested via the `VariableMap` trait.
-#[derive(Default)]
+#[derive(Default, Clone)]
 pub struct OsEnv(RefCell<HashSet<String>>);
 
 impl<'a> Env<'a> for OsEnv {

martin/src/args/pg.rs

@@ -29,7 +29,7 @@ pub enum BoundsCalcType {
     Skip,
 }
 
-#[derive(clap::Args, Debug, PartialEq, Default)]
+#[derive(clap::Args, Debug, PartialEq, Default, Clone)]
 #[command(about, version)]
 pub struct PgArgs {
     /// Specify how bounds should be computed for the spatial PG tables. [DEFAULT: quick]

martin/src/args/root.rs

@@ -18,6 +18,7 @@ use crate::{MartinResult, OptOneMany};
     version,
     after_help = "Use RUST_LOG environment variable to control logging level, e.g. RUST_LOG=debug or RUST_LOG=martin=debug. See https://docs.rs/env_logger/latest/env_logger/index.html#enabling-logging for more information."
 )]
+#[derive(Clone)]
 pub struct Args {
     #[command(flatten)]
     pub meta: MetaArgs,

martin/src/args/srv.rs

@@ -4,7 +4,7 @@ use serde::{Deserialize, Serialize};
 use crate::srv::{SrvConfig, KEEP_ALIVE_DEFAULT, LISTEN_ADDRESSES_DEFAULT};
 
 #[allow(clippy::doc_markdown)]
-#[derive(clap::Args, Debug, PartialEq, Default)]
+#[derive(clap::Args, Debug, PartialEq, Default, Clone)]
 #[command(about, version)]
 pub struct SrvArgs {
     #[arg(help = format!("Connection keep alive timeout. [DEFAULT: {KEEP_ALIVE_DEFAULT}]"), short, long)]

martin/src/bin/martin.rs

@@ -18,7 +18,7 @@ async fn start(args: Args) -> MartinResult<()> {
         info!("Config file is not specified, auto-detecting sources");
         Config::default()
     };
-
+    let args_cloned = args.clone();
     args.merge_into_config(&mut config, &env)?;
     config.finalize()?;
     let sources = config.resolve().await?;
@@ -29,7 +29,7 @@ async fn start(args: Args) -> MartinResult<()> {
         info!("Use --save-config to save or print Martin configuration.");
     }
 
-    let (server, listen_addresses) = new_server(config.srv, sources)?;
+    let (server, listen_addresses) = new_server(env, args_cloned, config.srv, sources)?;
     info!("Martin has been started on {listen_addresses}.");
     info!("Use http://{listen_addresses}/catalog to get the list of available sources.");
     server.await

martin/src/config.rs

@@ -25,6 +25,7 @@ use crate::{IdResolver, MartinResult, OptOneMany};
 
 pub type UnrecognizedValues = HashMap<String, serde_yaml::Value>;
 
+#[derive(Clone)]
 pub struct ServerState {
     pub cache: OptMainCache,
     pub tiles: TileSources,

martin/src/srv/fonts.rs

@@ -4,6 +4,7 @@ use actix_web::error::{ErrorBadRequest, ErrorNotFound};
 use actix_web::web::{Data, Path};
 use actix_web::{middleware, route, HttpResponse, Result as ActixResult};
 use serde::Deserialize;
+use tokio::sync::RwLock;
 
 use crate::fonts::{FontError, FontSources};
 use crate::srv::server::map_internal_error;
@@ -21,8 +22,13 @@ struct FontRequest {
     wrap = "middleware::Compress::default()"
 )]
 #[allow(clippy::unused_async)]
-async fn get_font(path: Path<FontRequest>, fonts: Data<FontSources>) -> ActixResult<HttpResponse> {
+async fn get_font(
+    path: Path<FontRequest>,
+    fonts: Data<RwLock<FontSources>>,
+) -> ActixResult<HttpResponse> {
     let data = fonts
+        .read()
+        .await
         .get_font_range(&path.fontstack, path.start, path.end)
         .map_err(map_font_error)?;
     Ok(HttpResponse::Ok()

martin/src/srv/server.rs

@@ -12,16 +12,19 @@ use actix_web::{middleware, route, web, App, HttpResponse, HttpServer, Responder
 use futures::TryFutureExt;
 #[cfg(feature = "lambda")]
 use lambda_web::{is_running_on_lambda, run_actix_on_lambda};
-use log::error;
+use log::{error, info};
 use serde::{Deserialize, Serialize};
+use tokio::sync::RwLock;
 
+use crate::args::{Args, OsEnv};
 use crate::config::ServerState;
 use crate::source::TileCatalog;
 use crate::srv::config::{SrvConfig, KEEP_ALIVE_DEFAULT, LISTEN_ADDRESSES_DEFAULT};
 use crate::srv::tiles::get_tile;
 use crate::srv::tiles_info::get_source_info;
+use crate::utils::OptMainCache;
 use crate::MartinError::BindingError;
-use crate::MartinResult;
+use crate::{read_config, Config, MartinResult, TileSources};
 
 /// List of keywords that cannot be used as source IDs. Some of these are reserved for future use.
 /// Reserved keywords must never end in a "dot number" (e.g. ".1").
@@ -76,21 +79,86 @@ async fn get_health() -> impl Responder {
         .message_body("OK")
 }
 
+#[allow(clippy::too_many_arguments)]
+#[route("/refresh", method = "POST")]
+#[allow(clippy::unused_async)]
+async fn refresh_catalog(
+    args: Data<Args>,
+    env: Data<OsEnv>,
+    srv_config: Data<RwLock<SrvConfig>>,
+    catalog: Data<RwLock<Catalog>>,
+    state: Data<RwLock<ServerState>>,
+    tiles: Data<RwLock<TileSources>>,
+    cache: Data<RwLock<OptMainCache>>,
+    #[cfg(feature = "sprites")] sprites: Data<RwLock<crate::sprites::SpriteSources>>,
+    #[cfg(feature = "fonts")] fonts: Data<RwLock<crate::fonts::FontSources>>,
+) -> actix_web::error::Result<HttpResponse> {
+    let mut config = if let Some(ref cfg_filename) = args.meta.config {
+        info!("Using {} to refresh catalog", cfg_filename.display());
+        read_config(cfg_filename, env.get_ref()).map_err(map_internal_error)?
+    } else {
+        info!("Config file is not specified, an default config will be used to refresh catalog");
+        Config::default()
+    };
+    let cloned_args = (**args).clone();
+    cloned_args
+        .merge_into_config(&mut config, env.get_ref())
+        .map_err(map_internal_error)?;
+
+    config.finalize().map_err(map_internal_error)?;
+
+    let sources = config.resolve().await.map_err(map_internal_error)?;
+
+    // update these two guards
+    let new_srv_config = config.srv;
+    let new_state = sources;
+    let new_catalog = Catalog::new(&new_state).map_err(map_internal_error)?;
+    let new_tiles = new_state.tiles.clone();
+    let new_cache = new_state.cache.clone();
+
+    let mut srv_config = srv_config.write().await;
+    let mut state = state.write().await;
+    let mut catalog = catalog.write().await;
+    let mut tiles = tiles.write().await;
+    let mut cache = cache.write().await;
+
+    #[cfg(feature = "sprites")]
+    {
+        let mut sprites = sprites.write().await;
+        *sprites = new_state.sprites.clone();
+    }
+    #[cfg(feature = "fonts")]
+    {
+        let mut fonts = fonts.write().await;
+        *fonts = new_state.fonts.clone();
+    }
+
+    *srv_config = new_srv_config;
+    *state = new_state;
+    *catalog = new_catalog;
+    *tiles = new_tiles;
+    *cache = new_cache;
+
+    Ok(HttpResponse::Ok().finish())
+}
+
 #[route(
     "/catalog",
     method = "GET",
     method = "HEAD",
     wrap = "middleware::Compress::default()"
 )]
 #[allow(clippy::unused_async)]
-async fn get_catalog(catalog: Data<Catalog>) -> impl Responder {
-    HttpResponse::Ok().json(catalog)
+async fn get_catalog(catalog: Data<RwLock<Catalog>>) -> impl Responder {
+    let catalog = catalog.read().await;
+    HttpResponse::Ok().json(&*catalog)
 }
 
 pub fn router(cfg: &mut web::ServiceConfig) {
     cfg.service(get_health)
         .service(get_index)
         .service(get_catalog)
+        .service(refresh_catalog)
         .service(get_source_info)
         .service(get_tile);
 
@@ -105,7 +173,12 @@ pub fn router(cfg: &mut web::ServiceConfig) {
 type Server = Pin<Box<dyn Future<Output = MartinResult<()>>>>;
 
 /// Create a future for an Actix web server together with the listening address.
-pub fn new_server(config: SrvConfig, state: ServerState) -> MartinResult<(Server, String)> {
+pub fn new_server(
+    env: OsEnv,
+    args: Args,
+    config: SrvConfig,
+    state: ServerState,
+) -> MartinResult<(Server, String)> {
     let catalog = Catalog::new(&state)?;
 
     let keep_alive = Duration::from_secs(config.keep_alive.unwrap_or(KEEP_ALIVE_DEFAULT));
@@ -121,17 +194,20 @@ pub fn new_server(config: SrvConfig, state: ServerState) -> MartinResult<(Server
             .allowed_methods(vec!["GET"]);
 
         let app = App::new()
-            .app_data(Data::new(state.tiles.clone()))
-            .app_data(Data::new(state.cache.clone()));
+            .app_data(Data::new(RwLock::new(state.tiles.clone())))
+            .app_data(Data::new(RwLock::new(state.cache.clone())))
+            .app_data(Data::new(RwLock::new(state.clone())));
 
         #[cfg(feature = "sprites")]
-        let app = app.app_data(Data::new(state.sprites.clone()));
+        let app = app.app_data(Data::new(RwLock::new(state.sprites.clone())));
 
         #[cfg(feature = "fonts")]
-        let app = app.app_data(Data::new(state.fonts.clone()));
+        let app = app.app_data(Data::new(RwLock::new(state.fonts.clone())));
 
-        app.app_data(Data::new(catalog.clone()))
-            .app_data(Data::new(config.clone()))
+        app.app_data(Data::new(env.clone()))
+            .app_data(Data::new(args.clone()))
+            .app_data(Data::new(RwLock::new(catalog.clone())))
+            .app_data(Data::new(RwLock::new(config.clone())))
             .wrap(cors_middleware)
             .wrap(middleware::NormalizePath::new(TrailingSlash::MergeOnly))
             .wrap(middleware::Logger::default())

martin/src/srv/sprites.rs

@@ -5,6 +5,7 @@ use actix_web::http::header::ContentType;
 use actix_web::web::{Data, Path};
 use actix_web::{middleware, route, HttpResponse, Result as ActixResult};
 use spreet::Spritesheet;
+use tokio::sync::RwLock;
 
 use crate::sprites::{SpriteError, SpriteSources};
 use crate::srv::server::map_internal_error;
@@ -13,8 +14,9 @@ use crate::srv::SourceIDsRequest;
 #[route("/sprite/{source_ids}.png", method = "GET", method = "HEAD")]
 async fn get_sprite_png(
     path: Path<SourceIDsRequest>,
-    sprites: Data<SpriteSources>,
+    sprites: Data<RwLock<SpriteSources>>,
 ) -> ActixResult<HttpResponse> {
+    let sprites = sprites.read().await;
     let sheet = get_sprite(&path, &sprites).await?;
     Ok(HttpResponse::Ok()
         .content_type(ContentType::png())
@@ -29,8 +31,9 @@ async fn get_sprite_png(
 )]
 async fn get_sprite_json(
     path: Path<SourceIDsRequest>,
-    sprites: Data<SpriteSources>,
+    sprites: Data<RwLock<SpriteSources>>,
 ) -> ActixResult<HttpResponse> {
+    let sprites = sprites.read().await;
     let sheet = get_sprite(&path, &sprites).await?;
     Ok(HttpResponse::Ok().json(sheet.get_index()))
 }

martin/src/srv/tiles.rs

@@ -12,6 +12,7 @@ use martin_tile_utils::{
     decode_brotli, decode_gzip, encode_brotli, encode_gzip, Encoding, Format, TileCoord, TileInfo,
 };
 use serde::Deserialize;
+use tokio::sync::RwLock;
 
 use crate::args::PreferredEncoding;
 use crate::source::{Source, TileSources, UrlQuery};
@@ -38,19 +39,23 @@ pub struct TileRequest {
 #[route("/{source_ids}/{z}/{x}/{y}", method = "GET", method = "HEAD")]
 async fn get_tile(
     req: HttpRequest,
-    srv_config: Data<SrvConfig>,
+    srv_config: Data<RwLock<SrvConfig>>,
     path: Path<TileRequest>,
-    sources: Data<TileSources>,
-    cache: Data<OptMainCache>,
+    sources: Data<RwLock<TileSources>>,
+    cache: Data<RwLock<OptMainCache>>,
 ) -> ActixResult<HttpResponse> {
+    let srv_config = srv_config.read().await;
+    let cache = cache.read().await;
+    let sources = sources.read().await;
+
     let src = DynTileSource::new(
-        sources.as_ref(),
+        &sources,
         &path.source_ids,
         Some(path.z),
         req.query_string(),
         req.get_header::<AcceptEncoding>(),
         srv_config.preferred_encoding,
-        cache.as_ref().as_ref(),
+        cache.as_ref(),
     )?;
 
     src.get_http_response(TileCoord {