From 31e46cdbfbeff16d5c246527c1008c40856e1baa Mon Sep 17 00:00:00 2001 From: David Dal Busco Date: Thu, 9 Nov 2023 20:18:14 +0100 Subject: [PATCH] feat: certification v2 (#269) --- Cargo.lock | 157 ++++++++- dfx.json | 3 +- docker/build | 68 ++-- src/declarations/satellite/satellite.did.d.ts | 6 + .../satellite/satellite.factory.did.js | 10 +- .../satellite/satellite.factory.did.mjs | 10 +- src/satellite/Cargo.toml | 2 + src/satellite/satellite.did | 3 + src/satellite/src/impls.rs | 8 +- src/satellite/src/lib.rs | 115 +++---- src/satellite/src/storage/cert.rs | 51 --- .../src/storage/certification/cert.rs | 121 +++++++ .../src/storage/certification/constants.rs | 10 + .../src/storage/certification/impls.rs | 306 ++++++++++++++++++ .../src/storage/certification/mod.rs | 6 + .../src/storage/certification/tree.rs | 203 ++++++++++++ .../src/storage/certification/tree_utils.rs | 114 +++++++ .../src/storage/certification/types.rs | 12 + src/satellite/src/storage/constants.rs | 14 +- src/satellite/src/storage/custom_domains.rs | 2 +- src/satellite/src/storage/http.rs | 167 ---------- src/satellite/src/storage/http/headers.rs | 84 +++++ src/satellite/src/storage/http/mod.rs | 4 + src/satellite/src/storage/http/response.rs | 120 +++++++ src/satellite/src/storage/http/types.rs | 55 ++++ src/satellite/src/storage/http/utils.rs | 139 ++++++++ src/satellite/src/storage/impls.rs | 61 +--- src/satellite/src/storage/mod.rs | 5 +- src/satellite/src/storage/rewrites.rs | 71 ++-- src/satellite/src/storage/routing.rs | 211 ++++++++++++ src/satellite/src/storage/runtime.rs | 61 +++- src/satellite/src/storage/store.rs | 115 +++---- src/satellite/src/storage/types.rs | 108 +++---- src/satellite/src/storage/url.rs | 32 +- 34 files changed, 1885 insertions(+), 569 deletions(-) delete mode 100644 src/satellite/src/storage/cert.rs create mode 100644 src/satellite/src/storage/certification/cert.rs create mode 100644 src/satellite/src/storage/certification/constants.rs create mode 100644 src/satellite/src/storage/certification/impls.rs create mode 100644 src/satellite/src/storage/certification/mod.rs create mode 100644 src/satellite/src/storage/certification/tree.rs create mode 100644 src/satellite/src/storage/certification/tree_utils.rs create mode 100644 src/satellite/src/storage/certification/types.rs delete mode 100644 src/satellite/src/storage/http.rs create mode 100644 src/satellite/src/storage/http/headers.rs create mode 100644 src/satellite/src/storage/http/mod.rs create mode 100644 src/satellite/src/storage/http/response.rs create mode 100644 src/satellite/src/storage/http/types.rs create mode 100644 src/satellite/src/storage/http/utils.rs create mode 100644 src/satellite/src/storage/routing.rs diff --git a/Cargo.lock b/Cargo.lock index 6c82097ba..b5c6e652c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + [[package]] name = "aho-corasick" version = "1.1.2" @@ -35,6 +41,12 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" +[[package]] +name = "base64" +version = "0.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2" + [[package]] name = "binread" version = "2.2.0" @@ -83,6 +95,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +[[package]] +name = "bytes" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" + [[package]] name = "candid" version = "0.9.11" @@ -243,6 +261,16 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +[[package]] +name = "flate2" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" @@ -388,6 +416,30 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "http" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "ic-cbor" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "767fe244224c02f2adad1d43909de93d35d5caada28eed3f270d89735ba5bdd0" +dependencies = [ + "candid", + "ic-certification", + "leb128", + "nom", + "thiserror", +] + [[package]] name = "ic-cdk" version = "0.11.3" @@ -429,6 +481,31 @@ dependencies = [ "slotmap", ] +[[package]] +name = "ic-certificate-verification" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bc3fa76751b637bb0c0e37f4089d7f5a3fbacfed3c5fbcfef0d6797a54d63f" +dependencies = [ + "candid", + "ic-cbor", + "ic-certification", + "leb128", + "miracl_core_bls12381", + "nom", + "thiserror", +] + +[[package]] +name = "ic-certification" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a59dc342d11b2067e19d0f146bdec3674de921303ffc762f114d201ebbe0e68a" +dependencies = [ + "hex", + "sha2", +] + [[package]] name = "ic-certified-map" version = "0.4.0" @@ -455,6 +532,39 @@ dependencies = [ "sha2", ] +[[package]] +name = "ic-representation-independent-hash" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "410e3ccb16b86e8c4bca7270981370f6961492f1921f29f5dd1de621efb869ea" +dependencies = [ + "leb128", + "sha2", +] + +[[package]] +name = "ic-response-verification" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc87254b89cea2b84fb2e3101527e457d01da1bdfdbf8b8699dcbc40ad11dcea" +dependencies = [ + "base64 0.21.4", + "candid", + "flate2", + "hex", + "http", + "ic-cbor", + "ic-certificate-verification", + "ic-certification", + "ic-representation-independent-hash", + "leb128", + "log", + "nom", + "sha2", + "thiserror", + "urlencoding", +] + [[package]] name = "ic-stable-structures" version = "0.5.6" @@ -502,6 +612,12 @@ dependencies = [ "regex", ] +[[package]] +name = "itoa" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" + [[package]] name = "lazy_static" version = "1.4.0" @@ -532,6 +648,27 @@ version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +dependencies = [ + "adler", +] + +[[package]] +name = "miracl_core_bls12381" +version = "4.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d07cbe42e2a8dd41df582fb8e00fc24d920b5561cc301fcb6d14e2e0434b500f" + [[package]] name = "mission_control" version = "0.0.7" @@ -545,6 +682,16 @@ dependencies = [ "shared", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "num-bigint" version = "0.4.4" @@ -742,7 +889,7 @@ checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" name = "satellite" version = "0.0.12" dependencies = [ - "base64", + "base64 0.13.1", "candid", "ciborium", "globset", @@ -750,6 +897,8 @@ dependencies = [ "ic-cdk", "ic-cdk-macros", "ic-certified-map", + "ic-representation-independent-hash", + "ic-response-verification", "ic-stable-structures 0.6.0", "regex", "serde", @@ -998,6 +1147,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "version_check" version = "0.9.4" diff --git a/dfx.json b/dfx.json index 2cdb1b6eb..3c7751585 100644 --- a/dfx.json +++ b/dfx.json @@ -34,7 +34,8 @@ "declarations": { "node_compatibility": true }, - "optimize": "cycles" + "optimize": "cycles", + "metadata": [{ "name": "supported_certificate_versions", "content": "1,2" }] }, "orbiter": { "candid": "src/orbiter/orbiter.did", diff --git a/docker/build b/docker/build index 6d5dcfb22..a439964ad 100755 --- a/docker/build +++ b/docker/build @@ -11,17 +11,16 @@ cd "$SCRIPTS_DIR/.." ######### function title() { - echo "Builds Juno Canisters" + echo "Build Juno Canister" } function usage() { cat << EOF Usage: - $0 [--only-dependencies] [--mission_control] [--satellite] [--console] [--observatory] [--orbiter] + $0 [--mission_control] [--satellite] [--console] [--observatory] [--orbiter] Options: - --only-dependencies only build rust dependencies (no js build, no wasm optimization) --mission_control build the mission_control canister (default) --satellite build the satellite canister --console build the console canister @@ -33,15 +32,15 @@ EOF function help() { cat << EOF -Builds the Mission Control, Satellite and Console canisters. +Build the Mission Control, Satellite, Orbiter, Observatory or Console canister. NOTE: This requires a working rust toolchain as well as ic-wasm. EOF } -ONLY_DEPS= -CANISTERS=() +CERTIFICATION= +CANISTER= while [[ $# -gt 0 ]] do @@ -52,29 +51,26 @@ do help exit 0 ;; - --only-dependencies) - ONLY_DEPS=1 - shift - ;; --mission_control) - CANISTERS+=("mission_control") - shift + CANISTER="mission_control" + break ;; --satellite) - CANISTERS+=("satellite") - shift + CANISTER="satellite" + CERTIFICATION="true" + break ;; --console) - CANISTERS+=("console") - shift + CANISTER="console" + break ;; --observatory) - CANISTERS+=("observatory") - shift + CANISTER="observatory" + break ;; --orbiter) - CANISTERS+=("orbiter") - shift + CANISTER="orbiter" + break ;; *) echo "ERROR: unknown argument $1" @@ -87,8 +83,8 @@ do done # build Mission Control by default -if [ ${#CANISTERS[@]} -eq 0 ]; then - CANISTERS=("mission_control") +if [ ${#CANISTER} -eq 0 ]; then + CANISTER=("mission_control") fi # Checking for dependencies @@ -129,23 +125,23 @@ function build_canister() { RUSTFLAGS="$RUSTFLAGS" cargo build "${cargo_build_args[@]}" - if [ "$ONLY_DEPS" != "1" ] - then - CARGO_TARGET_DIR="${CARGO_TARGET_DIR:-$SRC_DIR/../../target/}" + CARGO_TARGET_DIR="${CARGO_TARGET_DIR:-$SRC_DIR/../../target/}" - ic-wasm \ - "$CARGO_TARGET_DIR/$TARGET/release/$canister.wasm" \ - -o "./$canister.wasm" \ - shrink + ic-wasm \ + "$CARGO_TARGET_DIR/$TARGET/release/$canister.wasm" \ + -o "./$canister.wasm" \ + shrink - # adds the content of $canister.did to the `icp:public candid:service` custom section of the public metadata in the wasm - ic-wasm "$canister.wasm" -o "$canister.wasm" metadata candid:service -f "$SRC_DIR/$canister.did" -v public + # adds the content of $canister.did to the `icp:public candid:service` custom section of the public metadata in the wasm + ic-wasm "$canister.wasm" -o "$canister.wasm" metadata candid:service -f "$SRC_DIR/$canister.did" -v public - gzip "./$canister.wasm" + if [ "$CERTIFICATION" == "true" ] + then + # indicate support for certificate version 1 and 2 in the canister metadata + ic-wasm "$canister.wasm" -o "$canister.wasm" metadata supported_certificate_versions -d "1,2" -v public fi + + gzip "./$canister.wasm" } -for canister in "${CANISTERS[@]}" -do - build_canister "$canister" -done +build_canister "$CANISTER" diff --git a/src/declarations/satellite/satellite.did.d.ts b/src/declarations/satellite/satellite.did.d.ts index d813cead3..5b6aba629 100644 --- a/src/declarations/satellite/satellite.did.d.ts +++ b/src/declarations/satellite/satellite.did.d.ts @@ -60,6 +60,7 @@ export interface HttpRequest { method: string; body: Uint8Array | number[]; headers: Array<[string, string]>; + certificate_version: [] | [number]; } export interface HttpResponse { body: Uint8Array | number[]; @@ -152,6 +153,11 @@ export interface SetRule { export interface StorageConfig { rewrites: Array<[string, string]>; headers: Array<[string, Array<[string, string]>]>; + redirects: [] | [Array<[string, StorageConfigRedirect]>]; +} +export interface StorageConfigRedirect { + status_code: number; + location: string; } export interface StreamingCallbackHttpResponse { token: [] | [StreamingCallbackToken]; diff --git a/src/declarations/satellite/satellite.factory.did.js b/src/declarations/satellite/satellite.factory.did.js index 03683ab06..c74c5dfae 100644 --- a/src/declarations/satellite/satellite.factory.did.js +++ b/src/declarations/satellite/satellite.factory.did.js @@ -21,9 +21,14 @@ export const idlFactory = ({ IDL }) => { }); const DelDoc = IDL.Record({ updated_at: IDL.Opt(IDL.Nat64) }); const RulesType = IDL.Variant({ Db: IDL.Null, Storage: IDL.Null }); + const StorageConfigRedirect = IDL.Record({ + status_code: IDL.Nat16, + location: IDL.Text + }); const StorageConfig = IDL.Record({ rewrites: IDL.Vec(IDL.Tuple(IDL.Text, IDL.Text)), - headers: IDL.Vec(IDL.Tuple(IDL.Text, IDL.Vec(IDL.Tuple(IDL.Text, IDL.Text)))) + headers: IDL.Vec(IDL.Tuple(IDL.Text, IDL.Vec(IDL.Tuple(IDL.Text, IDL.Text)))), + redirects: IDL.Opt(IDL.Vec(IDL.Tuple(IDL.Text, StorageConfigRedirect))) }); const Config = IDL.Record({ storage: StorageConfig }); const Doc = IDL.Record({ @@ -37,7 +42,8 @@ export const idlFactory = ({ IDL }) => { url: IDL.Text, method: IDL.Text, body: IDL.Vec(IDL.Nat8), - headers: IDL.Vec(IDL.Tuple(IDL.Text, IDL.Text)) + headers: IDL.Vec(IDL.Tuple(IDL.Text, IDL.Text)), + certificate_version: IDL.Opt(IDL.Nat16) }); const Memory = IDL.Variant({ Heap: IDL.Null, Stable: IDL.Null }); const StreamingCallbackToken = IDL.Record({ diff --git a/src/declarations/satellite/satellite.factory.did.mjs b/src/declarations/satellite/satellite.factory.did.mjs index 03683ab06..c74c5dfae 100644 --- a/src/declarations/satellite/satellite.factory.did.mjs +++ b/src/declarations/satellite/satellite.factory.did.mjs @@ -21,9 +21,14 @@ export const idlFactory = ({ IDL }) => { }); const DelDoc = IDL.Record({ updated_at: IDL.Opt(IDL.Nat64) }); const RulesType = IDL.Variant({ Db: IDL.Null, Storage: IDL.Null }); + const StorageConfigRedirect = IDL.Record({ + status_code: IDL.Nat16, + location: IDL.Text + }); const StorageConfig = IDL.Record({ rewrites: IDL.Vec(IDL.Tuple(IDL.Text, IDL.Text)), - headers: IDL.Vec(IDL.Tuple(IDL.Text, IDL.Vec(IDL.Tuple(IDL.Text, IDL.Text)))) + headers: IDL.Vec(IDL.Tuple(IDL.Text, IDL.Vec(IDL.Tuple(IDL.Text, IDL.Text)))), + redirects: IDL.Opt(IDL.Vec(IDL.Tuple(IDL.Text, StorageConfigRedirect))) }); const Config = IDL.Record({ storage: StorageConfig }); const Doc = IDL.Record({ @@ -37,7 +42,8 @@ export const idlFactory = ({ IDL }) => { url: IDL.Text, method: IDL.Text, body: IDL.Vec(IDL.Nat8), - headers: IDL.Vec(IDL.Tuple(IDL.Text, IDL.Text)) + headers: IDL.Vec(IDL.Tuple(IDL.Text, IDL.Text)), + certificate_version: IDL.Opt(IDL.Nat16) }); const Memory = IDL.Variant({ Heap: IDL.Null, Stable: IDL.Null }); const StreamingCallbackToken = IDL.Record({ diff --git a/src/satellite/Cargo.toml b/src/satellite/Cargo.toml index d080e782e..5a56d6e7c 100644 --- a/src/satellite/Cargo.toml +++ b/src/satellite/Cargo.toml @@ -24,4 +24,6 @@ globset = "0.4.13" hex = "0.4.3" ic-stable-structures = "0.6.0" ciborium = "0.2.1" +ic-response-verification = "1.2.0" +ic-representation-independent-hash = "1.2.0" shared = { path = "../shared" } diff --git a/src/satellite/satellite.did b/src/satellite/satellite.did index b46d7623a..7440b997a 100644 --- a/src/satellite/satellite.did +++ b/src/satellite/satellite.did @@ -51,6 +51,7 @@ type HttpRequest = record { method : text; body : vec nat8; headers : vec record { text; text }; + certificate_version : opt nat16; }; type HttpResponse = record { body : vec nat8; @@ -128,7 +129,9 @@ type SetRule = record { type StorageConfig = record { rewrites : vec record { text; text }; headers : vec record { text; vec record { text; text } }; + redirects : opt vec record { text; StorageConfigRedirect }; }; +type StorageConfigRedirect = record { status_code : nat16; location : text }; type StreamingCallbackHttpResponse = record { token : opt StreamingCallbackToken; body : vec nat8; diff --git a/src/satellite/src/impls.rs b/src/satellite/src/impls.rs index 7c6023bdf..ccf3a9861 100644 --- a/src/satellite/src/impls.rs +++ b/src/satellite/src/impls.rs @@ -2,8 +2,9 @@ use crate::db::types::state::DbHeapState; use crate::memory::init_stable_state; use crate::rules::constants::{DEFAULT_ASSETS_COLLECTIONS, DEFAULT_DB_COLLECTIONS}; use crate::rules::types::rules::{Memory, Rule}; -use crate::storage::rewrites::init_rewrites; -use crate::storage::types::config::{StorageConfig, StorageConfigHeaders}; +use crate::storage::types::config::{ + StorageConfig, StorageConfigHeaders, StorageConfigRedirects, StorageConfigRewrites, +}; use crate::storage::types::state::StorageHeapState; use crate::types::state::{HeapState, RuntimeState, State}; use ic_cdk::api::time; @@ -63,7 +64,8 @@ impl Default for HeapState { })), config: StorageConfig { headers: StorageConfigHeaders::default(), - rewrites: init_rewrites(), + rewrites: StorageConfigRewrites::default(), + redirects: Some(StorageConfigRedirects::default()), }, custom_domains: HashMap::new(), }; diff --git a/src/satellite/src/lib.rs b/src/satellite/src/lib.rs index 95164460b..fb4bb6a5f 100644 --- a/src/satellite/src/lib.rs +++ b/src/satellite/src/lib.rs @@ -21,24 +21,26 @@ use crate::rules::store::{ }; use crate::rules::types::interface::{DelRule, SetRule}; use crate::rules::types::rules::Rule; -use crate::storage::http::{ - build_encodings, build_headers, create_token, error_response, streaming_strategy, +use crate::storage::constants::{RESPONSE_STATUS_CODE_200, RESPONSE_STATUS_CODE_405}; +use crate::storage::http::response::{ + build_asset_response, build_redirect_response, error_response, }; +use crate::storage::http::utils::create_token; +use crate::storage::routing::get_routing; use crate::storage::store::{ commit_batch, create_batch, create_chunk, delete_asset, delete_assets, delete_domain, get_config as get_storage_config, get_content_chunks, get_custom_domains, get_public_asset, - get_public_asset_for_url, init_certified_assets, list_assets as list_assets_store, - set_config as set_storage_config, set_domain, + init_certified_assets, list_assets as list_assets_store, set_config as set_storage_config, + set_domain, }; +use crate::storage::types::config::StorageConfigRewrites; use crate::storage::types::domain::{CustomDomains, DomainName}; -use crate::storage::types::http::{ - HttpRequest, HttpResponse, StreamingCallbackHttpResponse, StreamingCallbackToken, +use crate::storage::types::http_request::{ + Routing, RoutingDefault, RoutingRedirect, RoutingRewrite, }; -use crate::storage::types::http_request::PublicAsset; use crate::storage::types::interface::{ AssetNoContent, CommitBatch, InitAssetKey, InitUploadResult, UploadChunk, UploadChunkResult, }; -use crate::storage::types::store::Asset; use crate::types::core::CollectionKey; use crate::types::interface::{Config, RulesType}; use crate::types::list::ListResults; @@ -60,6 +62,9 @@ use shared::controllers::{assert_max_number_of_controllers, init_controllers}; use shared::types::interface::{DeleteControllersArgs, SegmentArgs, SetControllersArgs}; use shared::types::state::{ControllerScope, Controllers}; use std::mem; +use storage::http::types::{ + HttpRequest, HttpResponse, StreamingCallbackHttpResponse, StreamingCallbackToken, +}; use types::list::ListParams; #[init] @@ -119,6 +124,13 @@ fn post_upgrade() { .expect("Failed to decode the state of the satellite in post_upgrade hook."); STATE.with(|s| *s.borrow_mut() = state); + // TODO: to be removed. + // Post upgrade hook to reset the rewrites after upgrading to certification v2 because the fallback to /index.html is handled differently now. + STATE.with(|s| { + let state = &mut s.borrow_mut(); + state.heap.storage.config.rewrites = StorageConfigRewrites::default(); + }); + init_certified_assets(); } @@ -281,75 +293,44 @@ fn http_request( url, headers: req_headers, body: _, + certificate_version, }: HttpRequest, ) -> HttpResponse { if method != "GET" { - return error_response(405, "Method Not Allowed.".to_string()); + return error_response(RESPONSE_STATUS_CODE_405, "Method Not Allowed.".to_string()); } - let result = get_public_asset_for_url(url); + let result = get_routing(url, true); match result { - Ok(PublicAsset { - asset, - url: requested_url, - }) => match asset { - Some((asset, memory)) => { - let encodings = build_encodings(req_headers); - - for encoding_type in encodings.iter() { - if let Some(encoding) = asset.encodings.get(encoding_type) { - let headers = - build_headers(&requested_url, &asset, encoding, encoding_type); - - let Asset { - key, - headers: _, - encodings: _, - created_at: _, - updated_at: _, - } = &asset; - - match headers { - Ok(headers) => { - let body = get_content_chunks(encoding, 0, &memory); - - match body { - Some(body) => { - return HttpResponse { - body: body.clone(), - headers: headers.clone(), - status_code: 200, - streaming_strategy: streaming_strategy( - key, - encoding, - encoding_type, - &headers, - &memory, - ), - } - } - None => { - error_response(500, "No chunks found.".to_string()); - } - } - } - Err(err) => { - return error_response( - 405, - ["Permission denied. Invalid headers. ", err].join(""), - ); - } - } - } - } - - error_response(500, "No asset encoding found.".to_string()) + Ok(routing) => match routing { + Routing::Default(RoutingDefault { url, asset }) => build_asset_response( + url, + req_headers, + certificate_version, + asset, + None, + RESPONSE_STATUS_CODE_200, + ), + Routing::Rewrite(RoutingRewrite { + url, + asset, + source, + status_code, + }) => build_asset_response( + url, + req_headers, + certificate_version, + asset, + Some(source), + status_code, + ), + Routing::Redirect(RoutingRedirect { url, redirect }) => { + build_redirect_response(url, certificate_version, &redirect) } - None => error_response(404, "No asset found.".to_string()), }, Err(err) => error_response( - 405, + RESPONSE_STATUS_CODE_405, ["Permission denied. Cannot perform this operation. ", err].join(""), ), } diff --git a/src/satellite/src/storage/cert.rs b/src/satellite/src/storage/cert.rs deleted file mode 100644 index 46519fe40..000000000 --- a/src/satellite/src/storage/cert.rs +++ /dev/null @@ -1,51 +0,0 @@ -use crate::storage::types::assets::AssetHashes; -use crate::storage::types::http::HeaderField; -use base64::encode; -use ic_cdk::api::{data_certificate, set_certified_data}; -use ic_certified_map::{labeled, labeled_hash, AsHashTree}; -use serde::Serialize; -use serde_cbor::ser::Serializer; - -const LABEL_ASSETS: &[u8] = b"http_assets"; - -pub fn update_certified_data(asset_hashes: &AssetHashes) { - let prefixed_root_hash = &labeled_hash(LABEL_ASSETS, &asset_hashes.tree.root_hash()); - set_certified_data(&prefixed_root_hash[..]); -} - -pub fn build_asset_certificate_header( - asset_hashes: &AssetHashes, - url: String, -) -> Result { - let certificate = data_certificate(); - - match certificate { - None => Err("No certificate found."), - Some(certificate) => build_asset_certificate_header_impl(&certificate, asset_hashes, &url), - } -} - -fn build_asset_certificate_header_impl( - certificate: &Vec, - asset_hashes: &AssetHashes, - url: &String, -) -> Result { - let witness = asset_hashes.tree.witness(url.as_bytes()); - let tree = labeled(LABEL_ASSETS, witness); - - let mut serializer = Serializer::new(vec![]); - serializer.self_describe().unwrap(); - let result = tree.serialize(&mut serializer); - - match result { - Err(_err) => Err("Failed to serialize a hash tree."), - Ok(_serialize) => Ok(HeaderField( - "IC-Certificate".to_string(), - format!( - "certificate=:{}:, tree=:{}:", - encode(certificate), - encode(serializer.into_inner()) - ), - )), - } -} diff --git a/src/satellite/src/storage/certification/cert.rs b/src/satellite/src/storage/certification/cert.rs new file mode 100644 index 000000000..0039e23ff --- /dev/null +++ b/src/satellite/src/storage/certification/cert.rs @@ -0,0 +1,121 @@ +use crate::storage::certification::constants::{ + IC_CERTIFICATE_EXPRESSION_HEADER, IC_CERTIFICATE_HEADER, +}; +use crate::storage::certification::tree_utils::response_headers_expression; +use crate::storage::certification::types::certified::CertifiedAssetHashes; +use crate::storage::http::types::HeaderField; +use crate::types::core::Blob; +use base64::encode; +use ic_cdk::api::{data_certificate, set_certified_data}; +use serde::Serialize; +use serde_cbor::ser::Serializer; + +pub fn update_certified_data(asset_hashes: &CertifiedAssetHashes) { + let prefixed_root_hash = &asset_hashes.root_hash(); + set_certified_data(&prefixed_root_hash[..]); +} + +pub fn build_asset_certificate_header( + asset_hashes: &CertifiedAssetHashes, + url: String, + certificate_version: &Option, + rewrite_source: &Option, +) -> Result { + let certificate = data_certificate(); + + match certificate { + None => Err("No certificate found."), + Some(certificate) => match certificate_version { + None | Some(1) => { + build_asset_certificate_header_v1_impl(&certificate, asset_hashes, &url) + } + Some(2) => build_asset_certificate_header_v2_impl( + &certificate, + asset_hashes, + &url, + rewrite_source, + ), + _ => Err("Unsupported certificate version to certify headers."), + }, + } +} + +pub fn build_certified_expression( + asset_headers: &[HeaderField], + certificate_version: &Option, +) -> Result, &'static str> { + match certificate_version { + None | Some(1) => Ok(None), + Some(2) => Ok(Some(HeaderField( + IC_CERTIFICATE_EXPRESSION_HEADER.to_string(), + response_headers_expression(asset_headers), + ))), + _ => Err("Unsupported certificate version to certify expression."), + } +} + +fn build_asset_certificate_header_v1_impl( + certificate: &Blob, + asset_hashes: &CertifiedAssetHashes, + url: &str, +) -> Result { + let tree = asset_hashes.witness_v1(url); + + let mut serializer = Serializer::new(vec![]); + serializer.self_describe().unwrap(); + let result = tree.serialize(&mut serializer); + + match result { + Err(_err) => Err("Failed to serialize a hash tree."), + Ok(_serialize) => Ok(HeaderField( + IC_CERTIFICATE_HEADER.to_string(), + format!( + "certificate=:{}:, tree=:{}:", + encode(certificate), + encode(serializer.into_inner()) + ), + )), + } +} + +fn build_asset_certificate_header_v2_impl( + certificate: &Blob, + asset_hashes: &CertifiedAssetHashes, + url: &str, + rewrite_source: &Option, +) -> Result { + assert!(url.starts_with('/')); + + let tree = match rewrite_source { + None => asset_hashes.witness_v2(url), + Some(_) => asset_hashes.witness_rewrite_v2(url), + }; + + let mut serializer = Serializer::new(vec![]); + serializer.self_describe().unwrap(); + let result = tree.serialize(&mut serializer); + + match result { + Err(_err) => Err("Failed to serialize a hash tree."), + Ok(_serialize) => { + let mut expr_path_serializer = Serializer::new(vec![]); + expr_path_serializer.self_describe().unwrap(); + + let path = asset_hashes.expr_path_v2(url, rewrite_source); + let result_path = path.serialize(&mut expr_path_serializer); + + match result_path { + Err(_err) => Err("Failed to serialize path."), + Ok(_serialize) => Ok(HeaderField( + IC_CERTIFICATE_HEADER.to_string(), + format!( + "certificate=:{}:, tree=:{}:, expr_path=:{}:, version=2", + encode(certificate), + encode(serializer.into_inner()), + encode(expr_path_serializer.into_inner()) + ), + )), + } + } + } +} diff --git a/src/satellite/src/storage/certification/constants.rs b/src/satellite/src/storage/certification/constants.rs new file mode 100644 index 000000000..d5baf7cff --- /dev/null +++ b/src/satellite/src/storage/certification/constants.rs @@ -0,0 +1,10 @@ +/// Certification +pub const LABEL_ASSETS_V1: &[u8] = b"http_assets"; +pub const LABEL_ASSETS_V2: &[u8] = b"http_expr"; +pub const LABEL_HTTP_EXPR: &str = "http_expr"; +pub const EXACT_MATCH_TERMINATOR: &str = "<$>"; +pub const WILDCARD_MATCH_TERMINATOR: &str = "<*>"; +pub const IC_CERTIFICATE_HEADER: &str = "IC-Certificate"; +pub const IC_CERTIFICATE_EXPRESSION_HEADER: &str = "IC-CertificateExpression"; +pub const IC_STATUS_CODE_PSEUDO_HEADER: &str = ":ic-cert-status"; +pub const IC_CERTIFICATE_EXPRESSION: &str = r#"default_certification(ValidationArgs{certification:Certification{no_request_certification:Empty{},response_certification:ResponseCertification{certified_response_headers:ResponseHeaderList{headers:[{headers}]}}}})"#; diff --git a/src/satellite/src/storage/certification/impls.rs b/src/satellite/src/storage/certification/impls.rs new file mode 100644 index 000000000..a03a00d60 --- /dev/null +++ b/src/satellite/src/storage/certification/impls.rs @@ -0,0 +1,306 @@ +use crate::storage::certification::constants::{ + EXACT_MATCH_TERMINATOR, LABEL_ASSETS_V1, LABEL_ASSETS_V2, WILDCARD_MATCH_TERMINATOR, +}; +use crate::storage::certification::tree::merge_hash_trees; +use crate::storage::certification::tree_utils::{ + fallback_paths, nested_tree_expr_path, nested_tree_key, nested_tree_path, +}; +use crate::storage::certification::types::certified::CertifiedAssetHashes; +use crate::storage::constants::{ + ENCODING_CERTIFICATION_ORDER, RESPONSE_STATUS_CODE_200, RESPONSE_STATUS_CODE_404, + ROOT_404_HTML, ROOT_INDEX_HTML, ROOT_PATH, +}; +use crate::storage::http::headers::{build_headers, build_redirect_headers}; +use crate::storage::http::types::{HeaderField, StatusCode}; +use crate::storage::types::config::StorageConfig; +use crate::storage::types::state::FullPath; +use crate::storage::types::store::Asset; +use crate::storage::url::alternative_paths; +use ic_certified_map::{fork, fork_hash, labeled, labeled_hash, AsHashTree, Hash, HashTree}; +use sha2::{Digest, Sha256}; + +impl CertifiedAssetHashes { + /// Returns the root_hash of the asset certification tree. + pub fn root_hash(&self) -> Hash { + fork_hash( + // NB: Labels added in lexicographic order. + &labeled_hash(LABEL_ASSETS_V1, &self.tree_v1.root_hash()), + &labeled_hash(LABEL_ASSETS_V2, &self.tree_v2.root_hash()), + ) + } + + pub fn witness_v1(&self, path: &str) -> HashTree { + let witness = self.tree_v1.witness(path.as_bytes()); + fork( + labeled(LABEL_ASSETS_V1, witness), + HashTree::Pruned(labeled_hash(LABEL_ASSETS_V2, &self.tree_v2.root_hash())), + ) + } + + pub fn witness_v2(&self, absolute_path: &str) -> HashTree { + assert!(absolute_path.starts_with('/')); + + let segments = nested_tree_path(absolute_path, EXACT_MATCH_TERMINATOR); + let witness = self.tree_v2.witness(&segments); + + fork( + HashTree::Pruned(labeled_hash(LABEL_ASSETS_V1, &self.tree_v1.root_hash())), + labeled(LABEL_ASSETS_V2, witness), + ) + } + + pub fn witness_rewrite_v2(&self, absolute_path: &str) -> HashTree { + assert!(absolute_path.starts_with('/')); + + // Witness incorrect url: e.g. /1234 + let segments = nested_tree_path(absolute_path, EXACT_MATCH_TERMINATOR); + let absence_proof = self.tree_v2.witness(&segments); + + // Fallback to search for non conflicting rewrites starting the search from root + let fallback_paths = fallback_paths(segments.clone()); + + // Witness fallback paths with the absence of proof to validate it can be rewritten + let combined_proof = fallback_paths + .into_iter() + .fold(absence_proof, |accumulator, path| { + let new_proof = self.tree_v2.witness(&[path]); + merge_hash_trees(accumulator, new_proof) + }); + + fork( + HashTree::Pruned(labeled_hash(LABEL_ASSETS_V1, &self.tree_v1.root_hash())), + labeled(LABEL_ASSETS_V2, combined_proof), + ) + } + + pub fn expr_path_v2( + &self, + absolute_path: &str, + rewrite_source: &Option, + ) -> Vec { + match rewrite_source { + None => nested_tree_expr_path(absolute_path, EXACT_MATCH_TERMINATOR), + Some(rewrite_source) => { + nested_tree_expr_path(rewrite_source, WILDCARD_MATCH_TERMINATOR) + } + } + } + + pub fn insert(&mut self, asset: &Asset, config: &StorageConfig) { + let full_path = asset.key.full_path.clone(); + + for encoding_type in ENCODING_CERTIFICATION_ORDER.iter() { + if let Some(encoding) = asset.encodings.get(*encoding_type) { + self.insert_v1(&full_path, encoding.sha256); + self.insert_v2( + &full_path, + &build_headers(asset, encoding, &encoding_type.to_string(), config), + RESPONSE_STATUS_CODE_200, + encoding.sha256, + ); + + return; + } + } + } + + fn insert_v1(&mut self, full_path: &FullPath, sha256: Hash) { + self.tree_v1.insert(full_path.clone(), sha256); + + let alt_paths = alternative_paths(full_path); + + match alt_paths { + None => (), + Some(alt_paths) => { + for alt_path in alt_paths { + self.tree_v1.insert(alt_path, sha256); + } + } + } + } + + fn insert_v2( + &mut self, + full_path: &FullPath, + headers: &[HeaderField], + status_code: StatusCode, + sha256: Hash, + ) { + self.tree_v2.insert( + &nested_tree_key( + full_path, + headers, + sha256, + EXACT_MATCH_TERMINATOR, + status_code, + ), + vec![], + ); + + let alt_paths = alternative_paths(full_path); + + match alt_paths { + None => (), + Some(alt_paths) => { + for alt_path in alt_paths { + self.tree_v2.insert( + &nested_tree_key( + &alt_path, + headers, + sha256, + EXACT_MATCH_TERMINATOR, + RESPONSE_STATUS_CODE_200, + ), + vec![], + ); + } + } + } + + // Rewrite ** to / + if *full_path == *ROOT_INDEX_HTML { + let has_404 = self + .tree_v2 + .contains_path(&nested_tree_path(ROOT_404_HTML, WILDCARD_MATCH_TERMINATOR)); + + if !has_404 { + self.insert_rewrite_into_tree_v2( + &ROOT_PATH.to_string(), + headers, + sha256, + RESPONSE_STATUS_CODE_200, + ); + } + } + + // Rewrite ** to /404 + if *full_path == *ROOT_404_HTML { + let index_tree_path = nested_tree_path(ROOT_INDEX_HTML, WILDCARD_MATCH_TERMINATOR); + + let has_index = self.tree_v2.contains_path(&index_tree_path); + + if has_index { + // Delete existing rewrite to root with /index.html in the tree to enter the new rewrite to /404.html + self.delete_from_tree_v2(&ROOT_PATH.to_string(), WILDCARD_MATCH_TERMINATOR); + } + + // TODO: RESPONSE_STATUS_CODE_404 service worker does not support 404 yet + self.insert_rewrite_into_tree_v2( + &ROOT_PATH.to_string(), + headers, + sha256, + RESPONSE_STATUS_CODE_200, + ); + } + } + + pub fn insert_redirect_v2( + &mut self, + full_path: &FullPath, + status_code: StatusCode, + location: &str, + ) { + let headers = build_redirect_headers(location); + + let sha256 = Sha256::digest(Vec::new().clone()).into(); + + self.insert_v2(full_path, &headers, status_code, sha256); + } + + pub fn insert_rewrite_v2( + &mut self, + full_path: &FullPath, + asset: &Asset, + config: &StorageConfig, + ) { + for encoding_type in ENCODING_CERTIFICATION_ORDER.iter() { + if let Some(encoding) = asset.encodings.get(*encoding_type) { + self.insert_rewrite_into_tree_v2( + full_path, + &build_headers(asset, encoding, &encoding_type.to_string(), config), + encoding.sha256, + RESPONSE_STATUS_CODE_200, + ); + + return; + } + } + } + + fn insert_rewrite_into_tree_v2( + &mut self, + full_path: &FullPath, + headers: &[HeaderField], + sha256: Hash, + status_code: StatusCode, + ) { + self.tree_v2.insert( + &nested_tree_key( + full_path, + headers, + sha256, + WILDCARD_MATCH_TERMINATOR, + status_code, + ), + vec![], + ); + } + + pub fn delete(&mut self, asset: &Asset) { + let full_path = asset.key.full_path.clone(); + + self.delete_v1(&full_path); + self.delete_v2(&full_path); + } + + fn delete_v1(&mut self, full_path: &String) { + self.tree_v1.delete(full_path.clone().as_bytes()); + + let alt_paths = alternative_paths(full_path); + + match alt_paths { + None => (), + Some(alt_paths) => { + for alt_path in alt_paths { + self.tree_v1.delete(alt_path.as_bytes()); + } + } + } + } + + fn delete_v2(&mut self, full_path: &FullPath) { + self.delete_from_tree_v2(full_path, EXACT_MATCH_TERMINATOR); + + let alt_paths = alternative_paths(full_path); + + match alt_paths { + None => (), + Some(alt_paths) => { + for alt_path in alt_paths { + self.delete_from_tree_v2(&alt_path, EXACT_MATCH_TERMINATOR); + } + } + } + + // Delete rewrite ** to /404 + if *full_path == *ROOT_404_HTML { + self.delete_from_tree_v2(&ROOT_PATH.to_string(), WILDCARD_MATCH_TERMINATOR); + } + + // Delete rewrite ** to / + if *full_path == *ROOT_INDEX_HTML { + let has_404 = self + .tree_v2 + .contains_path(&nested_tree_path(ROOT_404_HTML, WILDCARD_MATCH_TERMINATOR)); + + if !has_404 { + self.delete_from_tree_v2(&ROOT_PATH.to_string(), WILDCARD_MATCH_TERMINATOR); + } + } + } + + fn delete_from_tree_v2(&mut self, full_path: &FullPath, terminator: &str) { + self.tree_v2 + .delete(&nested_tree_path(full_path, terminator)); + } +} diff --git a/src/satellite/src/storage/certification/mod.rs b/src/satellite/src/storage/certification/mod.rs new file mode 100644 index 000000000..b97485f11 --- /dev/null +++ b/src/satellite/src/storage/certification/mod.rs @@ -0,0 +1,6 @@ +pub mod cert; +mod constants; +mod impls; +mod tree; +mod tree_utils; +pub mod types; diff --git a/src/satellite/src/storage/certification/tree.rs b/src/satellite/src/storage/certification/tree.rs new file mode 100644 index 000000000..b758f57ad --- /dev/null +++ b/src/satellite/src/storage/certification/tree.rs @@ -0,0 +1,203 @@ +#![allow(dead_code)] // we don't need all the features provided here + +/// Source: https://github.com/dfinity/sdk/blob/master/src/canisters/frontend/ic-certified-assets/src/asset_certification/tree.rs +use ic_certified_map::{AsHashTree, HashTree, RbTree}; + +pub trait NestedTreeKeyRequirements: Clone + AsRef<[u8]> + 'static {} +pub trait NestedTreeValueRequirements: AsHashTree + 'static {} +impl NestedTreeKeyRequirements for T where T: Clone + AsRef<[u8]> + 'static {} +impl NestedTreeValueRequirements for T where T: AsHashTree + 'static {} + +#[derive(Debug, Clone)] +pub enum NestedTree { + Leaf(V), + Nested(RbTree>), +} + +impl Default for NestedTree { + fn default() -> Self { + NestedTree::Nested(RbTree::>::new()) + } +} + +impl AsHashTree for NestedTree { + fn root_hash(&self) -> ic_certified_map::Hash { + match self { + NestedTree::Leaf(a) => a.root_hash(), + NestedTree::Nested(tree) => tree.root_hash(), + } + } + + fn as_hash_tree(&self) -> HashTree<'_> { + match self { + NestedTree::Leaf(a) => a.as_hash_tree(), + NestedTree::Nested(tree) => tree.as_hash_tree(), + } + } +} + +impl NestedTree { + #[allow(dead_code)] + pub fn get(&self, path: &[K]) -> Option<&V> { + if let Some(key) = path.get(0) { + match self { + NestedTree::Leaf(_) => None, + NestedTree::Nested(tree) => tree + .get(key.as_ref()) + .and_then(|child| child.get(&path[1..])), + } + } else { + match self { + NestedTree::Leaf(value) => Some(value), + NestedTree::Nested(_) => None, + } + } + } + + /// Returns true if there is a leaf at the specified path + pub fn contains_leaf(&self, path: &[K]) -> bool { + if let Some(key) = path.get(0) { + match self { + NestedTree::Leaf(_) => false, + NestedTree::Nested(tree) => tree + .get(key.as_ref()) + .map(|child| child.contains_leaf(&path[1..])) + .unwrap_or(false), + } + } else { + matches!(self, NestedTree::Leaf(_)) + } + } + + /// Returns true if there is a leaf or a subtree at the specified path + pub fn contains_path(&self, path: &[K]) -> bool { + if let Some(key) = path.get(0) { + match self { + NestedTree::Leaf(_) => false, + NestedTree::Nested(tree) => tree + .get(key.as_ref()) + .map(|child| child.contains_path(&path[1..])) + .unwrap_or(false), + } + } else { + true + } + } + + pub fn insert(&mut self, path: &[K], value: V) { + if let Some(key) = path.get(0) { + match self { + NestedTree::Leaf(_) => { + *self = NestedTree::default(); + self.insert(path, value); + } + NestedTree::Nested(tree) => { + if tree.get(key.as_ref()).is_some() { + tree.modify(key.as_ref(), |child| child.insert(&path[1..], value)); + } else { + tree.insert(key.clone(), NestedTree::default()); + self.insert(path, value); + } + } + } + } else { + *self = NestedTree::Leaf(value); + } + } + + pub fn delete(&mut self, path: &[K]) { + if let Some(key) = path.get(0) { + match self { + NestedTree::Leaf(_) => {} + NestedTree::Nested(tree) => { + tree.modify(key.as_ref(), |child| child.delete(&path[1..])); + } + } + } else { + *self = NestedTree::default(); + } + } + + pub fn witness(&self, path: &[K]) -> HashTree { + if let Some(key) = path.get(0) { + match self { + NestedTree::Leaf(value) => value.as_hash_tree(), + NestedTree::Nested(tree) => { + tree.nested_witness(key.as_ref(), |tree| tree.witness(&path[1..])) + } + } + } else { + self.as_hash_tree() + } + } +} + +pub fn merge_hash_trees<'a>(lhs: HashTree<'a>, rhs: HashTree<'a>) -> HashTree<'a> { + use HashTree::{Empty, Fork, Labeled, Leaf, Pruned}; + + match (lhs, rhs) { + (Pruned(l), Pruned(r)) => { + if l != r { + panic!("merge_hash_trees: inconsistent hashes"); + } + Pruned(l) + } + (Pruned(_), r) => r, + (l, Pruned(_)) => l, + (Fork(l), Fork(r)) => Fork(Box::new(( + merge_hash_trees(l.0, r.0), + merge_hash_trees(l.1, r.1), + ))), + (Labeled(l_label, l), Labeled(r_label, r)) => { + if l_label != r_label { + panic!("merge_hash_trees: inconsistent hash tree labels"); + } + Labeled(l_label, Box::new(merge_hash_trees(*l, *r))) + } + (Empty, Empty) => Empty, + (Leaf(l), Leaf(r)) => { + if l != r { + panic!("merge_hash_trees: inconsistent leaves"); + } + Leaf(l) + } + (_l, _r) => { + panic!("merge_hash_trees: inconsistent tree structure"); + } + } +} + +#[test] +fn nested_tree_operation() { + let mut tree: NestedTree<&str, Vec> = NestedTree::default(); + // insertion + tree.insert(&["one", "two"], vec![2]); + tree.insert(&["one", "three"], vec![3]); + assert_eq!(tree.get(&["one", "two"]), Some(&vec![2])); + assert_eq!(tree.get(&["one", "two", "three"]), None); + assert_eq!(tree.get(&["one"]), None); + assert!(tree.contains_leaf(&["one", "two"])); + assert!(tree.contains_path(&["one"])); + assert!(!tree.contains_leaf(&["one", "two", "three"])); + assert!(!tree.contains_path(&["one", "two", "three"])); + assert!(!tree.contains_leaf(&["one"])); + + // deleting non-existent key doesn't do anything + tree.delete(&["one", "two", "three"]); + assert_eq!(tree.get(&["one", "two"]), Some(&vec![2])); + assert!(tree.contains_leaf(&["one", "two"])); + + // deleting existing key works + tree.delete(&["one", "three"]); + assert_eq!(tree.get(&["one", "two"]), Some(&vec![2])); + assert_eq!(tree.get(&["one", "three"]), None); + assert!(tree.contains_leaf(&["one", "two"])); + assert!(!tree.contains_leaf(&["one", "three"])); + + // deleting subtree works + tree.delete(&["one"]); + assert_eq!(tree.get(&["one", "two"]), None); + assert_eq!(tree.get(&["one"]), None); + assert!(!tree.contains_leaf(&["one", "two"])); + assert!(!tree.contains_leaf(&["one"])); +} diff --git a/src/satellite/src/storage/certification/tree_utils.rs b/src/satellite/src/storage/certification/tree_utils.rs new file mode 100644 index 000000000..b08c4f182 --- /dev/null +++ b/src/satellite/src/storage/certification/tree_utils.rs @@ -0,0 +1,114 @@ +use crate::storage::certification::constants::{ + EXACT_MATCH_TERMINATOR, IC_CERTIFICATE_EXPRESSION, IC_CERTIFICATE_EXPRESSION_HEADER, + IC_STATUS_CODE_PSEUDO_HEADER, LABEL_HTTP_EXPR, WILDCARD_MATCH_TERMINATOR, +}; +use crate::storage::http::types::{HeaderField, StatusCode}; +use crate::storage::types::state::FullPath; +use crate::types::core::Blob; +use ic_certified_map::Hash; +use ic_representation_independent_hash::{representation_independent_hash, Value}; +use sha2::{Digest, Sha256}; + +pub fn nested_tree_key( + full_path: &FullPath, + headers: &[HeaderField], + body_hash: Hash, + terminator: &str, + status_code: StatusCode, +) -> Vec { + let mut segments = nested_tree_path(full_path, terminator); + + let expr_hash: Hash = Sha256::digest(response_headers_expression(headers)).into(); + segments.push(Vec::from(expr_hash.as_slice())); + + segments.push(vec![]); + segments.push(Vec::from(response_hash(headers, status_code, &body_hash))); + + segments +} + +pub fn nested_tree_path(full_path: &str, terminator: &str) -> Vec { + assert!(full_path.starts_with('/')); + + let mut segments: Vec = full_path + .split('/') + .map(str::as_bytes) + .map(Vec::from) + .collect(); + segments.remove(0); // remove leading empty string due to absolute path + segments.push(terminator.as_bytes().to_vec()); + + segments +} + +pub fn fallback_paths(paths: Vec) -> Vec { + let mut fallback_paths = Vec::new(); + + // starting at 1 because "http_expr" is always the starting element + for i in 1..paths.len() { + let mut without_trailing_slash: Vec = paths.as_slice()[0..i].to_vec(); + let mut with_trailing_slash = without_trailing_slash.clone(); + without_trailing_slash.push(EXACT_MATCH_TERMINATOR.as_bytes().to_vec()); + with_trailing_slash.push("".as_bytes().to_vec()); + with_trailing_slash.push(WILDCARD_MATCH_TERMINATOR.as_bytes().to_vec()); + + fallback_paths.extend(without_trailing_slash); + fallback_paths.extend(with_trailing_slash); + } + + fallback_paths +} + +pub fn nested_tree_expr_path(absolute_path: &str, terminator: &str) -> Vec { + assert!(absolute_path.starts_with('/')); + + // "/" => ["", ""] + // "/index.html" => ["", "index.html"] + // "/hello/index.html" => ["", "hello", "index.html"] + let mut path: Vec = absolute_path.split('/').map(str::to_string).collect(); + // replace the first empty split segment (due to absolute path) with "http_expr" + *path.get_mut(0).unwrap() = LABEL_HTTP_EXPR.to_string(); + path.push(terminator.to_string()); + path +} + +fn response_hash(headers: &[HeaderField], status_code: StatusCode, body_hash: &Hash) -> Hash { + // certification v2 spec: + // Response hash is the hash of the concatenation of + // - representation-independent hash of headers + // - hash of the response body + // + // The representation-independent hash of headers consist of + // - all certified headers (here all headers), plus + // - synthetic header `:ic-cert-status` with value + + let mut certified_headers = headers + .iter() + .map(|HeaderField(header, value)| { + (header.to_ascii_lowercase(), Value::String(value.clone())) + }) + .collect::>(); + + certified_headers.push(( + IC_CERTIFICATE_EXPRESSION_HEADER.to_ascii_lowercase(), + Value::String(response_headers_expression(headers)), + )); + + certified_headers.push(( + IC_STATUS_CODE_PSEUDO_HEADER.to_string(), + Value::Number(status_code.into()), + )); + + let header_hash = representation_independent_hash(&certified_headers); + Sha256::digest([header_hash.as_ref(), body_hash].concat()).into() +} + +pub fn response_headers_expression(headers: &[HeaderField]) -> String { + let headers = headers + .iter() + .map(|field: &HeaderField| format!("\"{}\"", field.0)) + .collect::>() + .join(","); + + IC_CERTIFICATE_EXPRESSION.replace("{headers}", &headers) +} diff --git a/src/satellite/src/storage/certification/types.rs b/src/satellite/src/storage/certification/types.rs new file mode 100644 index 000000000..a02ae84c8 --- /dev/null +++ b/src/satellite/src/storage/certification/types.rs @@ -0,0 +1,12 @@ +pub mod certified { + use crate::storage::certification::tree::NestedTree; + use crate::types::core::Blob; + use ic_certified_map::{Hash, RbTree}; + use std::clone::Clone; + + #[derive(Default, Clone)] + pub struct CertifiedAssetHashes { + pub tree_v1: RbTree, + pub tree_v2: NestedTree, + } +} diff --git a/src/satellite/src/storage/constants.rs b/src/satellite/src/storage/constants.rs index bbfc0e55f..9f04ca29f 100644 --- a/src/satellite/src/storage/constants.rs +++ b/src/satellite/src/storage/constants.rs @@ -1,3 +1,5 @@ +use crate::storage::http::types::StatusCode; + pub static ASSET_ENCODING_NO_COMPRESSION: &str = "identity"; pub static ENCODING_CERTIFICATION_ORDER: &[&str] = &[ ASSET_ENCODING_NO_COMPRESSION, @@ -7,4 +9,14 @@ pub static ENCODING_CERTIFICATION_ORDER: &[&str] = &[ "br", ]; pub static BN_WELL_KNOWN_CUSTOM_DOMAINS: &str = "/.well-known/ic-domains"; -pub static REWRITE_TO_ROOT_INDEX_HTML: (&str, &str) = ("**", "/index.html"); + +pub static ROOT_PATH: &str = "/"; +pub static ROOT_INDEX_HTML: &str = "/index.html"; +pub static ROOT_404_HTML: &str = "/404.html"; +pub static ROOT_PATHS: [&str; 5] = ["/index.html", "/index", "/", "/404", "/404.html"]; + +pub static RESPONSE_STATUS_CODE_200: StatusCode = 200; +pub static RESPONSE_STATUS_CODE_404: StatusCode = 404; +pub static RESPONSE_STATUS_CODE_405: StatusCode = 405; +pub static RESPONSE_STATUS_CODE_406: StatusCode = 406; +pub static RESPONSE_STATUS_CODE_500: StatusCode = 500; diff --git a/src/satellite/src/storage/custom_domains.rs b/src/satellite/src/storage/custom_domains.rs index c0a720412..108d13708 100644 --- a/src/satellite/src/storage/custom_domains.rs +++ b/src/satellite/src/storage/custom_domains.rs @@ -1,5 +1,5 @@ use crate::storage::constants::{ASSET_ENCODING_NO_COMPRESSION, BN_WELL_KNOWN_CUSTOM_DOMAINS}; -use crate::storage::types::http::HeaderField; +use crate::storage::http::types::HeaderField; use crate::storage::types::store::{Asset, AssetEncoding, AssetKey}; use ic_cdk::api::time; use ic_cdk::id; diff --git a/src/satellite/src/storage/http.rs b/src/satellite/src/storage/http.rs deleted file mode 100644 index 9768b942b..000000000 --- a/src/satellite/src/storage/http.rs +++ /dev/null @@ -1,167 +0,0 @@ -use crate::memory::STATE; -use crate::rules::types::rules::Memory; -use globset::Glob; -use hex::encode; -use ic_cdk::id; -use serde_bytes::ByteBuf; - -use crate::storage::cert::build_asset_certificate_header; -use crate::storage::constants::ASSET_ENCODING_NO_COMPRESSION; -use crate::storage::store::get_config; -use crate::storage::types::config::StorageConfig; -use crate::storage::types::http::{ - CallbackFunc, HeaderField, HttpResponse, StreamingCallbackToken, StreamingStrategy, -}; -use crate::storage::types::state::StorageRuntimeState; -use crate::storage::types::store::{Asset, AssetEncoding, AssetKey, EncodingType}; - -pub fn streaming_strategy( - key: &AssetKey, - encoding: &AssetEncoding, - encoding_type: &str, - headers: &[HeaderField], - memory: &Memory, -) -> Option { - let streaming_token: Option = - create_token(key, 0, encoding, encoding_type, headers, memory); - - streaming_token.map(|streaming_token| StreamingStrategy::Callback { - callback: CallbackFunc::new(id(), "http_request_streaming_callback".to_string()), - token: streaming_token, - }) -} - -pub fn create_token( - key: &AssetKey, - chunk_index: usize, - encoding: &AssetEncoding, - encoding_type: &str, - headers: &[HeaderField], - memory: &Memory, -) -> Option { - if chunk_index + 1 >= encoding.content_chunks.len() { - return None; - } - - Some(StreamingCallbackToken { - full_path: key.full_path.clone(), - token: key.token.clone(), - headers: headers.to_owned(), - index: chunk_index + 1, - sha256: Some(ByteBuf::from(encoding.sha256)), - encoding_type: encoding_type.to_owned(), - memory: memory.clone(), - }) -} - -pub fn build_headers( - url: &str, - asset: &Asset, - encoding: &AssetEncoding, - encoding_type: &EncodingType, -) -> Result, &'static str> { - let certified_header = build_certified_headers(url)?; - - let mut headers = asset.headers.clone(); - - // The Accept-Ranges HTTP response header is a marker used by the server to advertise its support for partial requests from the client for file downloads. - headers.push(HeaderField( - "accept-ranges".to_string(), - "bytes".to_string(), - )); - - headers.push(HeaderField( - "etag".to_string(), - format!("\"{}\"", encode(encoding.sha256)), - )); - - // Header for certification - headers.push(certified_header); - - if encoding_type.clone() != *ASSET_ENCODING_NO_COMPRESSION { - headers.push(HeaderField( - "Content-Encoding".to_string(), - encoding_type.to_string(), - )); - } - - // Headers provided as configuration of the storage - let config_headers = build_config_headers(url); - - Ok([headers, config_headers, security_headers()].concat()) -} - -fn build_config_headers(requested_path: &str) -> Vec { - let StorageConfig { - headers: config_headers, - rewrites: _, - } = get_config(); - - config_headers - .iter() - .filter(|(source, _)| { - let glob = Glob::new(source); - - match glob { - Err(_) => false, - Ok(glob) => { - let matcher = glob.compile_matcher(); - matcher.is_match(requested_path) - } - } - }) - .flat_map(|(_, headers)| headers.clone()) - .collect() -} - -fn build_certified_headers(url: &str) -> Result { - STATE.with(|state| build_certified_headers_impl(url, &state.borrow().runtime.storage)) -} - -fn build_certified_headers_impl( - url: &str, - state: &StorageRuntimeState, -) -> Result { - build_asset_certificate_header(&state.asset_hashes, url.to_owned()) -} - -// Source: NNS-dapp -/// List of recommended security headers as per https://owasp.org/www-satellite-secure-headers/ -/// These headers enable browser security features (like limit access to platform apis and set -/// iFrame policies, etc.). -fn security_headers() -> Vec { - vec![ - HeaderField("X-Frame-Options".to_string(), "DENY".to_string()), - HeaderField("X-Content-Type-Options".to_string(), "nosniff".to_string()), - HeaderField( - "Strict-Transport-Security".to_string(), - "max-age=31536000 ; includeSubDomains".to_string(), - ), - // "Referrer-Policy: no-referrer" would be more strict, but breaks local dev deployment - // same-origin is still ok from a security perspective - HeaderField("Referrer-Policy".to_string(), "same-origin".to_string()), - ] -} - -pub fn build_encodings(headers: Vec) -> Vec { - let mut encodings: Vec = vec![]; - for HeaderField(name, value) in headers.iter() { - if name.eq_ignore_ascii_case("Accept-Encoding") { - for v in value.split(',') { - encodings.push(v.trim().to_string()); - } - } - } - encodings.push(ASSET_ENCODING_NO_COMPRESSION.to_string()); - - encodings -} - -pub fn error_response(status_code: u16, body: String) -> HttpResponse { - HttpResponse { - body: body.as_bytes().to_vec(), - headers: Vec::new(), - status_code, - streaming_strategy: None, - } -} diff --git a/src/satellite/src/storage/http/headers.rs b/src/satellite/src/storage/http/headers.rs new file mode 100644 index 000000000..1865e62a7 --- /dev/null +++ b/src/satellite/src/storage/http/headers.rs @@ -0,0 +1,84 @@ +use crate::storage::constants::ASSET_ENCODING_NO_COMPRESSION; +use crate::storage::http::types::HeaderField; +use crate::storage::types::config::StorageConfig; +use crate::storage::types::store::{Asset, AssetEncoding, EncodingType}; +use crate::storage::url::matching_urls; +use hex::encode; + +pub fn build_headers( + asset: &Asset, + encoding: &AssetEncoding, + encoding_type: &EncodingType, + config: &StorageConfig, +) -> Vec { + let mut headers = asset.headers.clone(); + + // The Accept-Ranges HTTP response header is a marker used by the server to advertise its support for partial requests from the client for file downloads. + headers.push(HeaderField( + "accept-ranges".to_string(), + "bytes".to_string(), + )); + + headers.push(HeaderField( + "etag".to_string(), + format!("\"{}\"", encode(encoding.sha256)), + )); + + // Headers for security + headers.extend(security_headers()); + + if encoding_type.clone() != *ASSET_ENCODING_NO_COMPRESSION { + headers.push(HeaderField( + "Content-Encoding".to_string(), + encoding_type.to_string(), + )); + } + + // Headers build from the configuration + let config_headers = build_config_headers(&asset.key.full_path, config); + headers.extend(config_headers); + + headers +} + +pub fn build_redirect_headers(location: &str) -> Vec { + let mut headers = Vec::new(); + + // Headers for security + headers.extend(security_headers()); + + headers.push(HeaderField("Location".to_string(), location.to_string())); + + headers +} + +// Source: NNS-dapp +/// List of recommended security headers as per https://owasp.org/www-satellite-secure-headers/ +/// These headers enable browser security features (like limit access to platform apis and set +/// iFrame policies, etc.). +fn security_headers() -> Vec { + vec![ + HeaderField("X-Frame-Options".to_string(), "DENY".to_string()), + HeaderField("X-Content-Type-Options".to_string(), "nosniff".to_string()), + HeaderField( + "Strict-Transport-Security".to_string(), + "max-age=31536000 ; includeSubDomains".to_string(), + ), + // "Referrer-Policy: no-referrer" would be more strict, but breaks local dev deployment + // same-origin is still ok from a security perspective + HeaderField("Referrer-Policy".to_string(), "same-origin".to_string()), + ] +} + +pub fn build_config_headers( + requested_path: &str, + StorageConfig { + headers: config_headers, + .. + }: &StorageConfig, +) -> Vec { + matching_urls(requested_path, config_headers) + .iter() + .flat_map(|(_, headers)| headers.clone()) + .collect() +} diff --git a/src/satellite/src/storage/http/mod.rs b/src/satellite/src/storage/http/mod.rs new file mode 100644 index 000000000..79a54dcec --- /dev/null +++ b/src/satellite/src/storage/http/mod.rs @@ -0,0 +1,4 @@ +pub mod headers; +pub mod response; +pub mod types; +pub mod utils; diff --git a/src/satellite/src/storage/http/response.rs b/src/satellite/src/storage/http/response.rs new file mode 100644 index 000000000..154144638 --- /dev/null +++ b/src/satellite/src/storage/http/response.rs @@ -0,0 +1,120 @@ +use crate::rules::types::rules::Memory; +use crate::storage::constants::{ + RESPONSE_STATUS_CODE_404, RESPONSE_STATUS_CODE_406, RESPONSE_STATUS_CODE_500, +}; +use crate::storage::http::types::{HeaderField, HttpResponse, StatusCode}; +use crate::storage::http::utils::{ + build_encodings, build_response_headers, build_response_redirect_headers, streaming_strategy, +}; +use crate::storage::types::store::Asset; + +use crate::storage::store::get_content_chunks; +use crate::storage::types::config::StorageConfigRedirect; + +pub fn build_asset_response( + requested_url: String, + requested_headers: Vec, + certificate_version: Option, + asset: Option<(Asset, Memory)>, + rewrite_source: Option, + status_code: StatusCode, +) -> HttpResponse { + match asset { + Some((asset, memory)) => { + let encodings = build_encodings(requested_headers); + + for encoding_type in encodings.iter() { + if let Some(encoding) = asset.encodings.get(encoding_type) { + let headers = build_response_headers( + &requested_url, + &asset, + encoding, + encoding_type, + &certificate_version, + &rewrite_source, + ); + + let Asset { + key, + headers: _, + encodings: _, + created_at: _, + updated_at: _, + } = &asset; + + match headers { + Ok(headers) => { + let body = get_content_chunks(encoding, 0, &memory); + + // TODO: support for HTTP response 304 + // On hold til DFINITY foundation implements: + // "Add etag support to icx-proxy" - https://dfinity.atlassian.net/browse/BOUN-446 + // See const STATUS_CODES_TO_CERTIFY: [u16; 2] = [200, 304]; in sdk certified asset canister for implementation reference + + match body { + Some(body) => { + return HttpResponse { + body: body.clone(), + headers: headers.clone(), + status_code, + streaming_strategy: streaming_strategy( + key, + encoding, + encoding_type, + &headers, + &memory, + ), + } + } + None => { + error_response( + RESPONSE_STATUS_CODE_500, + "No chunks found.".to_string(), + ); + } + } + } + Err(err) => { + return error_response( + RESPONSE_STATUS_CODE_406, + ["Permission denied. Invalid headers. ", err].join(""), + ); + } + } + } + } + + error_response( + RESPONSE_STATUS_CODE_500, + "No asset encoding found.".to_string(), + ) + } + None => error_response(RESPONSE_STATUS_CODE_404, "No asset found.".to_string()), + } +} + +pub fn build_redirect_response( + requested_url: String, + certificate_version: Option, + redirect: &StorageConfigRedirect, +) -> HttpResponse { + let headers = + build_response_redirect_headers(&requested_url, &redirect.location, &certificate_version) + .unwrap(); + + HttpResponse { + body: Vec::new().clone(), + headers: headers.clone(), + status_code: redirect.status_code, + streaming_strategy: None, + } +} + +pub fn error_response(status_code: StatusCode, body: String) -> HttpResponse { + HttpResponse { + body: body.as_bytes().to_vec(), + headers: Vec::new(), + status_code, + streaming_strategy: None, + } +} diff --git a/src/satellite/src/storage/http/types.rs b/src/satellite/src/storage/http/types.rs new file mode 100644 index 000000000..d0459c7ba --- /dev/null +++ b/src/satellite/src/storage/http/types.rs @@ -0,0 +1,55 @@ +use crate::rules::types::rules::Memory; +use crate::storage::types::store::EncodingType; +use crate::types::core::Blob; +use candid::{define_function, CandidType}; +use serde::{Deserialize, Serialize}; +use serde_bytes::ByteBuf; + +#[derive(CandidType, Serialize, Deserialize, Clone)] +pub struct HeaderField(pub String, pub String); + +pub type StatusCode = u16; + +#[derive(CandidType, Deserialize, Clone)] +pub struct HttpRequest { + pub url: String, + pub method: String, + pub headers: Vec, + pub body: Blob, + pub certificate_version: Option, +} + +#[derive(CandidType, Deserialize, Clone)] +pub struct HttpResponse { + pub body: Blob, + pub headers: Vec, + pub status_code: StatusCode, + pub streaming_strategy: Option, +} + +define_function!(pub CallbackFunc : () -> () query); + +#[derive(CandidType, Deserialize, Clone)] +pub enum StreamingStrategy { + Callback { + token: StreamingCallbackToken, + callback: CallbackFunc, + }, +} + +#[derive(CandidType, Deserialize, Clone)] +pub struct StreamingCallbackToken { + pub full_path: String, + pub token: Option, + pub headers: Vec, + pub sha256: Option, + pub index: usize, + pub encoding_type: EncodingType, + pub memory: Memory, +} + +#[derive(CandidType, Deserialize, Clone)] +pub struct StreamingCallbackHttpResponse { + pub body: Blob, + pub token: Option, +} diff --git a/src/satellite/src/storage/http/utils.rs b/src/satellite/src/storage/http/utils.rs new file mode 100644 index 000000000..62cbca494 --- /dev/null +++ b/src/satellite/src/storage/http/utils.rs @@ -0,0 +1,139 @@ +use crate::memory::STATE; +use crate::rules::types::rules::Memory; +use crate::storage::certification::cert::{ + build_asset_certificate_header, build_certified_expression, +}; +use crate::storage::constants::ASSET_ENCODING_NO_COMPRESSION; +use crate::storage::http::headers::{build_headers, build_redirect_headers}; +use crate::storage::http::types::{ + CallbackFunc, HeaderField, StreamingCallbackToken, StreamingStrategy, +}; +use crate::storage::state::get_config; +use crate::storage::types::state::StorageRuntimeState; +use crate::storage::types::store::{Asset, AssetEncoding, AssetKey, EncodingType}; +use ic_cdk::id; +use serde_bytes::ByteBuf; + +pub fn streaming_strategy( + key: &AssetKey, + encoding: &AssetEncoding, + encoding_type: &str, + headers: &[HeaderField], + memory: &Memory, +) -> Option { + let streaming_token: Option = + create_token(key, 0, encoding, encoding_type, headers, memory); + + streaming_token.map(|streaming_token| StreamingStrategy::Callback { + callback: CallbackFunc::new(id(), "http_request_streaming_callback".to_string()), + token: streaming_token, + }) +} + +pub fn create_token( + key: &AssetKey, + chunk_index: usize, + encoding: &AssetEncoding, + encoding_type: &str, + headers: &[HeaderField], + memory: &Memory, +) -> Option { + if chunk_index + 1 >= encoding.content_chunks.len() { + return None; + } + + Some(StreamingCallbackToken { + full_path: key.full_path.clone(), + token: key.token.clone(), + headers: headers.to_owned(), + index: chunk_index + 1, + sha256: Some(ByteBuf::from(encoding.sha256)), + encoding_type: encoding_type.to_owned(), + memory: memory.clone(), + }) +} + +pub fn build_response_headers( + url: &str, + asset: &Asset, + encoding: &AssetEncoding, + encoding_type: &EncodingType, + certificate_version: &Option, + rewrite_source: &Option, +) -> Result, &'static str> { + let config = get_config(); + + let asset_headers = build_headers(asset, encoding, encoding_type, &config); + + extend_headers_with_certification(asset_headers, url, certificate_version, rewrite_source) +} + +pub fn build_response_redirect_headers( + url: &str, + location: &str, + certificate_version: &Option, +) -> Result, &'static str> { + let asset_headers = build_redirect_headers(location); + + extend_headers_with_certification(asset_headers, url, certificate_version, &None) +} + +fn extend_headers_with_certification( + asset_headers: Vec, + url: &str, + certificate_version: &Option, + rewrite_source: &Option, +) -> Result, &'static str> { + let certified_header = build_certified_headers(url, certificate_version, rewrite_source)?; + let certified_expression = build_certified_expression(&asset_headers, certificate_version)?; + + match certified_expression { + None => Ok([asset_headers, vec![certified_header]].concat()), + Some(certified_expression) => { + Ok([asset_headers, vec![certified_header, certified_expression]].concat()) + } + } +} + +fn build_certified_headers( + url: &str, + certificate_version: &Option, + rewrite_source: &Option, +) -> Result { + STATE.with(|state| { + build_certified_headers_impl( + url, + certificate_version, + rewrite_source, + &state.borrow().runtime.storage, + ) + }) +} + +fn build_certified_headers_impl( + url: &str, + certificate_version: &Option, + rewrite_source: &Option, + state: &StorageRuntimeState, +) -> Result { + build_asset_certificate_header( + &state.asset_hashes, + url.to_owned(), + certificate_version, + rewrite_source, + ) +} + +pub fn build_encodings(headers: Vec) -> Vec { + let mut encodings: Vec = vec![]; + for HeaderField(name, value) in headers.iter() { + if name.eq_ignore_ascii_case("Accept-Encoding") { + for v in value.split(',') { + encodings.push(v.trim().to_string()); + } + } + } + encodings.push(ASSET_ENCODING_NO_COMPRESSION.to_string()); + + encodings +} diff --git a/src/satellite/src/storage/impls.rs b/src/satellite/src/storage/impls.rs index f433049a4..9c239b69b 100644 --- a/src/satellite/src/storage/impls.rs +++ b/src/satellite/src/storage/impls.rs @@ -1,3 +1,8 @@ +use crate::storage::types::config::{StorageConfig, StorageConfigRedirects}; +use crate::storage::types::interface::AssetNoContent; +use crate::storage::types::state::{StableEncodingChunkKey, StableKey}; +use crate::storage::types::store::{Asset, AssetEncoding}; +use crate::types::core::{Blob, Compare}; use ic_cdk::api::time; use ic_stable_structures::storable::Bound; use ic_stable_structures::Storable; @@ -6,54 +11,6 @@ use shared::serializers::{deserialize_from_bytes, serialize_to_bytes}; use std::borrow::Cow; use std::cmp::Ordering; -use crate::storage::constants::ENCODING_CERTIFICATION_ORDER; -use crate::storage::types::assets::AssetHashes; -use crate::storage::types::interface::AssetNoContent; -use crate::storage::types::state::{StableEncodingChunkKey, StableKey}; -use crate::storage::types::store::{Asset, AssetEncoding}; -use crate::storage::url::alternative_paths; -use crate::types::core::{Blob, Compare}; - -impl AssetHashes { - pub(crate) fn insert(&mut self, asset: &Asset) { - let full_path = asset.key.full_path.clone(); - - for encoding_type in ENCODING_CERTIFICATION_ORDER.iter() { - if let Some(encoding) = asset.encodings.get(*encoding_type) { - self.tree.insert(full_path.clone(), encoding.sha256); - - let alt_paths = alternative_paths(&full_path); - - match alt_paths { - None => (), - Some(alt_paths) => { - for alt_path in alt_paths { - self.tree.insert(alt_path, encoding.sha256); - } - } - } - - return; - } - } - } - - pub(crate) fn delete(&mut self, full_path: &String) { - self.tree.delete(full_path.clone().as_bytes()); - - let alt_paths = alternative_paths(full_path); - - match alt_paths { - None => (), - Some(alt_paths) => { - for alt_path in alt_paths { - self.tree.delete(alt_path.as_bytes()); - } - } - } - } -} - impl From<&Vec> for AssetEncoding { fn from(content_chunks: &Vec) -> Self { let mut total_length: u128 = 0; @@ -77,6 +34,14 @@ impl From<&Vec> for AssetEncoding { } } +impl StorageConfig { + pub fn unwrap_redirects(&self) -> StorageConfigRedirects { + self.redirects + .clone() + .unwrap_or(StorageConfigRedirects::default()) + } +} + impl Compare for AssetNoContent { fn cmp_updated_at(&self, other: &Self) -> Ordering { self.updated_at.cmp(&other.updated_at) diff --git a/src/satellite/src/storage/mod.rs b/src/satellite/src/storage/mod.rs index 9cacd84bc..cbdf8061d 100644 --- a/src/satellite/src/storage/mod.rs +++ b/src/satellite/src/storage/mod.rs @@ -1,9 +1,10 @@ -mod cert; -mod constants; +mod certification; +pub mod constants; mod custom_domains; pub mod http; pub mod impls; pub mod rewrites; +pub mod routing; mod runtime; mod state; pub mod store; diff --git a/src/satellite/src/storage/rewrites.rs b/src/satellite/src/storage/rewrites.rs index 7453304ce..2d869a5c3 100644 --- a/src/satellite/src/storage/rewrites.rs +++ b/src/satellite/src/storage/rewrites.rs @@ -1,31 +1,64 @@ -use crate::storage::constants::REWRITE_TO_ROOT_INDEX_HTML; -use globset::Glob; +use crate::storage::constants::ROOT_PATHS; +use crate::storage::types::config::{StorageConfig, StorageConfigRedirect}; +use crate::storage::url::{matching_urls as matching_urls_utils, separator}; +use regex::Regex; +use std::cmp::Ordering; use std::collections::HashMap; -use crate::storage::types::config::{StorageConfig, StorageConfigRewrites}; - -pub fn init_rewrites() -> StorageConfigRewrites { - let (source, destination) = REWRITE_TO_ROOT_INDEX_HTML; - HashMap::from([(source.to_string(), destination.to_string())]) -} - -pub fn rewrite_url(requested_path: &str, config: &StorageConfig) -> Option { +pub fn rewrite_url(requested_path: &str, config: &StorageConfig) -> Option<(String, String)> { let StorageConfig { headers: _, rewrites, + redirects: _, } = config; - let rewrite = rewrites.iter().find(|(source, _)| { - let glob = Glob::new(source); + let matches = matching_urls(requested_path, rewrites); + + matches + .first() + .map(|(source, destination)| (rewrite_source_to_path(source).clone(), destination.clone())) +} + +pub fn rewrite_source_to_path(source: &String) -> String { + [separator(source.as_str()), source] + .join("") + .replace('*', "") +} + +pub fn is_html_route(path: &str) -> bool { + let re = Regex::new(r"^(?:/|(/[^/.]*(\.(html|htm))?(/[^/.]*)?)*)$").unwrap(); + re.is_match(path) +} + +pub fn is_root_path(path: &str) -> bool { + ROOT_PATHS.contains(&path) +} + +pub fn redirect_url(requested_path: &str, config: &StorageConfig) -> Option { + let redirects = config.unwrap_redirects(); + + let matches = matching_urls(requested_path, &redirects); + + matches.first().map(|(_, redirect)| redirect.clone()) +} + +fn matching_urls(requested_path: &str, config: &HashMap) -> Vec<(String, T)> { + let mut matches: Vec<(String, T)> = matching_urls_utils(requested_path, config); + + matches.sort_by(|(a, _), (b, _)| { + let a_parts: Vec<&str> = a.split('/').collect(); + let b_parts: Vec<&str> = b.split('/').collect(); + + // Compare the lengths first (in reverse order for longer length first - i.e. the rewrite with the more sub-paths first) + let length_cmp = b_parts.len().cmp(&a_parts.len()); - match glob { - Err(_) => false, - Ok(glob) => { - let matcher = glob.compile_matcher(); - matcher.is_match(requested_path) - } + if length_cmp == Ordering::Equal { + // If lengths are equal, sort alphabetically + a.cmp(b) + } else { + length_cmp } }); - rewrite.map(|(_, destination)| destination.clone()) + matches } diff --git a/src/satellite/src/storage/routing.rs b/src/satellite/src/storage/routing.rs new file mode 100644 index 000000000..baae475ab --- /dev/null +++ b/src/satellite/src/storage/routing.rs @@ -0,0 +1,211 @@ +use crate::rules::types::rules::Memory; +use crate::storage::constants::{ + RESPONSE_STATUS_CODE_200, RESPONSE_STATUS_CODE_404, ROOT_404_HTML, ROOT_INDEX_HTML, ROOT_PATH, +}; +use crate::storage::rewrites::{is_html_route, is_root_path, redirect_url, rewrite_url}; +use crate::storage::state::get_config; +use crate::storage::store::get_public_asset; +use crate::storage::types::http_request::{ + MapUrl, Routing, RoutingDefault, RoutingRedirect, RoutingRewrite, +}; +use crate::storage::types::state::FullPath; +use crate::storage::types::store::Asset; +use crate::storage::url::{map_alternative_paths, map_url}; + +pub fn get_routing( + url: String, + include_alternative_routing: bool, +) -> Result { + if url.is_empty() { + return Err("No url provided."); + } + + // The certification considers, and should only, the path of the URL. If query parameters, these should be omitted in the certificate. + // Likewise the memory contains only assets indexed with their respective path. + // e.g. + // url: /hello/something?param=123 + // path: /hello/something + + let MapUrl { path, token } = map_url(&url)?; + + // ⚠️ Limitation: requesting an url without extension try to resolve first a corresponding asset + // e.g. /.well-known/hello -> try to find /.well-known/hello.html + // Therefore if a file without extension is uploaded to the storage, it is important to not upload an .html file with the same name next to it or a folder/index.html + let alternative_asset = get_alternative_asset(&path, &token); + match alternative_asset { + None => (), + Some(alternative_asset) => { + return Ok(Routing::Default(RoutingDefault { + url: path.clone(), + asset: Some(alternative_asset), + })); + } + } + + // We return the asset that matches the effective path + let asset: Option<(Asset, Memory)> = get_public_asset(path.clone(), token.clone()); + + match asset { + None => (), + Some(_) => { + return Ok(Routing::Default(RoutingDefault { url: path, asset })); + } + } + + if include_alternative_routing { + // Search for potential redirect + let redirect = get_routing_redirect(&path); + + match redirect { + None => (), + Some(redirect) => { + return Ok(redirect); + } + } + + // Search for potential rewrite + let rewrite = get_routing_rewrite(&path, &token); + + match rewrite { + None => (), + Some(rewrite) => { + return Ok(rewrite); + } + } + + // Search for potential default rewrite for HTML pages + let root_rewrite = get_routing_root_rewrite(&path); + + match root_rewrite { + None => (), + Some(root_rewrite) => { + return Ok(root_rewrite); + } + } + } + + Ok(Routing::Default(RoutingDefault { + url: path, + asset: None, + })) +} + +fn get_alternative_asset(path: &String, token: &Option) -> Option<(Asset, Memory)> { + let alternative_paths = map_alternative_paths(path); + + for alternative_path in alternative_paths { + let asset: Option<(Asset, Memory)> = get_public_asset(alternative_path, token.clone()); + + // We return the first match + match asset { + None => (), + Some(_) => { + return asset; + } + } + } + + None +} + +fn get_routing_rewrite(path: &FullPath, token: &Option) -> Option { + // If we have found no asset, we try a rewrite rule + // This is for example useful for single-page app to redirect all urls to /index.html + let rewrite = rewrite_url(path, &get_config()); + + match rewrite { + None => (), + Some(rewrite) => { + let (source, destination) = rewrite; + + // Search for rewrite configured as an alternative path + // e.g. rewrite /demo/* to /sample + let rewrite_asset = get_alternative_asset(&destination, token); + + match rewrite_asset { + None => (), + Some(_) => { + return Some(Routing::Rewrite(RoutingRewrite { + url: path.clone(), + asset: rewrite_asset, + source, + status_code: RESPONSE_STATUS_CODE_200, + })); + } + } + + // Rewrite is maybe configured as an absolute path + // e.g. write /demo/* to /sample.html + let rewrite_absolute_asset: Option<(Asset, Memory)> = + get_public_asset(destination.clone(), token.clone()); + + match rewrite_absolute_asset { + None => (), + Some(_) => { + return Some(Routing::Rewrite(RoutingRewrite { + url: path.clone(), + asset: rewrite_absolute_asset, + source, + status_code: RESPONSE_STATUS_CODE_200, + })); + } + } + } + } + + None +} + +fn get_routing_root_rewrite(path: &FullPath) -> Option { + if is_html_route(path) && !is_root_path(path) { + // Search for potential /404.html to rewrite to + let asset_404: Option<(Asset, Memory)> = get_public_asset(ROOT_404_HTML.to_string(), None); + + // TODO: RESPONSE_STATUS_CODE_404 service worker does not support 404 yet + match asset_404 { + None => (), + Some(_) => { + return Some(Routing::Rewrite(RoutingRewrite { + url: path.clone(), + asset: asset_404, + source: ROOT_PATH.to_string(), + status_code: RESPONSE_STATUS_CODE_200, + })); + } + } + + // Search for potential /index.html to rewrite to + let asset_index: Option<(Asset, Memory)> = + get_public_asset(ROOT_INDEX_HTML.to_string(), None); + + match asset_index { + None => (), + Some(_) => { + return Some(Routing::Rewrite(RoutingRewrite { + url: path.clone(), + asset: asset_index, + source: ROOT_PATH.to_string(), + status_code: RESPONSE_STATUS_CODE_200, + })); + } + } + } + + None +} + +fn get_routing_redirect(path: &FullPath) -> Option { + let redirect = redirect_url(path, &get_config()); + + match redirect { + None => (), + Some(redirect) => { + return Some(Routing::Redirect(RoutingRedirect { + url: path.clone(), + redirect, + })); + } + } + + None +} diff --git a/src/satellite/src/storage/runtime.rs b/src/satellite/src/storage/runtime.rs index 899295069..bc8603c7c 100644 --- a/src/satellite/src/storage/runtime.rs +++ b/src/satellite/src/storage/runtime.rs @@ -1,7 +1,11 @@ use crate::memory::STATE; -use crate::storage::cert::update_certified_data; -use crate::storage::types::assets::AssetHashes; -use crate::storage::types::state::{Batches, Chunks, FullPath, StorageRuntimeState}; +use crate::storage::certification::cert::update_certified_data; +use crate::storage::certification::types::certified::CertifiedAssetHashes; +use crate::storage::rewrites::rewrite_source_to_path; +use crate::storage::routing::get_routing; +use crate::storage::types::config::StorageConfig; +use crate::storage::types::http_request::{Routing, RoutingDefault}; +use crate::storage::types::state::{Batches, Chunks, StorageRuntimeState}; use crate::storage::types::store::{Asset, Batch, Chunk}; use crate::types::state::{RuntimeState, State}; use ic_cdk::api::time; @@ -9,15 +13,37 @@ use ic_cdk::api::time; /// Certified assets pub fn init_certified_assets() { - fn init_asset_hashes(state: &State) -> AssetHashes { - let mut asset_hashes = AssetHashes::default(); + fn init_asset_hashes(state: &State) -> CertifiedAssetHashes { + let mut asset_hashes = CertifiedAssetHashes::default(); + + let config = &state.heap.storage.config; for (_key, asset) in state.heap.storage.assets.iter() { - asset_hashes.insert(asset); + asset_hashes.insert(asset, config); } for (_key, asset) in state.stable.assets.iter() { - asset_hashes.insert(&asset); + asset_hashes.insert(&asset, config); + } + + for (source, destination) in state.heap.storage.config.rewrites.clone() { + if let Ok(routing) = get_routing(destination, false) { + match routing { + Routing::Default(RoutingDefault { url: _, asset }) => { + let src_path = rewrite_source_to_path(&source); + + if let Some((asset, _)) = asset { + asset_hashes.insert_rewrite_v2(&src_path, &asset, config); + } + } + Routing::Rewrite(_) => (), + Routing::Redirect(_) => (), + } + } + } + + for (source, redirect) in state.heap.storage.config.unwrap_redirects() { + asset_hashes.insert_redirect_v2(&source, redirect.status_code, &redirect.location); } asset_hashes @@ -30,15 +56,18 @@ pub fn init_certified_assets() { }); } -pub fn update_certified_asset(asset: &Asset) { - STATE.with(|state| update_certified_asset_impl(asset, &mut state.borrow_mut().runtime)); +pub fn update_certified_asset(asset: &Asset, config: &StorageConfig) { + STATE.with(|state| update_certified_asset_impl(asset, config, &mut state.borrow_mut().runtime)); } -pub fn delete_certified_asset(full_path: &FullPath) { - STATE.with(|state| delete_certified_asset_impl(full_path, &mut state.borrow_mut().runtime)); +pub fn delete_certified_asset(asset: &Asset) { + STATE.with(|state| delete_certified_asset_impl(asset, &mut state.borrow_mut().runtime)); } -fn init_certified_assets_impl(asset_hashes: &AssetHashes, storage: &mut StorageRuntimeState) { +fn init_certified_assets_impl( + asset_hashes: &CertifiedAssetHashes, + storage: &mut StorageRuntimeState, +) { // 1. Init all asset in tree storage.asset_hashes = asset_hashes.clone(); @@ -46,17 +75,17 @@ fn init_certified_assets_impl(asset_hashes: &AssetHashes, storage: &mut StorageR update_certified_data(&storage.asset_hashes); } -fn update_certified_asset_impl(asset: &Asset, runtime: &mut RuntimeState) { +fn update_certified_asset_impl(asset: &Asset, config: &StorageConfig, runtime: &mut RuntimeState) { // 1. Replace or insert the new asset in tree - runtime.storage.asset_hashes.insert(asset); + runtime.storage.asset_hashes.insert(asset, config); // 2. Update the root hash and the canister certified data update_certified_data(&runtime.storage.asset_hashes); } -fn delete_certified_asset_impl(full_path: &FullPath, runtime: &mut RuntimeState) { +fn delete_certified_asset_impl(asset: &Asset, runtime: &mut RuntimeState) { // 1. Remove the asset in tree - runtime.storage.asset_hashes.delete(full_path); + runtime.storage.asset_hashes.delete(asset); // 2. Update the root hash and the canister certified data update_certified_data(&runtime.storage.asset_hashes); diff --git a/src/satellite/src/storage/store.rs b/src/satellite/src/storage/store.rs index 4390ad58c..a1984038f 100644 --- a/src/satellite/src/storage/store.rs +++ b/src/satellite/src/storage/store.rs @@ -16,9 +16,9 @@ use crate::rules::types::rules::{Memory, Rule}; use crate::rules::utils::{assert_create_rule, assert_rule, is_known_user, public_rule}; use crate::storage::constants::{ ASSET_ENCODING_NO_COMPRESSION, BN_WELL_KNOWN_CUSTOM_DOMAINS, ENCODING_CERTIFICATION_ORDER, + ROOT_404_HTML, ROOT_INDEX_HTML, }; use crate::storage::custom_domains::map_custom_domains_asset; -use crate::storage::rewrites::rewrite_url; use crate::storage::runtime::{ clear_batch as clear_runtime_batch, clear_expired_batches as clear_expired_runtime_batches, clear_expired_chunks as clear_expired_runtime_chunks, @@ -38,11 +38,9 @@ use crate::storage::state::{ }; use crate::storage::types::config::StorageConfig; use crate::storage::types::domain::{CustomDomain, CustomDomains, DomainName}; -use crate::storage::types::http_request::{MapUrl, PublicAsset}; use crate::storage::types::interface::{AssetNoContent, CommitBatch, InitAssetKey, UploadChunk}; use crate::storage::types::state::FullPath; use crate::storage::types::store::{Asset, AssetEncoding, AssetKey, Batch, Chunk, EncodingType}; -use crate::storage::url::{map_alternative_paths, map_url}; use crate::storage::utils::{filter_collection_values, filter_values}; use crate::types::core::{Blob, CollectionKey}; use crate::types::list::{ListParams, ListResults}; @@ -51,73 +49,6 @@ use crate::types::list::{ListParams, ListResults}; /// Getter, list and delete /// -pub fn get_public_asset_for_url(url: String) -> Result { - if url.is_empty() { - return Err("No url provided."); - } - - // The certification considers, and should only, the path of the URL. If query parameters, these should be omitted in the certificate. - // Likewise the memory contains only assets indexed with their respective path. - // e.g. - // url: /hello/something?param=123 - // path: /hello/something - - let MapUrl { path, token } = map_url(&url)?; - let alternative_paths = map_alternative_paths(&path); - - // ⚠️ Limitation: requesting an url without extension try to resolve first a corresponding asset - // e.g. /.well-known/hello -> try to find /.well-known/hello.html - // Therefore if a file without extension is uploaded to the storage, it is important to not upload an .html file with the same name next to it or a folder/index.html - - for alternative_path in alternative_paths { - let asset: Option<(Asset, Memory)> = get_public_asset(alternative_path, token.clone()); - - // We return the first match - match asset { - None => (), - Some(_) => { - return Ok(PublicAsset { url: path, asset }); - } - } - } - - // We return the asset that matches the effective path - let asset: Option<(Asset, Memory)> = get_public_asset(path.clone(), token.clone()); - - match asset { - None => (), - Some(_) => { - return Ok(PublicAsset { url: path, asset }); - } - } - - // If we have found no asset, we try a rewrite rule - // This is for example useful for single-page app to redirect all urls to /index.html - let rewrite = rewrite_url(&path, &get_config()); - - match rewrite { - None => (), - Some(rewrite) => { - let redirected_asset = get_public_asset(rewrite.clone(), token); - - match redirected_asset { - None => (), - Some(_) => { - return Ok(PublicAsset { - url: rewrite, - asset: redirected_asset, - }); - } - } - } - } - - Ok(PublicAsset { - url: path, - asset: None, - }) -} - pub fn get_content_chunks( encoding: &AssetEncoding, chunk_index: usize, @@ -132,8 +63,9 @@ pub fn delete_asset( full_path: FullPath, ) -> Result, String> { let controllers: Controllers = get_controllers(); + let config = get_config(); - secure_delete_asset_impl(caller, &controllers, collection, full_path) + secure_delete_asset_impl(caller, &controllers, collection, full_path, &config) } pub fn delete_assets(collection: &CollectionKey) -> Result<(), String> { @@ -239,10 +171,11 @@ fn secure_delete_asset_impl( controllers: &Controllers, collection: &CollectionKey, full_path: FullPath, + config: &StorageConfig, ) -> Result, String> { let rule = get_state_rule(collection)?; - delete_asset_impl(caller, controllers, full_path, collection, &rule) + delete_asset_impl(caller, controllers, full_path, collection, &rule, config) } fn delete_asset_impl( @@ -251,6 +184,7 @@ fn delete_asset_impl( full_path: FullPath, collection: &CollectionKey, rule: &Rule, + config: &StorageConfig, ) -> Result, String> { let asset = get_state_asset(collection, &full_path, rule); @@ -262,7 +196,17 @@ fn delete_asset_impl( } let deleted = delete_state_asset(collection, &full_path, rule); - delete_runtime_certified_asset(&full_path); + delete_runtime_certified_asset(&asset); + + // We just removed the rewrite for /404.html in the certification tree therefore if /index.html exists, we want to reintroduce it as rewrite + if *full_path == *ROOT_404_HTML { + if let Some(index_asset) = + get_state_asset(collection, &ROOT_INDEX_HTML.to_string(), rule) + { + update_runtime_certified_asset(&index_asset, config); + } + } + Ok(deleted) } } @@ -271,15 +215,21 @@ fn delete_asset_impl( fn delete_assets_impl(collection: &CollectionKey) -> Result<(), String> { let rule = get_state_rule(collection)?; - let full_paths: Vec = get_state_assets(collection, &rule) + let full_paths: Vec = get_state_assets(collection, &rule) .iter() .filter(|asset| asset.key.collection == collection.clone()) .map(|asset| asset.key.full_path.clone()) .collect(); for full_path in full_paths { - delete_state_asset(collection, &full_path, &rule); - delete_runtime_certified_asset(&full_path); + let deleted_asset = delete_state_asset(collection, &full_path, &rule); + + match deleted_asset { + None => {} + Some(deleted_asset) => { + delete_runtime_certified_asset(&deleted_asset); + } + } } Ok(()) @@ -305,7 +255,9 @@ pub fn create_chunk(caller: Principal, chunk: UploadChunk) -> Result Result<(), String> { let controllers: Controllers = get_controllers(); - commit_batch_impl(caller, &controllers, commit_batch) + let config = get_config(); + + commit_batch_impl(caller, &controllers, commit_batch, &config) } fn secure_create_batch_impl( @@ -420,6 +372,7 @@ fn commit_batch_impl( caller: Principal, controllers: &Controllers, commit_batch: CommitBatch, + config: &StorageConfig, ) -> Result<(), String> { let batch = get_runtime_batch(&commit_batch.batch_id); @@ -427,7 +380,7 @@ fn commit_batch_impl( None => Err(ERROR_CANNOT_COMMIT_BATCH.to_string()), Some(b) => { let asset = secure_commit_chunks(caller, controllers, commit_batch, &b)?; - update_runtime_certified_asset(&asset); + update_runtime_certified_asset(&asset, config); Ok(()) } } @@ -654,6 +607,8 @@ fn clear_expired_batches() { pub fn set_config(config: &StorageConfig) { insert_state_config(config); + + init_certified_assets(); } pub fn get_config() -> StorageConfig { @@ -704,7 +659,9 @@ fn update_custom_domains_asset() -> Result<(), String> { insert_state_asset(&collection, &full_path, &asset, &rule); - update_runtime_certified_asset(&asset); + let config = get_config(); + + update_runtime_certified_asset(&asset, &config); Ok(()) } diff --git a/src/satellite/src/storage/types.rs b/src/satellite/src/storage/types.rs index 1520197a1..3d65c7454 100644 --- a/src/satellite/src/storage/types.rs +++ b/src/satellite/src/storage/types.rs @@ -1,6 +1,6 @@ pub mod state { use crate::rules::types::rules::Rules; - use crate::storage::types::assets::AssetHashes; + use crate::storage::certification::types::certified::CertifiedAssetHashes; use crate::storage::types::config::StorageConfig; use crate::storage::types::domain::CustomDomains; use crate::storage::types::store::{Asset, Batch, Chunk, EncodingType}; @@ -46,22 +46,12 @@ pub mod state { pub struct StorageRuntimeState { pub chunks: Chunks, pub batches: Batches, - pub asset_hashes: AssetHashes, - } -} - -pub mod assets { - use ic_certified_map::{Hash, RbTree}; - use std::clone::Clone; - - #[derive(Default, Clone)] - pub struct AssetHashes { - pub tree: RbTree, + pub asset_hashes: CertifiedAssetHashes, } } pub mod store { - use crate::storage::types::http::HeaderField; + use crate::storage::http::types::HeaderField; use crate::storage::types::state::FullPath; use crate::types::core::{Blob, CollectionKey}; use candid::CandidType; @@ -128,7 +118,7 @@ pub mod interface { use candid::{CandidType, Deserialize}; use ic_certified_map::Hash; - use crate::storage::types::http::HeaderField; + use crate::storage::http::types::HeaderField; use crate::storage::types::state::FullPath; use crate::storage::types::store::{AssetKey, EncodingType}; use crate::types::core::{Blob, CollectionKey}; @@ -184,79 +174,34 @@ pub mod interface { } } -pub mod http { - use crate::rules::types::rules::Memory; - use crate::storage::types::store::EncodingType; - use crate::types::core::Blob; - use candid::{define_function, CandidType}; - use serde::{Deserialize, Serialize}; - use serde_bytes::ByteBuf; - - #[derive(CandidType, Serialize, Deserialize, Clone)] - pub struct HeaderField(pub String, pub String); - - #[derive(CandidType, Deserialize, Clone)] - pub struct HttpRequest { - pub url: String, - pub method: String, - pub headers: Vec, - pub body: Blob, - } - - #[derive(CandidType, Deserialize, Clone)] - pub struct HttpResponse { - pub body: Blob, - pub headers: Vec, - pub status_code: u16, - pub streaming_strategy: Option, - } - - define_function!(pub CallbackFunc : () -> () query); - - #[derive(CandidType, Deserialize, Clone)] - pub enum StreamingStrategy { - Callback { - token: StreamingCallbackToken, - callback: CallbackFunc, - }, - } - - #[derive(CandidType, Deserialize, Clone)] - pub struct StreamingCallbackToken { - pub full_path: String, - pub token: Option, - pub headers: Vec, - pub sha256: Option, - pub index: usize, - pub encoding_type: EncodingType, - pub memory: Memory, - } - - #[derive(CandidType, Deserialize, Clone)] - pub struct StreamingCallbackHttpResponse { - pub body: Blob, - pub token: Option, - } -} - pub mod config { - use crate::storage::types::http::HeaderField; + use crate::storage::http::types::{HeaderField, StatusCode}; use candid::CandidType; use serde::{Deserialize, Serialize}; use std::collections::HashMap; pub type StorageConfigHeaders = HashMap>; pub type StorageConfigRewrites = HashMap; + pub type StorageConfigRedirects = HashMap; #[derive(Default, CandidType, Serialize, Deserialize, Clone)] pub struct StorageConfig { pub headers: StorageConfigHeaders, pub rewrites: StorageConfigRewrites, + pub redirects: Option, + } + + #[derive(Default, CandidType, Serialize, Deserialize, Clone)] + pub struct StorageConfigRedirect { + pub location: String, + pub status_code: StatusCode, } } pub mod http_request { use crate::rules::types::rules::Memory; + use crate::storage::http::types::StatusCode; + use crate::storage::types::config::StorageConfigRedirect; use crate::storage::types::store::Asset; use candid::{CandidType, Deserialize}; @@ -267,9 +212,30 @@ pub mod http_request { } #[derive(CandidType, Deserialize, Clone)] - pub struct PublicAsset { + pub enum Routing { + Default(RoutingDefault), + Rewrite(RoutingRewrite), + Redirect(RoutingRedirect), + } + + #[derive(CandidType, Deserialize, Clone)] + pub struct RoutingDefault { + pub url: String, + pub asset: Option<(Asset, Memory)>, + } + + #[derive(CandidType, Deserialize, Clone)] + pub struct RoutingRewrite { pub url: String, pub asset: Option<(Asset, Memory)>, + pub source: String, + pub status_code: StatusCode, + } + + #[derive(CandidType, Deserialize, Clone)] + pub struct RoutingRedirect { + pub url: String, + pub redirect: StorageConfigRedirect, } } diff --git a/src/satellite/src/storage/url.rs b/src/satellite/src/storage/url.rs index fadc27337..0fce35ff5 100644 --- a/src/satellite/src/storage/url.rs +++ b/src/satellite/src/storage/url.rs @@ -1,5 +1,7 @@ use crate::storage::types::http_request::MapUrl; use crate::storage::types::state::FullPath; +use globset::Glob; +use std::collections::HashMap; use std::path::Path; use url::{ParseError, Url}; @@ -73,7 +75,12 @@ fn aliases_of(key: &String) -> Vec { // Determines possible original keys in case the supplied key is being aliaseded to. // Sort-of a reverse operation of `alias_of` fn aliased_by(key: &String) -> Option> { - if key.ends_with("/index.html") { + if key == "/index.html" { + Some(vec![ + key[..(key.len() - 5)].into(), + key[..(key.len() - 10)].into(), + ]) + } else if key.ends_with("/index.html") { Some(vec![ key[..(key.len() - 5)].into(), key[..(key.len() - 10)].into(), @@ -95,7 +102,7 @@ pub fn build_url(url: &String) -> Result { } /// Ensure path always will begin with a / -fn separator(url: &str) -> &str { +pub fn separator(url: &str) -> &str { if url.starts_with('/') { "" } else { @@ -118,3 +125,24 @@ fn map_token(parsed_url: Url) -> Option { None } + +pub fn matching_urls( + requested_path: &str, + config: &HashMap, +) -> Vec<(String, T)> { + config + .iter() + .filter(|(source, _)| { + let glob = Glob::new(source); + + match glob { + Err(_) => false, + Ok(glob) => { + let matcher = glob.compile_matcher(); + matcher.is_match(requested_path) + } + } + }) + .map(|(source, destination)| (source.clone(), destination.clone())) + .collect() +}