From 7312d276a122aaaac7cfad8591f9bda8ae95efa2 Mon Sep 17 00:00:00 2001 From: Andrey Ermilov Date: Sun, 1 Oct 2023 11:26:04 +0200 Subject: [PATCH] feat(config): Add external config --- Cargo.lock | 49 +++++++++++- config_hitbox.yaml | 24 ++++++ hitboxd/Cargo.toml | 3 +- hitboxd/src/config.rs | 49 ++++++------ hitboxd/src/endpoint.rs | 57 ++++++++++++++ hitboxd/src/external_configuration/backend.rs | 29 ++++++++ .../src/external_configuration/endpoint.rs | 51 +++++++++++++ hitboxd/src/external_configuration/group.rs | 1 + hitboxd/src/external_configuration/mod.rs | 27 +++++++ hitboxd/src/external_configuration/policy.rs | 13 ++++ .../src/external_configuration/upstream.rs | 46 ++++++++++++ hitboxd/src/layer.rs | 27 +++---- hitboxd/src/lib.rs | 3 + hitboxd/src/main.rs | 63 ++++------------ hitboxd/src/service.rs | 74 ++++++++++--------- 15 files changed, 389 insertions(+), 127 deletions(-) create mode 100644 config_hitbox.yaml create mode 100644 hitboxd/src/endpoint.rs create mode 100644 hitboxd/src/external_configuration/backend.rs create mode 100644 hitboxd/src/external_configuration/endpoint.rs create mode 100644 hitboxd/src/external_configuration/group.rs create mode 100644 hitboxd/src/external_configuration/mod.rs create mode 100644 hitboxd/src/external_configuration/policy.rs create mode 100644 hitboxd/src/external_configuration/upstream.rs diff --git a/Cargo.lock b/Cargo.lock index 7808b21..ee199c8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -350,6 +350,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + [[package]] name = "errno" version = "0.3.2" @@ -544,7 +550,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap", + "indexmap 1.9.3", "slab", "tokio", "tokio-util", @@ -557,6 +563,12 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +[[package]] +name = "hashbrown" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" + [[package]] name = "hdrhistogram" version = "7.5.2" @@ -689,11 +701,13 @@ dependencies = [ "hitbox", "hitbox-http", "hitbox-redis", + "hitbox-stretto", "hitbox-tower", "http", "hyper", "pin-project", "serde", + "serde_yaml", "tokio", "tower", "tower-http", @@ -805,7 +819,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", - "hashbrown", + "hashbrown 0.12.3", +] + +[[package]] +name = "indexmap" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" +dependencies = [ + "equivalent", + "hashbrown 0.14.0", ] [[package]] @@ -1322,6 +1346,19 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_yaml" +version = "0.9.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a49e178e4452f45cb61d0cd8cebc1b0fafd3e41929e996cef79aa3aca91f574" +dependencies = [ + "indexmap 2.0.0", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + [[package]] name = "sha1_smol" version = "1.0.0" @@ -1538,7 +1575,7 @@ dependencies = [ "futures-core", "futures-util", "hdrhistogram", - "indexmap", + "indexmap 1.9.3", "pin-project", "pin-project-lite", "rand", @@ -1697,6 +1734,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unsafe-libyaml" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28467d3e1d3c6586d8f25fa243f544f5800fec42d97032474e17222c2b75cfa" + [[package]] name = "url" version = "2.4.1" diff --git a/config_hitbox.yaml b/config_hitbox.yaml new file mode 100644 index 0000000..67ec3eb --- /dev/null +++ b/config_hitbox.yaml @@ -0,0 +1,24 @@ +upstreams: +- name: default-upstream + addresses: + - host: 127.0.0.1 + port: 8080 + scheme: http + +backends: +- type: inmemory + name: StrettoBackend + capacity: 10000000 + +endpoints: +- name: all + path: /{path}* + method: GET + key: + - Method + - !Path + path: /{path}* + predicates: + request: [] + response: + - !StatusCode 200 diff --git a/hitboxd/Cargo.toml b/hitboxd/Cargo.toml index 62f194d..9235805 100644 --- a/hitboxd/Cargo.toml +++ b/hitboxd/Cargo.toml @@ -7,10 +7,10 @@ edition = "2021" [dependencies] hitbox = { path = "../../hitbox/hitbox", features = ["metrics"] } -# hitbox-stretto = { path = "../../hitbox/hitbox-stretto" } hitbox-tower = { path = "../../hitbox/hitbox-tower" } hitbox-http = { path = "../../hitbox/hitbox-http", version = "0.1" } hitbox-redis = { path = "../../hitbox/hitbox-redis", version = "0.1" } +hitbox-stretto = { path = "../../hitbox/hitbox-stretto", version = "0.1" } actix-router = "0.5" hyper = { version = "0.14", features = ["full"] } tokio = { version = "1", features = ["full"] } @@ -24,3 +24,4 @@ futures = "0.3" pin-project = "1" chrono = "0.4" bytes = "1" +serde_yaml = "0.9.25" diff --git a/hitboxd/src/config.rs b/hitboxd/src/config.rs index 66c7911..b57e197 100644 --- a/hitboxd/src/config.rs +++ b/hitboxd/src/config.rs @@ -1,33 +1,40 @@ -use std::collections::HashMap; -use std::fmt::Debug; +use crate::external_configuration; +use hitbox::policy::PolicyConfig; +use hitbox_stretto::StrettoBackend; use std::sync::Arc; -use hitbox::predicate::Predicate; -use hitbox::Extractor; -use hitbox_http::CacheableHttpRequest; -use http::method::Method; -use hyper::Body; - -pub type BoxPredicate = Box> + Send + Sync>; -pub type BoxExtractor = Box> + Send + Sync>; - pub struct Config { - pub endpoints: HashMap>, + pub endpoints: Vec, } impl Config { pub fn new() -> Self { Config { - endpoints: HashMap::new(), + endpoints: Vec::new(), } } -} -#[derive(Debug)] -pub struct Endpoint { - pub name: String, - pub path: String, - pub methods: Vec, - pub request_predicate: Arc

, - pub extractors: Arc, + pub fn from_external( + config: external_configuration::Config, + backend: Arc, + ) -> Self { + let endpoints = config + .endpoints + .into_iter() + .map(|source_endpoint| crate::Endpoint { + name: source_endpoint.name, + routing: crate::endpoint::Routing { + path_pattern: source_endpoint.path, + methods: vec![source_endpoint.method], + }, + backend: backend.clone(), + upstreams: Vec::new(), + request_predicates: source_endpoint.predicates.request, + response_predicates: source_endpoint.predicates.response, + extractors: source_endpoint.key, + policy: PolicyConfig::default(), + }) + .collect(); + Config { endpoints } + } } diff --git a/hitboxd/src/endpoint.rs b/hitboxd/src/endpoint.rs new file mode 100644 index 0000000..1f65550 --- /dev/null +++ b/hitboxd/src/endpoint.rs @@ -0,0 +1,57 @@ +use hitbox::policy::PolicyConfig; +use hitbox_stretto::StrettoBackend; +use hitbox_tower::configuration::{RequestExtractor, RequestPredicate, ResponsePredicate}; +use hitbox_tower::EndpointConfig; +use hitbox_tower::Method; +use std::sync::Arc; + +#[derive(Clone)] +pub struct Routing { + pub path_pattern: String, + pub methods: Vec, +} + +#[derive(Clone)] +pub struct Upstream { + pub address: String, + pub port: u16, +} + +#[derive(Clone)] +pub struct Endpoint { + pub name: String, + pub routing: Routing, + pub backend: Arc, + pub upstreams: Vec, + pub request_predicates: Vec, + pub response_predicates: Vec, + pub extractors: Vec, + pub policy: PolicyConfig, +} + +impl Endpoint { + pub fn new(backend: Arc) -> Self { + Self { + name: String::new(), + routing: Routing { + path_pattern: String::new(), + methods: Vec::new(), + }, + backend, + upstreams: Vec::new(), + request_predicates: Vec::new(), + response_predicates: Vec::new(), + extractors: Vec::new(), + policy: Default::default(), + } + } + + pub fn to_endpoint_config(self) -> EndpointConfig { + EndpointConfig { + request_predicates: self.request_predicates, + response_predicates: self.response_predicates, + extractors: self.extractors, + policy: self.policy, + } + } +} diff --git a/hitboxd/src/external_configuration/backend.rs b/hitboxd/src/external_configuration/backend.rs new file mode 100644 index 0000000..51ae99a --- /dev/null +++ b/hitboxd/src/external_configuration/backend.rs @@ -0,0 +1,29 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Serialize, Deserialize)] +pub struct InMemory { + pub name: String, + pub capacity: u32, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(tag = "type")] +#[serde(rename_all = "lowercase")] +pub enum Backend { + InMemory(InMemory), +} + +impl Default for InMemory { + fn default() -> Self { + Self { + name: String::from("StrettoBackend"), + capacity: 10_000_000, + } + } +} + +impl Default for Backend { + fn default() -> Self { + Self::InMemory(InMemory::default()) + } +} diff --git a/hitboxd/src/external_configuration/endpoint.rs b/hitboxd/src/external_configuration/endpoint.rs new file mode 100644 index 0000000..c33e7c1 --- /dev/null +++ b/hitboxd/src/external_configuration/endpoint.rs @@ -0,0 +1,51 @@ +use hitbox_tower::{ + configuration::serializers::method, + configuration::{RequestExtractor, RequestPredicate, ResponsePredicate}, + Method, StatusCode, +}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Serialize, Deserialize)] +pub struct Predicates { + pub request: Vec, + pub response: Vec, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct Endpoint { + pub name: String, + pub path: String, + #[serde(with = "method")] + pub method: Method, + pub key: Vec, + pub predicates: Predicates, +} + +impl Default for Predicates { + fn default() -> Self { + Self { + request: Vec::new(), + response: vec![ResponsePredicate::StatusCode { + code: StatusCode::OK, + }], + } + } +} + +impl Default for Endpoint { + fn default() -> Self { + let default_extractors = vec![ + RequestExtractor::Method, + RequestExtractor::Path { + path: String::from("/{path}*"), + }, + ]; + Self { + name: String::from("all"), + path: String::from("/{path}*"), + method: Method::GET, + key: default_extractors, + predicates: Predicates::default(), + } + } +} diff --git a/hitboxd/src/external_configuration/group.rs b/hitboxd/src/external_configuration/group.rs new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/hitboxd/src/external_configuration/group.rs @@ -0,0 +1 @@ + diff --git a/hitboxd/src/external_configuration/mod.rs b/hitboxd/src/external_configuration/mod.rs new file mode 100644 index 0000000..82e8a1f --- /dev/null +++ b/hitboxd/src/external_configuration/mod.rs @@ -0,0 +1,27 @@ +mod backend; +mod endpoint; +mod group; +mod policy; +mod upstream; + +pub use backend::Backend; +pub use endpoint::Endpoint; +pub use policy::Policy; +pub use upstream::Upstream; + +#[derive(Debug, serde::Serialize, serde::Deserialize)] +pub struct Config { + pub upstreams: Vec, + pub backends: Vec, + pub endpoints: Vec, +} + +impl Default for Config { + fn default() -> Self { + Self { + upstreams: vec![Upstream::default()], + backends: vec![Backend::default()], + endpoints: vec![Endpoint::default()], + } + } +} diff --git a/hitboxd/src/external_configuration/policy.rs b/hitboxd/src/external_configuration/policy.rs new file mode 100644 index 0000000..fa9ed58 --- /dev/null +++ b/hitboxd/src/external_configuration/policy.rs @@ -0,0 +1,13 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Serialize, Deserialize)] +pub struct Enabled { + ttl: u16, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(untagged)] +pub enum Policy { + Enabled(Enabled), + Disabled, +} diff --git a/hitboxd/src/external_configuration/upstream.rs b/hitboxd/src/external_configuration/upstream.rs new file mode 100644 index 0000000..d6e1166 --- /dev/null +++ b/hitboxd/src/external_configuration/upstream.rs @@ -0,0 +1,46 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum Scheme { + Http, + Https, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct Address { + pub host: String, + pub port: u16, + pub scheme: Scheme, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct Upstream { + pub name: String, + pub addresses: Vec

, +} + +impl Default for Scheme { + fn default() -> Self { + Scheme::Http + } +} + +impl Default for Address { + fn default() -> Self { + Self { + host: String::from("127.0.0.1"), + port: 8080, + scheme: Default::default(), + } + } +} + +impl Default for Upstream { + fn default() -> Self { + Upstream { + name: String::from("default-upstream"), + addresses: vec![Address::default()], + } + } +} diff --git a/hitboxd/src/layer.rs b/hitboxd/src/layer.rs index 62e8739..f32523b 100644 --- a/hitboxd/src/layer.rs +++ b/hitboxd/src/layer.rs @@ -1,4 +1,5 @@ -use std::collections::HashMap; +use crate::Config; + use std::sync::Arc; use tower::Layer; @@ -6,32 +7,22 @@ use tower::Layer; use crate::CacheService; #[derive(Clone)] -pub struct Cache { - pub backends: Arc>>, - pub config: Arc, +pub struct Cache { + pub config: Arc, } -impl Cache { - pub fn new(backend: B, config: crate::Config) -> Cache { - let backends = vec![(String::from("InMemory"), Arc::new(backend))]; - let backends = HashMap::<_, _, std::collections::hash_map::RandomState>::from_iter( - backends.into_iter(), - ); +impl Cache { + pub fn new(config: crate::Config) -> Cache { Cache { - backends: Arc::new(backends), config: Arc::new(config), } } } -impl Layer for Cache { - type Service = CacheService; +impl Layer for Cache { + type Service = CacheService; fn layer(&self, upstream: S) -> Self::Service { - CacheService::new( - upstream, - Arc::clone(&self.backends), - Arc::clone(&self.config), - ) + CacheService::new(upstream, Arc::clone(&self.config)) } } diff --git a/hitboxd/src/lib.rs b/hitboxd/src/lib.rs index e0eed7a..d2b51a4 100644 --- a/hitboxd/src/lib.rs +++ b/hitboxd/src/lib.rs @@ -1,7 +1,10 @@ pub mod config; +pub mod endpoint; +pub mod external_configuration; pub mod layer; pub mod service; pub use config::Config; +pub use endpoint::Endpoint; pub use layer::Cache; pub use service::CacheService; diff --git a/hitboxd/src/main.rs b/hitboxd/src/main.rs index a9cc554..7a5829c 100644 --- a/hitboxd/src/main.rs +++ b/hitboxd/src/main.rs @@ -1,20 +1,9 @@ -use hitbox::{predicate::Predicate, Extractor}; -use hitbox_http::{ - extractors::{method::MethodExtractor, path::PathExtractor, NeutralExtractor}, - predicates::{ - request::{HeaderPredicate, QueryPredicate}, - NeutralRequestPredicate, - }, - CacheableHttpRequest, -}; -use hitbox_redis::RedisBackend; -// use hitbox_stretto::StrettoBackend; -use hitboxd::{config::Endpoint, Cache}; -use http::Method; -use hyper::{Body, Server}; -use std::{collections::HashMap, net::SocketAddr, sync::Arc}; - +use hitbox_stretto::StrettoBackend; +use hitboxd::{Cache, Config}; use hyper::http::{Request, Response}; +use hyper::{Body, Server}; +use std::fs; +use std::{net::SocketAddr, sync::Arc}; use tower::make::Shared; async fn handle(mut req: Request) -> Result, hyper::Error> { @@ -34,43 +23,21 @@ async fn handle(mut req: Request) -> Result, hyper::Error> #[tokio::main] async fn main() { + let file_path = String::from("config_hitbox.yaml"); + let contents = fs::read_to_string(file_path).expect("Should have been able to read the file"); + let external_config = + serde_yaml::from_str::(&contents).unwrap(); + let subscriber = tracing_subscriber::fmt().pretty().finish(); tracing::subscriber::set_global_default(subscriber).unwrap(); - let mut config = hitboxd::Config::new(); - let test_endpoint = Endpoint { - name: "test".to_owned(), - path: "/test/".to_owned(), - methods: vec![Method::GET], - request_predicate: Arc::new(Box::new( - NeutralRequestPredicate::new().query("cache".to_owned(), "true".to_owned()), - ) - as Box> + Send + Sync>), - extractors: Arc::new(Box::new(NeutralExtractor::new().method().path("{path}*")) - as Box> + Send + Sync>), - }; - let ip_endpoint = Endpoint { - name: "ip".to_owned(), - path: "/ip".to_owned(), - methods: vec![Method::GET], - request_predicate: Arc::new(Box::new( - NeutralRequestPredicate::new() - .query("cache".to_owned(), "true".to_owned()) - .header("x-cache".to_owned(), "enable".to_owned()), - ) - as Box> + Send + Sync>), - extractors: Arc::new(Box::new(NeutralExtractor::new().method().path("{path}*")) - as Box> + Send + Sync>), - }; - config.endpoints = HashMap::with_capacity(2); - config.endpoints.insert("test".to_owned(), test_endpoint); - config.endpoints.insert("ip".to_owned(), ip_endpoint); - - let backend = RedisBackend::builder().build().unwrap(); - // let inmemory = StrettoBackend::builder(10_000_000).finalize().unwrap(); + let inmemory_backend = StrettoBackend::builder(10).finalize().unwrap(); + let config = Config::from_external(external_config, Arc::new(inmemory_backend)); + let config = Arc::new(config); + let cache_layer = Cache { config }; let service = tower::ServiceBuilder::new() .layer(tower_http::trace::TraceLayer::new_for_http()) - .layer(Cache::new(backend, config)) + .layer(cache_layer) .service_fn(handle); let addr = SocketAddr::from(([127, 0, 0, 1], 3000)); diff --git a/hitboxd/src/service.rs b/hitboxd/src/service.rs index 4f6e83f..3476ee0 100644 --- a/hitboxd/src/service.rs +++ b/hitboxd/src/service.rs @@ -1,56 +1,54 @@ -use std::collections::HashMap; +use hitbox_stretto::StrettoBackend; +use hitbox_tower::{CacheConfig, EndpointConfig}; + use std::{fmt::Debug, sync::Arc}; -use actix_router::ResourceDef; -use hitbox::policy::PolicyConfig; -use hitbox::{backend::CacheBackend, fsm::CacheFuture}; -use hitbox_http::{ - predicates::NeutralResponsePredicate, CacheableHttpRequest, CacheableHttpResponse, FromBytes, -}; +use hitbox::fsm::CacheFuture; +use hitbox_http::{CacheableHttpRequest, CacheableHttpResponse, FromBytes}; use http::{Request, Response}; use hyper::body::{Body, HttpBody}; use tower::Service; use hitbox_tower::future::Transformer; -pub struct CacheService { +pub struct CacheService { upstream: S, - backends: Arc>>, + //backends: Arc>>, config: Arc, } -impl CacheService { +impl CacheService { pub fn new( upstream: S, - backends: Arc>>, + //backends: Arc>>, config: Arc, ) -> Self { CacheService { upstream, - backends, + //backends, config, } } } -impl Clone for CacheService +impl Clone for CacheService where S: Clone, - B: Clone, + //B: Clone, { fn clone(&self) -> Self { Self { upstream: self.upstream.clone(), - backends: Arc::clone(&self.backends), + //backends: Arc::clone(&self.backends), config: Arc::clone(&self.config), } } } -impl Service> for CacheService +impl Service> for CacheService where S: Service, Response = Response> + Clone + Send + 'static, - B: CacheBackend + Clone + Send + Sync + 'static, + //B: CacheBackend + Clone + Send + Sync + 'static, S::Future: Send, // debug bounds @@ -62,7 +60,7 @@ where type Response = Response; type Error = S::Error; type Future = CacheFuture< - B, + StrettoBackend, CacheableHttpRequest, CacheableHttpResponse, Transformer, @@ -76,28 +74,32 @@ where } fn call(&mut self, req: Request) -> Self::Future { - let endpoint = self - .config - .endpoints - .values() - .find(|endpoint| { - ResourceDef::new(endpoint.path.as_str()).is_match(req.uri().path()) - && endpoint.methods.contains(req.method()) - }) - .unwrap(); - let request_predicate = Arc::clone(&endpoint.request_predicate); - let extractors = Arc::clone(&endpoint.extractors); + let endpoint = self.config.endpoints.first().unwrap().clone(); + //let endpoint = self + //.config + //.endpoints + ////.values() + //.into_iter() + //.find(|endpoint| { + //ResourceDef::new(endpoint.path.as_str()).is_match(req.uri().path()) + //&& endpoint.methods.contains(req.method()) + //}) + //.unwrap(); let transformer = Transformer::new(self.upstream.clone()); - let backend = self.backends.get("InMemory").unwrap(); + let configuration = EndpointConfig { + request_predicates: endpoint.request_predicates, + response_predicates: endpoint.response_predicates, + extractors: endpoint.extractors, + policy: endpoint.policy, + }; CacheFuture::new( - backend.clone(), + endpoint.backend, CacheableHttpRequest::from_request(req), transformer, - request_predicate, - // Arc::new(NeutralRequestPredicate::new().query("cache".to_owned(), "true".to_owned())), - Arc::new(NeutralResponsePredicate::new()), - extractors, - Arc::new(PolicyConfig::default()), + Arc::new(configuration.request_predicates()), + Arc::new(configuration.response_predicates()), + Arc::new(configuration.extractors()), + Arc::new(configuration.policy()), ) } }