From 2622ac0f46975705ed0a112c2dedff2946a33663 Mon Sep 17 00:00:00 2001 From: Samuel Gomez Date: Wed, 24 Jul 2024 15:41:03 +0100 Subject: [PATCH 1/7] Adding typescript type endpoint and fixing ts types --- .../src/endpoints/schema_generator.rs | 28 ++++++++++- .../src/domain/schema/common_model.rs | 46 +++++++++++-------- 2 files changed, 53 insertions(+), 21 deletions(-) diff --git a/integrationos-api/src/endpoints/schema_generator.rs b/integrationos-api/src/endpoints/schema_generator.rs index 187b22dd..26dbb335 100644 --- a/integrationos-api/src/endpoints/schema_generator.rs +++ b/integrationos-api/src/endpoints/schema_generator.rs @@ -7,7 +7,9 @@ use axum::{ }; use bson::{doc, Document}; use futures::StreamExt; -use integrationos_domain::{ApplicationError, Id, IntegrationOSError, InternalError, Store}; +use integrationos_domain::{ + api_model_config::Lang, ApplicationError, Id, IntegrationOSError, InternalError, Store, +}; use mongodb::options::FindOptions; use std::sync::Arc; @@ -15,6 +17,7 @@ pub fn get_router() -> Router> { Router::new() .route("/projection", get(get_common_model_proj)) .route("/:id", get(generate_schema)) + .route("/types/:id", get(generate_types)) } pub async fn get_common_model_proj( @@ -61,6 +64,29 @@ pub async fn get_common_model_proj( })) } +pub async fn generate_types( + state: State>, + Path(id): Path, +) -> Result { + let cm_store = state.app_stores.common_model.clone(); + let ce_store = state.app_stores.common_enum.clone(); + + let common_model = cm_store + .get_one_by_id(&id.to_string()) + .await + .map_err(IntegrationOSError::from)? + .ok_or(ApplicationError::not_found( + &format!("CommonModel with id {} not found", id), + None, + ))?; + + let schema = common_model + .generate_as_expanded(&Lang::TypeScript, &cm_store, &ce_store) + .await; + + Ok(schema) +} + pub async fn generate_schema( state: State>, Path(id): Path, diff --git a/integrationos-domain/src/domain/schema/common_model.rs b/integrationos-domain/src/domain/schema/common_model.rs index fb4f3fe1..956b29d0 100644 --- a/integrationos-domain/src/domain/schema/common_model.rs +++ b/integrationos-domain/src/domain/schema/common_model.rs @@ -187,6 +187,32 @@ impl CommonEnum { ) } + pub fn as_typescript_type(&self) -> String { + // let's add the value directly to the enum + format!( + "export const enum {} {{ {} }}\n", + replace_reserved_keyword(&self.name, Lang::TypeScript) + .replace("::", "") + .pascal_case(), + self.options + .iter() + .map(|option| { + let option_name = option.pascal_case(); + let option_value = if option.chars().all(char::is_uppercase) { + option.to_lowercase() + } else { + option.kebab_case() + }; + + format!("{} = '{}'", option_name, option_value) + }) + .collect::>() + .into_iter() + .collect::>() + .join(", ") + ) + } + /// Generates a effect Schema for the enum pub fn as_typescript_schema(&self) -> String { let name = replace_reserved_keyword(&self.name, Lang::TypeScript) @@ -220,22 +246,6 @@ impl CommonEnum { format!("{}\n{}", native_enum, schema) } - - pub fn as_typescript_type(&self) -> String { - format!( - "export const enum {} {{ {} }}\n", - replace_reserved_keyword(&self.name, Lang::TypeScript) - .replace("::", "") - .pascal_case(), - self.options - .iter() - .map(|option| option.pascal_case()) - .collect::>() - .into_iter() - .collect::>() - .join(", ") - ) - } } impl DataType { @@ -625,10 +635,6 @@ impl CommonModel { } } - pub fn generate_as_typescript_schema(&self) -> String { - self.as_typescript_schema() - } - /// Generates the model as a string in the specified language /// with recursively expanded inner models and enums. /// This is useful for generating the entire model and its From ce653e12de60713efb02a86b19b8018134f42130 Mon Sep 17 00:00:00 2001 From: Samuel Gomez Date: Wed, 24 Jul 2024 21:30:33 +0100 Subject: [PATCH 2/7] Adding napi values to the rust schema generator --- integrationos-api/src/endpoints/mod.rs | 18 ++++++++--- .../src/endpoints/schema_generator.rs | 22 ++++++++++--- .../src/domain/connection/api_model_config.rs | 3 +- .../src/domain/schema/common_model.rs | 31 ++++++++++++++++++- 4 files changed, 63 insertions(+), 11 deletions(-) diff --git a/integrationos-api/src/endpoints/mod.rs b/integrationos-api/src/endpoints/mod.rs index f5829444..2146ecf3 100644 --- a/integrationos-api/src/endpoints/mod.rs +++ b/integrationos-api/src/endpoints/mod.rs @@ -11,6 +11,7 @@ use axum::{ Extension, Json, }; use bson::{doc, SerializerOptions}; +use futures::TryFutureExt; use http::{HeaderMap, HeaderValue, StatusCode}; use integrationos_cache::local::connection_cache::ConnectionCacheArcStrHeaderKey; use integrationos_domain::{ @@ -184,17 +185,26 @@ where ); let store = T::get_store(state.app_stores.clone()); - let count = store.count(query.filter.clone(), None); + let total = store + .collection + .estimated_document_count(None) + .map_err(|e| { + error!("Error counting documents: {e}"); + internal_server_error!() + }) + .await?; let find = store.get_many( Some(query.filter), None, - None, + Some(doc! { + "createdAt": -1 + }), Some(query.limit), Some(query.skip), ); - let res = match try_join!(count, find) { - Ok((total, rows)) => ReadResponse { + let res = match try_join!(find) { + Ok((rows,)) => ReadResponse { rows: rows.into_iter().map(T::public).collect(), skip: query.skip, limit: query.limit, diff --git a/integrationos-api/src/endpoints/schema_generator.rs b/integrationos-api/src/endpoints/schema_generator.rs index 26dbb335..420c391a 100644 --- a/integrationos-api/src/endpoints/schema_generator.rs +++ b/integrationos-api/src/endpoints/schema_generator.rs @@ -11,16 +11,18 @@ use integrationos_domain::{ api_model_config::Lang, ApplicationError, Id, IntegrationOSError, InternalError, Store, }; use mongodb::options::FindOptions; +use serde::Deserialize; use std::sync::Arc; pub fn get_router() -> Router> { Router::new() - .route("/projection", get(get_common_model_proj)) + .route("/projection", get(get_common_models_projections)) .route("/:id", get(generate_schema)) - .route("/types/:id", get(generate_types)) + .route("/types/:id/:lang", get(generate_types)) } -pub async fn get_common_model_proj( +#[tracing::instrument(name = "generate::schema::projection", skip(state))] +pub async fn get_common_models_projections( state: State>, ) -> Result>, IntegrationOSError> { let collection = state @@ -64,10 +66,19 @@ pub async fn get_common_model_proj( })) } +#[derive(Debug, Deserialize)] +struct TypeParams { + id: Id, + lang: Lang, +} + +#[tracing::instrument(name = "generate::schema::types", skip(state), fields(id = %id, lang = %lang))] pub async fn generate_types( state: State>, - Path(id): Path, + Path(TypeParams { id, lang }): Path, ) -> Result { + println!("id: {}, lang: {}", id, lang); + let cm_store = state.app_stores.common_model.clone(); let ce_store = state.app_stores.common_enum.clone(); @@ -81,12 +92,13 @@ pub async fn generate_types( ))?; let schema = common_model - .generate_as_expanded(&Lang::TypeScript, &cm_store, &ce_store) + .generate_as_expanded(&lang, &cm_store, &ce_store) .await; Ok(schema) } +#[tracing::instrument(name = "generate::schema", skip(state, id), fields(id = %id))] pub async fn generate_schema( state: State>, Path(id): Path, diff --git a/integrationos-domain/src/domain/connection/api_model_config.rs b/integrationos-domain/src/domain/connection/api_model_config.rs index cd57d677..57946494 100644 --- a/integrationos-domain/src/domain/connection/api_model_config.rs +++ b/integrationos-domain/src/domain/connection/api_model_config.rs @@ -3,6 +3,7 @@ use js_sandbox_ios::Script; use serde::{de::DeserializeOwned, Deserialize, Serialize}; use serde_json::Value; use std::collections::BTreeMap; +use strum::Display; use crate::{prelude::schema::json_schema::JsonSchema, IntegrationOSError, InternalError}; @@ -169,7 +170,7 @@ pub struct Compute { pub language: Lang, } -#[derive(Debug, Clone, Eq, PartialEq, Hash, Deserialize, Serialize, Default)] +#[derive(Debug, Clone, Eq, PartialEq, Hash, Deserialize, Serialize, Default, Display)] #[cfg_attr(feature = "dummy", derive(fake::Dummy))] #[serde(rename_all = "lowercase")] pub enum Lang { diff --git a/integrationos-domain/src/domain/schema/common_model.rs b/integrationos-domain/src/domain/schema/common_model.rs index 956b29d0..978e794d 100644 --- a/integrationos-domain/src/domain/schema/common_model.rs +++ b/integrationos-domain/src/domain/schema/common_model.rs @@ -187,6 +187,35 @@ impl CommonEnum { ) } + /// Generates a napi annotated enum for the enum rust type + pub fn as_rust_schema(&self) -> String { + format!( + "{} pub enum {} {{ {} }}\n", + "#[napi(string_enum = \"kebab-case\", js_name = {})]", + replace_reserved_keyword(&self.name, Lang::Rust) + .replace("::", "") + .pascal_case(), + self.options + .iter() + .map(|option| { + let option_name = option.pascal_case(); + let option_value = if option.chars().all(char::is_uppercase) { + option.to_lowercase() + } else { + option.kebab_case() + }; + + let option_annotation = format!("#[napi(value = \"{}\")]", option_value); + + format!("{} {}", option_annotation, option_name) + }) + .collect::>() + .into_iter() + .collect::>() + .join(", ") + ) + } + pub fn as_typescript_type(&self) -> String { // let's add the value directly to the enum format!( @@ -883,7 +912,7 @@ impl CommonModel { } visited_enums.insert(enum_model.id); - Some(enum_model.as_rust_type()) + Some(enum_model.as_rust_schema()) }) .collect::>() .into_iter() From fc1130506b40f69d8b8f28b41beda49d6a596bac Mon Sep 17 00:00:00 2001 From: Samuel Gomez Date: Wed, 24 Jul 2024 21:40:36 +0100 Subject: [PATCH 3/7] Using proper counter for filtered documents --- integrationos-api/src/endpoints/mod.rs | 22 +++++----------------- integrationos-domain/src/algebra/store.rs | 19 +++++++++++++++++-- 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/integrationos-api/src/endpoints/mod.rs b/integrationos-api/src/endpoints/mod.rs index 2146ecf3..d325f81d 100644 --- a/integrationos-api/src/endpoints/mod.rs +++ b/integrationos-api/src/endpoints/mod.rs @@ -11,7 +11,6 @@ use axum::{ Extension, Json, }; use bson::{doc, SerializerOptions}; -use futures::TryFutureExt; use http::{HeaderMap, HeaderValue, StatusCode}; use integrationos_cache::local::connection_cache::ConnectionCacheArcStrHeaderKey; use integrationos_domain::{ @@ -22,7 +21,6 @@ use mongodb::options::FindOneOptions; use serde::{de::DeserializeOwned, Deserialize, Serialize}; use serde_json::Value; use std::{collections::BTreeMap, fmt::Debug, sync::Arc}; -use tokio::try_join; use tracing::error; pub mod common_enum; @@ -185,30 +183,20 @@ where ); let store = T::get_store(state.app_stores.clone()); - let total = store - .collection - .estimated_document_count(None) - .map_err(|e| { - error!("Error counting documents: {e}"); - internal_server_error!() - }) - .await?; - let find = store.get_many( + let find = store.get_many_with_count( Some(query.filter), None, - Some(doc! { - "createdAt": -1 - }), + None, Some(query.limit), Some(query.skip), ); - let res = match try_join!(find) { - Ok((rows,)) => ReadResponse { + let res = match find.await { + Ok((rows, total)) => ReadResponse { rows: rows.into_iter().map(T::public).collect(), skip: query.skip, limit: query.limit, - total, + total: total as u64, }, Err(e) => { error!("Error reading from store: {e}"); diff --git a/integrationos-domain/src/algebra/store.rs b/integrationos-domain/src/algebra/store.rs index e4b1b415..28a16498 100644 --- a/integrationos-domain/src/algebra/store.rs +++ b/integrationos-domain/src/algebra/store.rs @@ -1,6 +1,7 @@ use crate::IntegrationOSError; use crate::Store; use bson::doc; +use futures::StreamExt; use futures::TryStreamExt; use mongodb::bson::Document; use mongodb::options::CountOptions; @@ -56,6 +57,19 @@ impl MongoStore limit: Option, skip: Option, ) -> Result, IntegrationOSError> { + self.get_many_with_count(filter, selection, sort, limit, skip) + .await + .map(|(r, _)| r) + } + + pub async fn get_many_with_count( + &self, + filter: Option, + selection: Option, + sort: Option, + limit: Option, + skip: Option, + ) -> Result<(Vec, usize), IntegrationOSError> { let mut filter_options = mongodb::options::FindOptions::default(); filter_options.sort = sort; filter_options.projection = selection; @@ -66,10 +80,11 @@ impl MongoStore filter_options.sort = Some(doc! { "createdAt": -1 }); } - let cursor = self.collection.find(filter, filter_options).await?; + let mut cursor = self.collection.find(filter.clone(), filter_options).await?; + let count = cursor.by_ref().count().await; let records = cursor.try_collect().await?; - Ok(records) + Ok((records, count.to_owned())) } pub async fn create_one(&self, data: &T) -> Result<(), IntegrationOSError> { From 1a6d0fd5380e43e41d4dcfc07d00ca71a86baaa2 Mon Sep 17 00:00:00 2001 From: Samuel Gomez Date: Wed, 24 Jul 2024 21:41:52 +0100 Subject: [PATCH 4/7] Using as_rust_type on rust type generation --- integrationos-domain/src/domain/schema/common_model.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integrationos-domain/src/domain/schema/common_model.rs b/integrationos-domain/src/domain/schema/common_model.rs index 978e794d..094e3b8e 100644 --- a/integrationos-domain/src/domain/schema/common_model.rs +++ b/integrationos-domain/src/domain/schema/common_model.rs @@ -912,7 +912,7 @@ impl CommonModel { } visited_enums.insert(enum_model.id); - Some(enum_model.as_rust_schema()) + Some(enum_model.as_rust_type()) }) .collect::>() .into_iter() From 4101bb73e87604d7453f04112c63cdafa091db7e Mon Sep 17 00:00:00 2001 From: Samuel Gomez Date: Wed, 24 Jul 2024 23:05:13 +0100 Subject: [PATCH 5/7] Reverting change --- integrationos-api/src/endpoints/mod.rs | 12 ++++++---- .../tests/api_tests/pagination_tests.rs | 24 +++++++++++-------- integrationos-domain/src/algebra/store.rs | 19 ++------------- 3 files changed, 24 insertions(+), 31 deletions(-) diff --git a/integrationos-api/src/endpoints/mod.rs b/integrationos-api/src/endpoints/mod.rs index d325f81d..bd8ffb20 100644 --- a/integrationos-api/src/endpoints/mod.rs +++ b/integrationos-api/src/endpoints/mod.rs @@ -21,6 +21,7 @@ use mongodb::options::FindOneOptions; use serde::{de::DeserializeOwned, Deserialize, Serialize}; use serde_json::Value; use std::{collections::BTreeMap, fmt::Debug, sync::Arc}; +use tokio::try_join; use tracing::error; pub mod common_enum; @@ -183,20 +184,23 @@ where ); let store = T::get_store(state.app_stores.clone()); - let find = store.get_many_with_count( + let total = store.count(query.filter.clone(), None); + let find = store.get_many( Some(query.filter), None, - None, + Some(doc! { + "createdAt": -1 + }), Some(query.limit), Some(query.skip), ); - let res = match find.await { + let res = match try_join!(find, total) { Ok((rows, total)) => ReadResponse { rows: rows.into_iter().map(T::public).collect(), skip: query.skip, limit: query.limit, - total: total as u64, + total, }, Err(e) => { error!("Error reading from store: {e}"); diff --git a/integrationos-api/tests/api_tests/pagination_tests.rs b/integrationos-api/tests/api_tests/pagination_tests.rs index c0336a55..279c4c75 100644 --- a/integrationos-api/tests/api_tests/pagination_tests.rs +++ b/integrationos-api/tests/api_tests/pagination_tests.rs @@ -36,6 +36,8 @@ async fn test_pagination() { ref config, } = req; + println!("{}", serde_json::to_string_pretty(&pipeline).unwrap()); + assert_eq!(name, pipeline.name); assert_eq!(key, pipeline.key); assert_eq!(source, pipeline.source); @@ -45,17 +47,17 @@ async fn test_pagination() { assert_eq!(config, pipeline.config.as_ref().unwrap()); pipelines.push(pipeline); - sleep(Duration::from_millis(1)).await; + sleep(Duration::from_millis(100)).await; } let pipelines: Vec = pipelines.into_iter().rev().collect(); check_response(&server, 1, 0, &pipelines[..1]).await; - check_response(&server, 10, 0, &pipelines).await; - check_response(&server, 0, 10, &pipelines[10..]).await; - check_response(&server, 5, 0, &pipelines[..5]).await; - check_response(&server, 5, 5, &pipelines[5..]).await; - check_response(&server, 5, 10, &pipelines[10..]).await; + // check_response(&server, 10, 0, &pipelines).await; + // check_response(&server, 0, 10, &pipelines[10..]).await; + // check_response(&server, 5, 0, &pipelines[..5]).await; + // check_response(&server, 5, 5, &pipelines[5..]).await; + // check_response(&server, 5, 10, &pipelines[10..]).await; } async fn check_response(server: &TestServer, limit: u64, skip: u64, pipelines: &[Pipeline]) { @@ -68,11 +70,13 @@ async fn check_response(server: &TestServer, limit: u64, skip: u64, pipelines: & ) .await .unwrap(); + assert_eq!(res.code, StatusCode::OK); let res: ReadResponse = serde_json::from_value(res.data).unwrap(); - assert_eq!(&res.rows, pipelines); - assert_eq!(res.limit, limit); - assert_eq!(res.skip, skip); - assert_eq!(res.total, 10); + println!("{:?}", res); + // assert_eq!(&res.rows, pipelines); + // assert_eq!(res.limit, limit); + // assert_eq!(res.skip, skip); + // assert_eq!(res.total, 10); } diff --git a/integrationos-domain/src/algebra/store.rs b/integrationos-domain/src/algebra/store.rs index 28a16498..e4b1b415 100644 --- a/integrationos-domain/src/algebra/store.rs +++ b/integrationos-domain/src/algebra/store.rs @@ -1,7 +1,6 @@ use crate::IntegrationOSError; use crate::Store; use bson::doc; -use futures::StreamExt; use futures::TryStreamExt; use mongodb::bson::Document; use mongodb::options::CountOptions; @@ -57,19 +56,6 @@ impl MongoStore limit: Option, skip: Option, ) -> Result, IntegrationOSError> { - self.get_many_with_count(filter, selection, sort, limit, skip) - .await - .map(|(r, _)| r) - } - - pub async fn get_many_with_count( - &self, - filter: Option, - selection: Option, - sort: Option, - limit: Option, - skip: Option, - ) -> Result<(Vec, usize), IntegrationOSError> { let mut filter_options = mongodb::options::FindOptions::default(); filter_options.sort = sort; filter_options.projection = selection; @@ -80,11 +66,10 @@ impl MongoStore filter_options.sort = Some(doc! { "createdAt": -1 }); } - let mut cursor = self.collection.find(filter.clone(), filter_options).await?; - let count = cursor.by_ref().count().await; + let cursor = self.collection.find(filter, filter_options).await?; let records = cursor.try_collect().await?; - Ok((records, count.to_owned())) + Ok(records) } pub async fn create_one(&self, data: &T) -> Result<(), IntegrationOSError> { From cc4107a68541fa2cbf044df02dc195b5cabfdf4a Mon Sep 17 00:00:00 2001 From: Samuel Gomez Date: Wed, 24 Jul 2024 23:06:02 +0100 Subject: [PATCH 6/7] Removing printlns --- .../tests/api_tests/pagination_tests.rs | 21 ++++++++----------- 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/integrationos-api/tests/api_tests/pagination_tests.rs b/integrationos-api/tests/api_tests/pagination_tests.rs index 279c4c75..ffedca0c 100644 --- a/integrationos-api/tests/api_tests/pagination_tests.rs +++ b/integrationos-api/tests/api_tests/pagination_tests.rs @@ -36,8 +36,6 @@ async fn test_pagination() { ref config, } = req; - println!("{}", serde_json::to_string_pretty(&pipeline).unwrap()); - assert_eq!(name, pipeline.name); assert_eq!(key, pipeline.key); assert_eq!(source, pipeline.source); @@ -53,11 +51,11 @@ async fn test_pagination() { let pipelines: Vec = pipelines.into_iter().rev().collect(); check_response(&server, 1, 0, &pipelines[..1]).await; - // check_response(&server, 10, 0, &pipelines).await; - // check_response(&server, 0, 10, &pipelines[10..]).await; - // check_response(&server, 5, 0, &pipelines[..5]).await; - // check_response(&server, 5, 5, &pipelines[5..]).await; - // check_response(&server, 5, 10, &pipelines[10..]).await; + check_response(&server, 10, 0, &pipelines).await; + check_response(&server, 0, 10, &pipelines[10..]).await; + check_response(&server, 5, 0, &pipelines[..5]).await; + check_response(&server, 5, 5, &pipelines[5..]).await; + check_response(&server, 5, 10, &pipelines[10..]).await; } async fn check_response(server: &TestServer, limit: u64, skip: u64, pipelines: &[Pipeline]) { @@ -74,9 +72,8 @@ async fn check_response(server: &TestServer, limit: u64, skip: u64, pipelines: & assert_eq!(res.code, StatusCode::OK); let res: ReadResponse = serde_json::from_value(res.data).unwrap(); - println!("{:?}", res); - // assert_eq!(&res.rows, pipelines); - // assert_eq!(res.limit, limit); - // assert_eq!(res.skip, skip); - // assert_eq!(res.total, 10); + assert_eq!(&res.rows, pipelines); + assert_eq!(res.limit, limit); + assert_eq!(res.skip, skip); + assert_eq!(res.total, 10); } From 1ce1f54e61058c9b292f935420ef0b0a2a00600d Mon Sep 17 00:00:00 2001 From: Samuel Gomez Date: Wed, 24 Jul 2024 23:07:51 +0100 Subject: [PATCH 7/7] Adding small comment --- integrationos-api/src/endpoints/mod.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/integrationos-api/src/endpoints/mod.rs b/integrationos-api/src/endpoints/mod.rs index bd8ffb20..7a120e0a 100644 --- a/integrationos-api/src/endpoints/mod.rs +++ b/integrationos-api/src/endpoints/mod.rs @@ -184,13 +184,12 @@ where ); let store = T::get_store(state.app_stores.clone()); + // TODO: Investigate how to improve performance here let total = store.count(query.filter.clone(), None); let find = store.get_many( Some(query.filter), None, - Some(doc! { - "createdAt": -1 - }), + None, Some(query.limit), Some(query.skip), );