diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 3fdb2fe329933..b8d7319fb4918 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "Inflector" @@ -10555,6 +10555,7 @@ dependencies = [ "tracing-loki", "tracing-subscriber", "uuid 1.11.0", + "windmill-macros", ] [[package]] @@ -10594,6 +10595,18 @@ dependencies = [ "windmill-common", ] +[[package]] +name = "windmill-macros" +version = "1.405.5" +dependencies = [ + "itertools 0.10.5", + "lazy_static", + "proc-macro2", + "quote", + "regex", + "syn 2.0.79", +] + [[package]] name = "windmill-parser" version = "1.409.2" diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 579dec6e6be9d..c8b68ac1b1c49 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -14,6 +14,7 @@ members = [ "./windmill-audit", "./windmill-git-sync", "./windmill-indexer", + "./windmill-macros", "./parsers/windmill-parser", "./parsers/windmill-parser-ts", "./parsers/windmill-parser-wasm", @@ -23,7 +24,7 @@ members = [ "./parsers/windmill-parser-py", "./parsers/windmill-parser-py-imports", "./parsers/windmill-sql-datatype-parser-wasm", - "./parsers/windmill-parser-yaml", + "./parsers/windmill-parser-yaml", "windmill-macros", ] [workspace.package] @@ -116,6 +117,7 @@ windmill-common = { path = "./windmill-common", default-features = false } windmill-audit = { path = "./windmill-audit" } windmill-git-sync = { path = "./windmill-git-sync" } windmill-indexer = {path = "./windmill-indexer"} +windmill-macros = {path = "./windmill-macros"} windmill-parser = { path = "./parsers/windmill-parser" } windmill-parser-ts = { path = "./parsers/windmill-parser-ts" } windmill-parser-py = { path = "./parsers/windmill-parser-py" } @@ -281,6 +283,8 @@ triomphe = "^0" tantivy = "0.22.0" +# Macro-related +proc-macro2 = "1.0" pulldown-cmark = "0.9" toml = "0.7" syn = { version = "2.0.74", features = ["full"] } diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index e975b5a655d10..817a6f0d024b4 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -4193,10 +4193,10 @@ async fn run_dependencies_job( JsonRawValue::from_string("true".to_string()).unwrap(), ); if language == ScriptLang::Bun { - let annotation = windmill_common::worker::get_annotation_ts(&raw_code); + let annotation = windmill_common::worker::TypeScriptAnnotations::parse(&raw_code); hm.insert( "npm_mode".to_string(), - JsonRawValue::from_string(annotation.npm_mode.to_string()).unwrap(), + JsonRawValue::from_string(annotation.npm.to_string()).unwrap(), ); } (PushArgs { extra: Some(hm), args: &ehm }, deps) diff --git a/backend/windmill-api/src/scripts.rs b/backend/windmill-api/src/scripts.rs index 63a5f49f92a98..9b91c6b5fb873 100644 --- a/backend/windmill-api/src/scripts.rs +++ b/backend/windmill-api/src/scripts.rs @@ -56,7 +56,7 @@ use windmill_common::{ utils::{ not_found_if_none, paginate, query_elems_from_hub, require_admin, Pagination, StripPath, }, - worker::{get_annotation_ts, to_raw_value}, + worker::to_raw_value, HUB_BASE_URL, }; use windmill_git_sync::{handle_deployment_metadata, DeployedObject}; @@ -606,8 +606,8 @@ async fn create_script_internal<'c>( }; let lang = if &ns.language == &ScriptLang::Bun || &ns.language == &ScriptLang::Bunnative { - let anns = get_annotation_ts(&ns.content); - if anns.native_mode { + let anns = windmill_common::worker::TypeScriptAnnotations::parse(&ns.content); + if anns.native { ScriptLang::Bunnative } else { ScriptLang::Bun diff --git a/backend/windmill-common/Cargo.toml b/backend/windmill-common/Cargo.toml index 546c339575162..f3adbfde841f5 100644 --- a/backend/windmill-common/Cargo.toml +++ b/backend/windmill-common/Cargo.toml @@ -57,6 +57,7 @@ futures-core.workspace = true async-stream.workspace = true const_format.workspace = true crc.workspace = true +windmill-macros.workspace = true [target.'cfg(not(target_env = "msvc"))'.dependencies] tikv-jemalloc-ctl = { optional = true, workspace = true } diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 89f2c148e0969..37ae61c8d67df 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -13,6 +13,7 @@ use std::{ sync::{atomic::AtomicBool, Arc}, }; use tokio::sync::RwLock; +use windmill_macros::annotations; use crate::{error, global_settings::CUSTOM_TAGS_SETTING, server::Smtp, DB}; @@ -303,64 +304,25 @@ fn parse_file(path: &str) -> Option { .flatten() } -pub struct TypeScriptAnnotations { - pub npm_mode: bool, - pub nodejs_mode: bool, - pub native_mode: bool, - pub nobundling: bool, -} - -pub fn get_annotation_ts(inner_content: &str) -> TypeScriptAnnotations { - let annotations = inner_content - .lines() - .take_while(|x| x.starts_with("//")) - .map(|x| x.to_string().replace("//", "").trim().to_string()) - .collect_vec(); - let nodejs_mode: bool = annotations.contains(&"nodejs".to_string()); - let npm_mode: bool = annotations.contains(&"npm".to_string()); - let native_mode: bool = annotations.contains(&"native".to_string()); - - //TODO: remove || npm_mode when bun build is more powerful - let nobundling: bool = - annotations.contains(&"nobundling".to_string()) || nodejs_mode || *DISABLE_BUNDLING; - - TypeScriptAnnotations { npm_mode, nodejs_mode, native_mode, nobundling } -} - +#[annotations("#")] pub struct PythonAnnotations { - pub no_uv: bool, pub no_cache: bool, + pub no_uv: bool, } -pub fn get_annotation_python(inner_content: &str) -> PythonAnnotations { - let annotations = inner_content - .lines() - .take_while(|x| x.starts_with("#")) - .map(|x| x.to_string().replace("#", "").trim().to_string()) - .collect_vec(); - - let no_uv: bool = annotations.contains(&"no_uv".to_string()); - let no_cache: bool = annotations.contains(&"no_cache".to_string()); - - PythonAnnotations { no_uv, no_cache } +#[annotations("//")] +pub struct TypeScriptAnnotations { + pub npm: bool, + pub nodejs: bool, + pub native: bool, + pub nobundling: bool, } +#[annotations("--")] pub struct SqlAnnotations { pub return_last_result: bool, } -pub fn get_sql_annotations(inner_content: &str) -> SqlAnnotations { - let annotations = inner_content - .lines() - .take_while(|x| x.starts_with("--")) - .map(|x| x.to_string().replace("--", "").trim().to_string()) - .collect_vec(); - - let return_last_result: bool = annotations.contains(&"return_last_result".to_string()); - - SqlAnnotations { return_last_result } -} - pub async fn load_cache(bin_path: &str, _remote_path: &str) -> (bool, String) { if tokio::fs::metadata(&bin_path).await.is_ok() { (true, format!("loaded from local cache: {}\n", bin_path)) diff --git a/backend/windmill-macros/Cargo.toml b/backend/windmill-macros/Cargo.toml new file mode 100644 index 0000000000000..100b4678d31a9 --- /dev/null +++ b/backend/windmill-macros/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "windmill-macros" +version.workspace = true +authors.workspace = true +edition.workspace = true + +[lib] +proc-macro = true + +[dependencies] +proc-macro2.workspace = true +quote.workspace = true +syn.workspace = true + +# Dependencies for tests +[dev-dependencies] +# tests/annotation.rs +lazy_static.workspace = true +itertools.workspace = true +regex.workspace = true diff --git a/backend/windmill-macros/src/lib.rs b/backend/windmill-macros/src/lib.rs new file mode 100644 index 0000000000000..25ff8cbaa2141 --- /dev/null +++ b/backend/windmill-macros/src/lib.rs @@ -0,0 +1,95 @@ +use proc_macro::TokenStream; +use quote::quote; +use syn::{parse_macro_input, Ident, ItemStruct, Lit}; + +#[proc_macro_attribute] +pub fn annotations(attr: TokenStream, item: TokenStream) -> TokenStream { + let input = parse_macro_input!(item as ItemStruct); + let name = input.ident.clone(); + let fields = input + .fields + .iter() + .map(|f| f.ident.clone().unwrap()) + .collect::>(); + + // Match on the literal to extract the string value + let comm_lit = match parse_macro_input!(attr as Lit) { + Lit::Str(lit_str) => lit_str.value(), // This will give "#" without quotes + _ => panic!("Expected a string literal"), + }; + + // Generate regex + let mut reg = format!("^{}|", &comm_lit); + { + for field in fields.iter() { + reg.push_str(&(field.to_string())); + reg.push_str("\\b"); + } + + reg.push_str(r#"|\w+"#); + } + // Example of generated regex: + // ^# + // |ann1\b|ann2\b|ann3\b|ann4\b + // |\w+ + + TokenStream::from(quote! { + #[derive(Default, Debug)] + #input + + impl std::ops::BitOrAssign for #name{ + fn bitor_assign(&mut self, rhs: Self) { + // Unfold fields + // Read more: https://docs.rs/quote/latest/quote/macro.quote.html#interpolation + #( self.#fields |= rhs.#fields; )* + } + } + + impl #name { + /// Autogenerated by windmill-macros + pub fn parse(inner_content: &str) -> Self{ + let mut res = Self::default(); + lazy_static::lazy_static! { + static ref RE: regex::Regex = regex::Regex::new(#reg).unwrap(); + } + // Create lines stream + let mut lines = inner_content.lines(); + 'outer: while let Some(line) = lines.next() { + // If comment sign(s) on the right place + let mut comms = false; + // New instance + // We will apply it if in line only annotations + let mut new = Self::default(); + + 'inner: for (i, mat) in RE.find_iter(line).enumerate() { + + match mat.as_str(){ + #comm_lit if i == 0 => { + comms = true; + continue 'inner; + }, + + // Will expand into something like: + // "ann1" => new.ann1 = true, + // "ann2" => new.ann2 = true, + // "ann3" => new.ann3 = true, + #( stringify!(#fields) => new.#fields = true, )* + // Non annotations + _ => continue 'outer, + }; + } + + if !comms { + // We dont want to continue if line does not start with # + return res; + } + + // Apply changes + res |= new; + } + + res + } + } + }) +} diff --git a/backend/windmill-macros/tests/annotations.rs b/backend/windmill-macros/tests/annotations.rs new file mode 100644 index 0000000000000..fd430dfaeb755 --- /dev/null +++ b/backend/windmill-macros/tests/annotations.rs @@ -0,0 +1,169 @@ +#[cfg(test)] +mod annotations_tests { + + extern crate windmill_macros; + use itertools::Itertools; + use windmill_macros::annotations; + + // Previous implementation. + // We have to make sure that new one works the same as old one + fn old(inner_content: &str) -> Annotations { + let annotations = inner_content + .lines() + .take_while(|x| x.starts_with("#")) + .map(|x| x.to_string().replace("#", "").trim().to_string()) + .collect_vec(); + + let ann1: bool = annotations.contains(&"ann1".to_string()); + let ann2: bool = annotations.contains(&"ann2".to_string()); + let ann3: bool = annotations.contains(&"ann3".to_string()); + let ann4: bool = annotations.contains(&"ann4".to_string()); + let ann5: bool = annotations.contains(&"ann5".to_string()); + + Annotations { ann1, ann2, ann3, ann4, ann5 } + } + + #[annotations("#")] + #[derive(Eq, PartialEq, Copy, Clone)] + pub struct Annotations { + pub ann1: bool, + pub ann2: bool, + pub ann3: bool, + pub ann4: bool, + pub ann5: bool, + } + + #[annotations("//")] + #[derive(Eq, PartialEq, Copy, Clone)] + pub struct SlashedAnnotations { + pub ann1: bool, + pub ann2: bool, + pub ann3: bool, + pub ann4: bool, + } + + #[annotations("--")] + #[derive(Eq, PartialEq, Copy, Clone)] + pub struct MinusedAnnotations { + pub ann1: bool, + pub ann2: bool, + } + + // e.g. rust, TS and JS + #[test] + fn slashed_annotations() { + let cont = "// ann1 +// ann2 +//ann3"; + assert_eq!( + SlashedAnnotations { ann1: true, ann2: true, ann3: true, ann4: false }, + SlashedAnnotations::parse(cont) + ); + } + + // e.g. Haskell, SQL + #[test] + fn minused_annotations() { + let cont = "-- ann1 +-- ann2"; + assert_eq!( + MinusedAnnotations { ann1: true, ann2: true }, + MinusedAnnotations::parse(cont) + ); + } + + #[test] + fn simple_integration() { + let cont = "# ann1"; + let expected = Annotations { ann1: true, ..Default::default() }; + assert_eq!(expected, old(cont)); + assert_eq!(expected, Annotations::parse(cont)); + } + + #[test] + fn multiline_integration() { + let cont = "# ann2 +# ann3 +# ann4 +# ann5 + "; + let expected = Annotations { + ann1: false, + ann2: true, + ann3: true, + ann4: true, + ann5: true, + // + }; + assert_eq!(expected, old(cont)); + assert_eq!(expected, Annotations::parse(cont)); + } + + #[test] + fn spacing_integration() { + // First line is ignored and not used + { + let cont = " +# ann2"; + let expected = Annotations { ..Default::default() }; + assert_eq!(expected, old(cont)); + assert_eq!(expected, Annotations::parse(cont)); + } + // Wrong spacing for ann3 + { + let cont = "# ann2 + # ann3"; + + let expected = Annotations { ann2: true, ..Default::default() }; + assert_eq!(expected, old(cont)); + assert_eq!(expected, Annotations::parse(cont)); + } + + // Drunk but valid spacing + { + let cont = "#ann1 +# ann2"; + let expected = Annotations { ann2: true, ann1: true, ..Default::default() }; + assert_eq!(expected, old(cont)); + assert_eq!(expected, Annotations::parse(cont)); + } + } + + #[test] + fn comments_inbetween_integration() { + let cont = "# ann2 +# Just comment, has nothing to do with annotations +# Another comment: ann1 ann2 ann3 +# ann4 is not valid annotation +# Actual annotation next line: +# ann5 + +# Should be ignored +# ann3 + "; + let expected = Annotations { ann2: true, ann5: true, ..Default::default() }; + assert_eq!(expected, old(cont)); + assert_eq!(expected, Annotations::parse(cont)); + } + + #[test] + fn hash_collision() { + // TODO + } + #[test] + fn non_matching_integration() { + { + let cont = r#" "ann1", ann2 "#; + let expected = Annotations::default(); + assert_eq!(expected, old(cont)); + assert_eq!(expected, Annotations::parse(cont)); + } + // Empty + { + let cont = ""; + let expected = Annotations::default(); + assert_eq!(expected, old(cont)); + assert_eq!(expected, Annotations::parse(cont)); + } + } +} diff --git a/backend/windmill-worker/src/bigquery_executor.rs b/backend/windmill-worker/src/bigquery_executor.rs index 1552686b79c94..243839268029b 100644 --- a/backend/windmill-worker/src/bigquery_executor.rs +++ b/backend/windmill-worker/src/bigquery_executor.rs @@ -5,7 +5,6 @@ use futures::{FutureExt, TryFutureExt}; use serde_json::{json, value::RawValue, Value}; use windmill_common::error::to_anyhow; use windmill_common::jobs::QueuedJob; -use windmill_common::worker::get_sql_annotations; use windmill_common::{error::Error, worker::to_raw_value}; use windmill_parser_sql::{ parse_bigquery_sig, parse_db_resource, parse_sql_blocks, parse_sql_statement_named_params, @@ -238,7 +237,7 @@ pub async fn do_bigquery( return Err(Error::BadRequest("Missing database argument".to_string())); }; - let annotations = get_sql_annotations(query); + let annotations = windmill_common::worker::SqlAnnotations::parse(query); let service_account = CustomServiceAccount::from_json(&database) .map_err(|e| Error::ExecutionErr(e.to_string()))?; diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 74a381544f7e5..07219185683a7 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -47,7 +47,7 @@ use windmill_common::{ get_latest_hash_for_path, jobs::{QueuedJob, PREPROCESSOR_FAKE_ENTRYPOINT}, scripts::ScriptLang, - worker::{exists_in_cache, get_annotation_ts, save_cache, write_file}, + worker::{exists_in_cache, save_cache, write_file}, DB, }; @@ -663,7 +663,7 @@ pub async fn prebundle_bun_script( if exists_in_cache(&local_path, &remote_path).await { return Ok(()); } - let annotation = get_annotation_ts(inner_content); + let annotation = windmill_common::worker::TypeScriptAnnotations::parse(inner_content); if annotation.nobundling { return Ok(()); } @@ -676,9 +676,9 @@ pub async fn prebundle_bun_script( &token, w_id, script_path, - if annotation.nodejs_mode { + if annotation.nodejs { LoaderMode::NodeBundle - } else if annotation.native_mode { + } else if annotation.native { LoaderMode::BrowserBundle } else { LoaderMode::BunBundle @@ -800,7 +800,7 @@ pub async fn handle_bun_job( new_args: &mut Option>>, occupancy_metrics: &mut OccupancyMetrics, ) -> error::Result> { - let mut annotation = windmill_common::worker::get_annotation_ts(inner_content); + let mut annotation = windmill_common::worker::TypeScriptAnnotations::parse(inner_content); let (mut has_bundle_cache, cache_logs, local_path, remote_path) = if requirements_o.is_some() && !annotation.nobundling && codebase.is_none() { @@ -822,7 +822,7 @@ pub async fn handle_bun_job( if !codebase.is_some() && !has_bundle_cache { let _ = write_file(job_dir, "main.ts", inner_content)?; - } else if !annotation.native_mode && codebase.is_none() { + } else if !annotation.native && codebase.is_none() { let _ = write_file(job_dir, "package.json", r#"{ "type": "module" }"#)?; }; @@ -830,7 +830,7 @@ pub async fn handle_bun_job( get_common_bun_proc_envs(Some(&base_internal_url)).await; if codebase.is_some() { - annotation.nodejs_mode = true + annotation.nodejs = true } let (main_override, apply_preprocessor) = match get_main_override(job.args.as_ref()) { Some(main_override) => { @@ -844,7 +844,7 @@ pub async fn handle_bun_job( }; #[cfg(not(feature = "enterprise"))] - if annotation.nodejs_mode || annotation.npm_mode { + if annotation.nodejs || annotation.npm { return Err(error::Error::ExecutionErr( "Nodejs / npm mode is an EE feature".to_string(), )); @@ -875,20 +875,20 @@ pub async fn handle_bun_job( pull_codebase(&job.workspace_id, codebase, job_dir).await?; } else if let Some(reqs) = requirements_o.as_ref() { let splitted = reqs.split(BUN_LOCKB_SPLIT).collect::>(); - if splitted.len() != 2 && !annotation.npm_mode { + if splitted.len() != 2 && !annotation.npm { return Err(error::Error::ExecutionErr( format!("Invalid requirements, expected to find //bun.lockb split pattern in reqs. Found: |{reqs}|") )); } let _ = write_file(job_dir, "package.json", &splitted[0])?; - let lockb = if annotation.npm_mode { "" } else { splitted[1] }; + let lockb = if annotation.npm { "" } else { splitted[1] }; if lockb != EMPTY_FILE { let mut skip_install = false; let mut create_buntar = false; let mut buntar_path = "".to_string(); - if !annotation.npm_mode { + if !annotation.npm { let _ = write_lockb(&splitted[1], job_dir).await?; let mut sha_path = sha2::Sha256::new(); @@ -921,7 +921,7 @@ pub async fn handle_bun_job( job_dir, worker_name, common_bun_proc_envs.clone(), - annotation.npm_mode, + annotation.npm, &mut Some(occupancy_metrics), ) .await?; @@ -963,7 +963,7 @@ pub async fn handle_bun_job( worker_name, false, None, - annotation.npm_mode, + annotation.npm, &mut Some(occupancy_metrics), ) .await?; @@ -971,19 +971,19 @@ pub async fn handle_bun_job( // } } - let mut init_logs = if annotation.native_mode { + let mut init_logs = if annotation.native { "\n\n--- NATIVE CODE EXECUTION ---\n".to_string() } else if has_bundle_cache { - if annotation.nodejs_mode { + if annotation.nodejs { "\n\n--- NODE BUNDLE SNAPSHOT EXECUTION ---\n".to_string() } else { "\n\n--- BUN BUNDLE SNAPSHOT EXECUTION ---\n".to_string() } } else if codebase.is_some() { "\n\n--- NODE CODEBASE SNAPSHOT EXECUTION ---\n".to_string() - } else if annotation.native_mode { + } else if annotation.native { "\n\n--- NATIVE CODE EXECUTION ---\n".to_string() - } else if annotation.nodejs_mode { + } else if annotation.nodejs { write_file(job_dir, "main.ts", &remove_pinned_imports(inner_content)?)?; "\n\n--- NODE CODE EXECUTION ---\n".to_string() } else { @@ -1003,7 +1003,7 @@ pub async fn handle_bun_job( } let write_wrapper_f = async { - if !has_bundle_cache && annotation.native_mode { + if !has_bundle_cache && annotation.native { return Ok(()) as error::Result<()>; } // let mut start = Instant::now(); @@ -1126,7 +1126,7 @@ try {{ let reserved_variables_args_out_f = async { let args_and_out_f = async { - if !annotation.native_mode { + if !annotation.native { create_args_and_out_file(&client, job, job_dir, db).await?; } Ok(()) as Result<()> @@ -1143,7 +1143,7 @@ try {{ let build_cache = !has_bundle_cache && !annotation.nobundling && !codebase.is_some() - && (requirements_o.is_some() || annotation.native_mode); + && (requirements_o.is_some() || annotation.native); let write_loader_f = async { if build_cache { @@ -1153,9 +1153,9 @@ try {{ &client.get_token().await, &job.workspace_id, &job.script_path(), - if annotation.nodejs_mode { + if annotation.nodejs { LoaderMode::NodeBundle - } else if annotation.native_mode { + } else if annotation.native { LoaderMode::BrowserBundle } else { LoaderMode::BunBundle @@ -1171,7 +1171,7 @@ try {{ &client.get_token().await, &job.workspace_id, &job.script_path(), - if annotation.nodejs_mode { + if annotation.nodejs { LoaderMode::Node } else { LoaderMode::Bun @@ -1217,7 +1217,7 @@ try {{ } } } - if !annotation.native_mode { + if !annotation.native { let ex_wrapper = read_file_content(&format!("{job_dir}/wrapper.mjs")).await?; write_file( job_dir, @@ -1231,7 +1231,7 @@ try {{ } fs::remove_file(format!("{job_dir}/main.ts"))?; has_bundle_cache = true; - } else if annotation.nodejs_mode { + } else if annotation.nodejs { generate_wrapper_mjs( job_dir, &job.workspace_id, @@ -1247,9 +1247,14 @@ try {{ .await?; } } - if annotation.native_mode { + if annotation.native { #[cfg(not(feature = "deno_core"))] - return Ok(to_raw_value("").unwrap()); + { + tracing::error!( + r#""deno_core" feature is not activated, but "//native" annotation used. Returning empty value..."# + ); + return Ok(to_raw_value("").unwrap()); + } #[cfg(feature = "deno_core")] { @@ -1309,14 +1314,7 @@ try {{ job_dir, "run.config.proto", &NSJAIL_CONFIG_RUN_BUN_CONTENT - .replace( - "{LANG}", - if annotation.nodejs_mode { - "nodejs" - } else { - "bun" - }, - ) + .replace("{LANG}", if annotation.nodejs { "nodejs" } else { "bun" }) .replace("{JOB_DIR}", job_dir) .replace("{CACHE_DIR}", BUN_CACHE_DIR) .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) @@ -1324,7 +1322,7 @@ try {{ "{SHARED_MOUNT}", &shared_mount.replace( "/tmp/shared", - if annotation.nodejs_mode { + if annotation.nodejs { "/tmp/nodejs/shared" } else { "/tmp/bun/shared" @@ -1334,7 +1332,7 @@ try {{ )?; let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); - let args = if annotation.nodejs_mode { + let args = if annotation.nodejs { vec![ "--config", "run.config.proto", @@ -1378,7 +1376,7 @@ try {{ .stderr(Stdio::piped()); start_child_process(nsjail_cmd, NSJAIL_PATH.as_str()).await? } else { - let cmd = if annotation.nodejs_mode { + let cmd = if annotation.nodejs { let script_path = format!("{job_dir}/wrapper.mjs"); let mut bun_cmd = Command::new(&*NODE_BIN_PATH); @@ -1430,7 +1428,7 @@ try {{ start_child_process( cmd, - if annotation.nodejs_mode { + if annotation.nodejs { &*NODE_BIN_PATH } else { &*BUN_PATH @@ -1535,10 +1533,10 @@ pub async fn start_worker( let common_bun_proc_envs: HashMap = get_common_bun_proc_envs(Some(&base_internal_url)).await; - let mut annotation = windmill_common::worker::get_annotation_ts(inner_content); + let mut annotation = windmill_common::worker::TypeScriptAnnotations::parse(inner_content); //TODO: remove this when bun dedicated workers work without issues - annotation.nodejs_mode = true; + annotation.nodejs = true; let context = variables::get_reserved_variables( db, @@ -1592,7 +1590,7 @@ pub async fn start_worker( job_dir, worker_name, common_bun_proc_envs.clone(), - annotation.npm_mode, + annotation.npm, &mut None, ) .await?; @@ -1613,7 +1611,7 @@ pub async fn start_worker( worker_name, false, None, - annotation.npm_mode, + annotation.npm, &mut None, ) .await?; @@ -1691,7 +1689,7 @@ for await (const line of Readline.createInterface({{ input: process.stdin }})) { token, w_id, script_path, - if annotation.nodejs_mode { + if annotation.nodejs { LoaderMode::Node } else { LoaderMode::Bun @@ -1700,7 +1698,7 @@ for await (const line of Readline.createInterface({{ input: process.stdin }})) { .await?; } - if annotation.nodejs_mode && !codebase.is_some() { + if annotation.nodejs && !codebase.is_some() { generate_wrapper_mjs( job_dir, w_id, @@ -1716,7 +1714,7 @@ for await (const line of Readline.createInterface({{ input: process.stdin }})) { .await?; } - if annotation.nodejs_mode { + if annotation.nodejs { let script_path = format!("{job_dir}/wrapper.mjs"); handle_dedicated_process( diff --git a/backend/windmill-worker/src/mssql_executor.rs b/backend/windmill-worker/src/mssql_executor.rs index f50bad4619daa..7c4724418c8b2 100644 --- a/backend/windmill-worker/src/mssql_executor.rs +++ b/backend/windmill-worker/src/mssql_executor.rs @@ -9,7 +9,7 @@ use tokio::net::TcpStream; use tokio_util::compat::TokioAsyncWriteCompatExt; use uuid::Uuid; use windmill_common::error::{self, Error}; -use windmill_common::worker::{get_sql_annotations, to_raw_value}; +use windmill_common::worker::to_raw_value; use windmill_common::{error::to_anyhow, jobs::QueuedJob}; use windmill_parser_sql::{parse_db_resource, parse_mssql_sig}; use windmill_queue::{append_logs, CanceledBy}; @@ -68,7 +68,7 @@ pub async fn do_mssql( return Err(Error::BadRequest("Missing database argument".to_string())); }; - let annotations = get_sql_annotations(query); + let annotations = windmill_common::worker::SqlAnnotations::parse(query); let mut config = Config::new(); diff --git a/backend/windmill-worker/src/mysql_executor.rs b/backend/windmill-worker/src/mysql_executor.rs index 03cde1d508e54..3ed6a9bf34be4 100644 --- a/backend/windmill-worker/src/mysql_executor.rs +++ b/backend/windmill-worker/src/mysql_executor.rs @@ -13,7 +13,7 @@ use tokio::sync::Mutex; use windmill_common::{ error::{to_anyhow, Error}, jobs::QueuedJob, - worker::{get_sql_annotations, to_raw_value}, + worker::to_raw_value, }; use windmill_parser_sql::{ parse_db_resource, parse_mysql_sig, parse_sql_blocks, parse_sql_statement_named_params, @@ -148,7 +148,7 @@ pub async fn do_mysql( return Err(Error::BadRequest("Missing database argument".to_string())); }; - let annotations = get_sql_annotations(query); + let annotations = windmill_common::worker::SqlAnnotations::parse(query); let opts = OptsBuilder::default() .db_name(Some(database.database)) diff --git a/backend/windmill-worker/src/pg_executor.rs b/backend/windmill-worker/src/pg_executor.rs index 7c4192cf66903..7702b73b03f5d 100644 --- a/backend/windmill-worker/src/pg_executor.rs +++ b/backend/windmill-worker/src/pg_executor.rs @@ -30,7 +30,7 @@ use tokio_postgres::{ }; use uuid::Uuid; use windmill_common::error::{self, Error}; -use windmill_common::worker::{get_sql_annotations, to_raw_value, CLOUD_HOSTED}; +use windmill_common::worker::{to_raw_value, CLOUD_HOSTED}; use windmill_common::{error::to_anyhow, jobs::QueuedJob}; use windmill_parser::{Arg, Typ}; use windmill_parser_sql::{ @@ -191,7 +191,7 @@ pub async fn do_postgresql( return Err(Error::BadRequest("Missing database argument".to_string())); }; - let annotations = get_sql_annotations(query); + let annotations = windmill_common::worker::SqlAnnotations::parse(query); let sslmode = match database.sslmode.as_deref() { Some("allow") => "prefer".to_string(), diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index e1d9c391d689d..2c8b3f2c84082 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -898,7 +898,7 @@ async fn handle_python_deps( let requirements = match requirements_o { Some(r) => r, None => { - let annotation = windmill_common::worker::get_annotation_python(inner_content); + let annotation = windmill_common::worker::PythonAnnotations::parse(inner_content); let mut already_visited = vec![]; let requirements = windmill_parser_py_imports::parse_python_imports( diff --git a/backend/windmill-worker/src/snowflake_executor.rs b/backend/windmill-worker/src/snowflake_executor.rs index bf9dae41ff394..f89b29832a34f 100644 --- a/backend/windmill-worker/src/snowflake_executor.rs +++ b/backend/windmill-worker/src/snowflake_executor.rs @@ -9,7 +9,6 @@ use serde_json::{json, value::RawValue, Value}; use sha2::{Digest, Sha256}; use std::collections::HashMap; use windmill_common::error::to_anyhow; -use windmill_common::worker::get_sql_annotations; use windmill_common::jobs::QueuedJob; use windmill_common::{error::Error, worker::to_raw_value}; @@ -266,7 +265,7 @@ pub async fn do_snowflake( return Err(Error::BadRequest("Missing database argument".to_string())); }; - let annotations = get_sql_annotations(query); + let annotations = windmill_common::worker::SqlAnnotations::parse(query); let qualified_username = format!( "{}.{}", diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index e6458375cdbc1..c2209d9e73012 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -12,7 +12,7 @@ use windmill_common::flows::{FlowModule, FlowModuleValue}; use windmill_common::get_latest_deployed_hash_for_path; use windmill_common::jobs::JobPayload; use windmill_common::scripts::ScriptHash; -use windmill_common::worker::{get_annotation_ts, to_raw_value, to_raw_value_owned, write_file}; +use windmill_common::worker::{to_raw_value, to_raw_value_owned, write_file}; use windmill_common::{ error::{self, to_anyhow}, flows::FlowValue, @@ -953,10 +953,10 @@ async fn lock_modules<'c>( } if language == ScriptLang::Bun || language == ScriptLang::Bunnative { - let anns = get_annotation_ts(&content); - if anns.native_mode && language == ScriptLang::Bun { + let anns = windmill_common::worker::TypeScriptAnnotations::parse(&content); + if anns.native && language == ScriptLang::Bun { language = ScriptLang::Bunnative; - } else if !anns.native_mode && language == ScriptLang::Bunnative { + } else if !anns.native && language == ScriptLang::Bunnative { language = ScriptLang::Bun; }; } @@ -1003,10 +1003,10 @@ async fn lock_modules<'c>( fn skip_creating_new_lock(language: &ScriptLang, content: &str) -> bool { if language == &ScriptLang::Bun || language == &ScriptLang::Bunnative { - let anns = get_annotation_ts(&content); - if anns.native_mode && language == &ScriptLang::Bun { + let anns = windmill_common::worker::TypeScriptAnnotations::parse(&content); + if anns.native && language == &ScriptLang::Bun { return false; - } else if !anns.native_mode && language == &ScriptLang::Bunnative { + } else if !anns.native && language == &ScriptLang::Bunnative { return false; }; } @@ -1077,11 +1077,13 @@ async fn lock_modules_app( match new_lock { Ok(new_lock) => { append_logs(&job.id, &job.workspace_id, logs, db).await; - let anns = get_annotation_ts(&content); - let nlang = if anns.native_mode && language == ScriptLang::Bun { + let anns = + windmill_common::worker::TypeScriptAnnotations::parse( + &content, + ); + let nlang = if anns.native && language == ScriptLang::Bun { Some(ScriptLang::Bunnative) - } else if !anns.native_mode && language == ScriptLang::Bunnative - { + } else if !anns.native && language == ScriptLang::Bunnative { Some(ScriptLang::Bun) } else { None @@ -1437,7 +1439,7 @@ async fn capture_dependency_job( } ScriptLang::Bun | ScriptLang::Bunnative => { let npm_mode = npm_mode.unwrap_or_else(|| { - windmill_common::worker::get_annotation_ts(job_raw_code).npm_mode + windmill_common::worker::TypeScriptAnnotations::parse(job_raw_code).npm }); if !raw_deps { let _ = write_file(job_dir, "main.ts", job_raw_code)?;