Skip to content

Commit

Permalink
Annotations refactor (#4512)
Browse files Browse the repository at this point in the history
* Refactor annotation system

* Move tests to different location

* Limit annotations per line (15)

* Write initial benchmark

* Optimize

* More optimizations

* Rewrite with 3x speed!

* Remove benchmarks

They were in wrong place and for development

* Polishing

* Remove unused leftover

* Integrate new annotation system

* Add comment explanation

* Remove `hashe-based matching`

Turned to be it is impure and non-deterministic method

* Add `tracing::error` for `// native` without `deno_core`

If you build backend without `deno_core` feature flag, and add //native
annotation to ts script, it would just silently exit.

This commit prints error to logs

* add more tests

* Update annotations.rs

---------

Co-authored-by: Ruben Fiszel <[email protected]>
  • Loading branch information
pyranota and rubenfiszel authored Oct 17, 2024
1 parent b7ad19b commit b237873
Show file tree
Hide file tree
Showing 17 changed files with 386 additions and 124 deletions.
15 changes: 14 additions & 1 deletion backend/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 5 additions & 1 deletion backend/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ members = [
"./windmill-audit",
"./windmill-git-sync",
"./windmill-indexer",
"./windmill-macros",
"./parsers/windmill-parser",
"./parsers/windmill-parser-ts",
"./parsers/windmill-parser-wasm",
Expand All @@ -23,7 +24,7 @@ members = [
"./parsers/windmill-parser-py",
"./parsers/windmill-parser-py-imports",
"./parsers/windmill-sql-datatype-parser-wasm",
"./parsers/windmill-parser-yaml",
"./parsers/windmill-parser-yaml", "windmill-macros",
]

[workspace.package]
Expand Down Expand Up @@ -116,6 +117,7 @@ windmill-common = { path = "./windmill-common", default-features = false }
windmill-audit = { path = "./windmill-audit" }
windmill-git-sync = { path = "./windmill-git-sync" }
windmill-indexer = {path = "./windmill-indexer"}
windmill-macros = {path = "./windmill-macros"}
windmill-parser = { path = "./parsers/windmill-parser" }
windmill-parser-ts = { path = "./parsers/windmill-parser-ts" }
windmill-parser-py = { path = "./parsers/windmill-parser-py" }
Expand Down Expand Up @@ -281,6 +283,8 @@ triomphe = "^0"

tantivy = "0.22.0"

# Macro-related
proc-macro2 = "1.0"
pulldown-cmark = "0.9"
toml = "0.7"
syn = { version = "2.0.74", features = ["full"] }
Expand Down
4 changes: 2 additions & 2 deletions backend/windmill-api/src/jobs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4193,10 +4193,10 @@ async fn run_dependencies_job(
JsonRawValue::from_string("true".to_string()).unwrap(),
);
if language == ScriptLang::Bun {
let annotation = windmill_common::worker::get_annotation_ts(&raw_code);
let annotation = windmill_common::worker::TypeScriptAnnotations::parse(&raw_code);
hm.insert(
"npm_mode".to_string(),
JsonRawValue::from_string(annotation.npm_mode.to_string()).unwrap(),
JsonRawValue::from_string(annotation.npm.to_string()).unwrap(),
);
}
(PushArgs { extra: Some(hm), args: &ehm }, deps)
Expand Down
6 changes: 3 additions & 3 deletions backend/windmill-api/src/scripts.rs
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ use windmill_common::{
utils::{
not_found_if_none, paginate, query_elems_from_hub, require_admin, Pagination, StripPath,
},
worker::{get_annotation_ts, to_raw_value},
worker::to_raw_value,
HUB_BASE_URL,
};
use windmill_git_sync::{handle_deployment_metadata, DeployedObject};
Expand Down Expand Up @@ -606,8 +606,8 @@ async fn create_script_internal<'c>(
};

let lang = if &ns.language == &ScriptLang::Bun || &ns.language == &ScriptLang::Bunnative {
let anns = get_annotation_ts(&ns.content);
if anns.native_mode {
let anns = windmill_common::worker::TypeScriptAnnotations::parse(&ns.content);
if anns.native {
ScriptLang::Bunnative
} else {
ScriptLang::Bun
Expand Down
1 change: 1 addition & 0 deletions backend/windmill-common/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ futures-core.workspace = true
async-stream.workspace = true
const_format.workspace = true
crc.workspace = true
windmill-macros.workspace = true

[target.'cfg(not(target_env = "msvc"))'.dependencies]
tikv-jemalloc-ctl = { optional = true, workspace = true }
58 changes: 10 additions & 48 deletions backend/windmill-common/src/worker.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ use std::{
sync::{atomic::AtomicBool, Arc},
};
use tokio::sync::RwLock;
use windmill_macros::annotations;

use crate::{error, global_settings::CUSTOM_TAGS_SETTING, server::Smtp, DB};

Expand Down Expand Up @@ -303,64 +304,25 @@ fn parse_file<T: FromStr>(path: &str) -> Option<T> {
.flatten()
}

pub struct TypeScriptAnnotations {
pub npm_mode: bool,
pub nodejs_mode: bool,
pub native_mode: bool,
pub nobundling: bool,
}

pub fn get_annotation_ts(inner_content: &str) -> TypeScriptAnnotations {
let annotations = inner_content
.lines()
.take_while(|x| x.starts_with("//"))
.map(|x| x.to_string().replace("//", "").trim().to_string())
.collect_vec();
let nodejs_mode: bool = annotations.contains(&"nodejs".to_string());
let npm_mode: bool = annotations.contains(&"npm".to_string());
let native_mode: bool = annotations.contains(&"native".to_string());

//TODO: remove || npm_mode when bun build is more powerful
let nobundling: bool =
annotations.contains(&"nobundling".to_string()) || nodejs_mode || *DISABLE_BUNDLING;

TypeScriptAnnotations { npm_mode, nodejs_mode, native_mode, nobundling }
}

#[annotations("#")]
pub struct PythonAnnotations {
pub no_uv: bool,
pub no_cache: bool,
pub no_uv: bool,
}

pub fn get_annotation_python(inner_content: &str) -> PythonAnnotations {
let annotations = inner_content
.lines()
.take_while(|x| x.starts_with("#"))
.map(|x| x.to_string().replace("#", "").trim().to_string())
.collect_vec();

let no_uv: bool = annotations.contains(&"no_uv".to_string());
let no_cache: bool = annotations.contains(&"no_cache".to_string());

PythonAnnotations { no_uv, no_cache }
#[annotations("//")]
pub struct TypeScriptAnnotations {
pub npm: bool,
pub nodejs: bool,
pub native: bool,
pub nobundling: bool,
}

#[annotations("--")]
pub struct SqlAnnotations {
pub return_last_result: bool,
}

pub fn get_sql_annotations(inner_content: &str) -> SqlAnnotations {
let annotations = inner_content
.lines()
.take_while(|x| x.starts_with("--"))
.map(|x| x.to_string().replace("--", "").trim().to_string())
.collect_vec();

let return_last_result: bool = annotations.contains(&"return_last_result".to_string());

SqlAnnotations { return_last_result }
}

pub async fn load_cache(bin_path: &str, _remote_path: &str) -> (bool, String) {
if tokio::fs::metadata(&bin_path).await.is_ok() {
(true, format!("loaded from local cache: {}\n", bin_path))
Expand Down
20 changes: 20 additions & 0 deletions backend/windmill-macros/Cargo.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
[package]
name = "windmill-macros"
version.workspace = true
authors.workspace = true
edition.workspace = true

[lib]
proc-macro = true

[dependencies]
proc-macro2.workspace = true
quote.workspace = true
syn.workspace = true

# Dependencies for tests
[dev-dependencies]
# tests/annotation.rs
lazy_static.workspace = true
itertools.workspace = true
regex.workspace = true
95 changes: 95 additions & 0 deletions backend/windmill-macros/src/lib.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
use proc_macro::TokenStream;
use quote::quote;
use syn::{parse_macro_input, Ident, ItemStruct, Lit};

#[proc_macro_attribute]
pub fn annotations(attr: TokenStream, item: TokenStream) -> TokenStream {
let input = parse_macro_input!(item as ItemStruct);
let name = input.ident.clone();
let fields = input
.fields
.iter()
.map(|f| f.ident.clone().unwrap())
.collect::<Vec<Ident>>();

// Match on the literal to extract the string value
let comm_lit = match parse_macro_input!(attr as Lit) {
Lit::Str(lit_str) => lit_str.value(), // This will give "#" without quotes
_ => panic!("Expected a string literal"),
};

// Generate regex
let mut reg = format!("^{}|", &comm_lit);
{
for field in fields.iter() {
reg.push_str(&(field.to_string()));
reg.push_str("\\b");
}

reg.push_str(r#"|\w+"#);
}
// Example of generated regex:
// ^#
// |ann1\b|ann2\b|ann3\b|ann4\b
// |\w+

TokenStream::from(quote! {
#[derive(Default, Debug)]
#input

impl std::ops::BitOrAssign for #name{
fn bitor_assign(&mut self, rhs: Self) {
// Unfold fields
// Read more: https://docs.rs/quote/latest/quote/macro.quote.html#interpolation
#( self.#fields |= rhs.#fields; )*
}
}

impl #name {
/// Autogenerated by windmill-macros
pub fn parse(inner_content: &str) -> Self{
let mut res = Self::default();
lazy_static::lazy_static! {
static ref RE: regex::Regex = regex::Regex::new(#reg).unwrap();
}
// Create lines stream
let mut lines = inner_content.lines();
'outer: while let Some(line) = lines.next() {
// If comment sign(s) on the right place
let mut comms = false;
// New instance
// We will apply it if in line only annotations
let mut new = Self::default();

'inner: for (i, mat) in RE.find_iter(line).enumerate() {

match mat.as_str(){
#comm_lit if i == 0 => {
comms = true;
continue 'inner;
},

// Will expand into something like:
// "ann1" => new.ann1 = true,
// "ann2" => new.ann2 = true,
// "ann3" => new.ann3 = true,
#( stringify!(#fields) => new.#fields = true, )*
// Non annotations
_ => continue 'outer,
};
}

if !comms {
// We dont want to continue if line does not start with #
return res;
}

// Apply changes
res |= new;
}

res
}
}
})
}
Loading

0 comments on commit b237873

Please sign in to comment.