Feat(kcl-vet): add ast builder for kcl-vet. (kcl-lang#224)

* add invalid test cases

* add test case for invalid json/yaml

* fix merge conflicts

* Feat(kcl-vet): add ast builder for kcl-vet.

add ast expr builder for kcl-vet from json/yaml Value.

issue kcl-lang#67

* add some comments

* add some more test cases

* add test cases for no schema name

* add test case for unsupported u64

* add test cases for yaml with tag

* rm useless test case

kclvm/tools/src/lib.rs

@@ -3,6 +3,7 @@ pub mod lint;
 pub mod printer;
 pub mod query;
 mod util;
+pub mod vet;
 
 #[macro_use]
 extern crate kclvm_error;

kclvm/tools/src/util/loader.rs

@@ -5,7 +5,11 @@ use anyhow::{bail, Context, Result};
 pub(crate) trait Loader<T> {
     fn load(&self) -> Result<T>;
 }
-pub(crate) enum LoaderKind {
+
+/// Types of verifiable files currently supported by KCL-Vet,
+/// currently only YAML files and Json files are supported.
+#[derive(Clone, Copy)]
+pub enum LoaderKind {
     YAML,
     JSON,
 }
@@ -41,6 +45,10 @@ impl DataLoader {
     pub(crate) fn get_data(&self) -> &str {
         &self.content
     }
+
+    pub(crate) fn get_kind(&self) -> &LoaderKind {
+        &self.kind
+    }
 }
 
 impl Loader<serde_json::Value> for DataLoader {

kclvm/tools/src/util/test_datas/test_invalid.yaml

@@ -1,5 +1,2 @@
-{
-    "name": "John Doe",
-        "city": "London"
+"name": "John Doe",
 invalid
-

kclvm/tools/src/util/tests.rs

@@ -221,7 +221,7 @@ websites:
                     panic!("unreachable")
                 }
                 Err(err) => {
-                    assert_eq!(format!("{:?}", err), "Failed to String '{\n    \"name\": \"John Doe\",\n        \"city\": \"London\"\ninvalid\n\n' to Yaml\n\nCaused by:\n    did not find expected ',' or '}' at line 4 column 1, while parsing a flow mapping");
+                    assert_eq!(format!("{:?}", err), "Failed to String '\"name\": \"John Doe\",\ninvalid\n' to Yaml\n\nCaused by:\n    did not find expected key at line 1 column 19, while parsing a block mapping");
                 }
             }
         }

kclvm/tools/src/vet/expr_builder.rs

@@ -0,0 +1,308 @@
+use kclvm_ast::{
+    ast::{
+        ConfigEntry, ConfigEntryOperation, ConfigExpr, Expr, ExprContext, Identifier, ListExpr,
+        NameConstant, NameConstantLit, Node, NodeRef, NumberLit, NumberLitValue, SchemaExpr,
+        StringLit,
+    },
+    node_ref,
+};
+
+use crate::util::loader::{DataLoader, Loader, LoaderKind};
+use anyhow::{bail, Context, Result};
+
+trait ExprGenerator<T> {
+    fn generate(&self, value: &T) -> Result<NodeRef<Expr>>;
+}
+
+/// `ExprBuilder` will generate ast expr from Json/Yaml.
+/// `Object` in Json and `Mapping` in Yaml is mapped to `Schema Expr`.
+/// You should set `schema_name` for `Schema Expr` before using `ExprBuilder`.
+pub(crate) struct ExprBuilder {
+    schema_name: Option<String>,
+    loader: DataLoader,
+}
+
+impl ExprBuilder {
+    pub(crate) fn new_with_file_path(
+        schema_name: Option<String>,
+        kind: LoaderKind,
+        file_path: String,
+    ) -> Result<Self> {
+        let loader = DataLoader::new_with_file_path(kind, &file_path)
+            .with_context(|| format!("Failed to Load '{}'", file_path))?;
+
+        Ok(Self {
+            schema_name,
+            loader,
+        })
+    }
+
+    pub(crate) fn new_with_str(
+        schema_name: Option<String>,
+        kind: LoaderKind,
+        content: String,
+    ) -> Result<Self> {
+        let loader = DataLoader::new_with_str(kind, &content)
+            .with_context(|| format!("Failed to Parse String '{}'", content))?;
+
+        Ok(Self {
+            schema_name,
+            loader,
+        })
+    }
+
+    /// Generate ast expr from Json/Yaml depends on `LoaderKind`.
+    pub(crate) fn build(&self) -> Result<NodeRef<Expr>> {
+        match self.loader.get_kind() {
+            LoaderKind::JSON => {
+                let value = <DataLoader as Loader<serde_json::Value>>::load(&self.loader)
+                    .with_context(|| format!("Failed to Load JSON"))?;
+                Ok(self
+                    .generate(&value)
+                    .with_context(|| format!("Failed to Load JSON"))?)
+            }
+            LoaderKind::YAML => {
+                let value = <DataLoader as Loader<serde_yaml::Value>>::load(&self.loader)
+                    .with_context(|| format!("Failed to Load YAML"))?;
+                Ok(self
+                    .generate(&value)
+                    .with_context(|| format!("Failed to Load YAML"))?)
+            }
+        }
+    }
+}
+
+impl ExprGenerator<serde_yaml::Value> for ExprBuilder {
+    fn generate(&self, value: &serde_yaml::Value) -> Result<NodeRef<Expr>> {
+        match value {
+            serde_yaml::Value::Null => Ok(node_ref!(Expr::NameConstantLit(NameConstantLit {
+                value: NameConstant::None,
+            }))),
+            serde_yaml::Value::Bool(j_bool) => {
+                let name_const = match NameConstant::try_from(*j_bool) {
+                    Ok(nc) => nc,
+                    Err(_) => {
+                        bail!("Failed to Load Validated File")
+                    }
+                };
+
+                Ok(node_ref!(Expr::NameConstantLit(NameConstantLit {
+                    value: name_const
+                })))
+            }
+            serde_yaml::Value::Number(j_num) => {
+                if j_num.is_f64() {
+                    let number_lit = match j_num.as_f64() {
+                        Some(num_f64) => num_f64,
+                        None => {
+                            bail!("Failed to Load Validated File")
+                        }
+                    };
+
+                    Ok(node_ref!(Expr::NumberLit(NumberLit {
+                        binary_suffix: None,
+                        value: NumberLitValue::Float(number_lit)
+                    })))
+                } else if j_num.is_i64() {
+                    let number_lit = match j_num.as_i64() {
+                        Some(j_num) => j_num,
+                        None => {
+                            bail!("Failed to Load Validated File")
+                        }
+                    };
+
+                    Ok(node_ref!(Expr::NumberLit(NumberLit {
+                        binary_suffix: None,
+                        value: NumberLitValue::Int(number_lit)
+                    })))
+                } else {
+                    bail!("Failed to Load Validated File, Unsupported Unsigned 64");
+                }
+            }
+            serde_yaml::Value::String(j_string) => {
+                let str_lit = match StringLit::try_from(j_string.to_string()) {
+                    Ok(s) => s,
+                    Err(_) => {
+                        bail!("Failed to Load Validated File")
+                    }
+                };
+                Ok(node_ref!(Expr::StringLit(str_lit)))
+            }
+            serde_yaml::Value::Sequence(j_arr) => {
+                let mut j_arr_ast_nodes: Vec<NodeRef<Expr>> = Vec::new();
+                for j_arr_item in j_arr {
+                    j_arr_ast_nodes.push(
+                        self.generate(j_arr_item)
+                            .with_context(|| format!("Failed to Load Validated File"))?,
+                    );
+                }
+                Ok(node_ref!(Expr::List(ListExpr {
+                    ctx: ExprContext::Load,
+                    elts: j_arr_ast_nodes
+                })))
+            }
+            serde_yaml::Value::Mapping(j_map) => {
+                let mut config_entries: Vec<NodeRef<ConfigEntry>> = Vec::new();
+
+                for (k, v) in j_map.iter() {
+                    let k = self
+                        .generate(k)
+                        .with_context(|| format!("Failed to Load Validated File"))?;
+                    let v = self
+                        .generate(v)
+                        .with_context(|| format!("Failed to Load Validated File"))?;
+
+                    let config_entry = node_ref!(ConfigEntry {
+                        key: Some(k),
+                        value: v,
+                        operation: ConfigEntryOperation::Union,
+                        insert_index: -1
+                    });
+                    config_entries.push(config_entry);
+                }
+
+                let config_expr = node_ref!(Expr::Config(ConfigExpr {
+                    items: config_entries
+                }));
+
+                match &self.schema_name {
+                    Some(s_name) => {
+                        let iden = node_ref!(Identifier {
+                            names: vec![s_name.to_string()],
+                            pkgpath: String::new(),
+                            ctx: ExprContext::Load
+                        });
+                        Ok(node_ref!(Expr::Schema(SchemaExpr {
+                            name: iden,
+                            config: config_expr,
+                            args: vec![],
+                            kwargs: vec![]
+                        })))
+                    }
+                    None => Ok(config_expr),
+                }
+            }
+            serde_yaml::Value::Tagged(_) => {
+                bail!("Failed to Load Validated File, Unsupported Yaml Tagged.")
+            }
+        }
+    }
+}
+
+impl ExprGenerator<serde_json::Value> for ExprBuilder {
+    fn generate(&self, value: &serde_json::Value) -> Result<NodeRef<Expr>> {
+        match value {
+            serde_json::Value::Null => Ok(node_ref!(Expr::NameConstantLit(NameConstantLit {
+                value: NameConstant::None,
+            }))),
+            serde_json::Value::Bool(j_bool) => {
+                let name_const = match NameConstant::try_from(*j_bool) {
+                    Ok(nc) => nc,
+                    Err(_) => {
+                        bail!("Failed to Load Validated File")
+                    }
+                };
+
+                Ok(node_ref!(Expr::NameConstantLit(NameConstantLit {
+                    value: name_const
+                })))
+            }
+            serde_json::Value::Number(j_num) => {
+                if j_num.is_f64() {
+                    let number_lit = match j_num.as_f64() {
+                        Some(num_f64) => num_f64,
+                        None => {
+                            bail!("Failed to Load Validated File")
+                        }
+                    };
+
+                    Ok(node_ref!(Expr::NumberLit(NumberLit {
+                        binary_suffix: None,
+                        value: NumberLitValue::Float(number_lit)
+                    })))
+                } else if j_num.is_i64() {
+                    let number_lit = match j_num.as_i64() {
+                        Some(j_num) => j_num,
+                        None => {
+                            bail!("Failed to Load Validated File")
+                        }
+                    };
+
+                    Ok(node_ref!(Expr::NumberLit(NumberLit {
+                        binary_suffix: None,
+                        value: NumberLitValue::Int(number_lit)
+                    })))
+                } else {
+                    bail!("Failed to Load Validated File, Unsupported Unsigned 64");
+                }
+            }
+            serde_json::Value::String(j_string) => {
+                let str_lit = match StringLit::try_from(j_string.to_string()) {
+                    Ok(s) => s,
+                    Err(_) => {
+                        bail!("Failed to Load Validated File")
+                    }
+                };
+
+                Ok(node_ref!(Expr::StringLit(str_lit)))
+            }
+            serde_json::Value::Array(j_arr) => {
+                let mut j_arr_ast_nodes: Vec<NodeRef<Expr>> = Vec::new();
+                for j_arr_item in j_arr {
+                    j_arr_ast_nodes.push(
+                        self.generate(j_arr_item)
+                            .with_context(|| format!("Failed to Load Validated File"))?,
+                    );
+                }
+                Ok(node_ref!(Expr::List(ListExpr {
+                    ctx: ExprContext::Load,
+                    elts: j_arr_ast_nodes
+                })))
+            }
+            serde_json::Value::Object(j_map) => {
+                let mut config_entries: Vec<NodeRef<ConfigEntry>> = Vec::new();
+
+                for (k, v) in j_map.iter() {
+                    let k = match StringLit::try_from(k.to_string()) {
+                        Ok(s) => s,
+                        Err(_) => {
+                            bail!("Failed to Load Validated File")
+                        }
+                    };
+                    let v = self
+                        .generate(v)
+                        .with_context(|| format!("Failed to Load Validated File"))?;
+
+                    let config_entry = node_ref!(ConfigEntry {
+                        key: Some(node_ref!(Expr::StringLit(k))),
+                        value: v,
+                        operation: ConfigEntryOperation::Union,
+                        insert_index: -1
+                    });
+                    config_entries.push(config_entry);
+                }
+
+                let config_expr = node_ref!(Expr::Config(ConfigExpr {
+                    items: config_entries
+                }));
+
+                match &self.schema_name {
+                    Some(s_name) => {
+                        let iden = node_ref!(Identifier {
+                            names: vec![s_name.to_string()],
+                            pkgpath: String::new(),
+                            ctx: ExprContext::Load
+                        });
+                        Ok(node_ref!(Expr::Schema(SchemaExpr {
+                            name: iden,
+                            config: config_expr,
+                            args: vec![],
+                            kwargs: vec![]
+                        })))
+                    }
+                    None => Ok(config_expr),
+                }
+            }
+        }
+    }
+}

kclvm/tools/src/vet/mod.rs

@@ -0,0 +1,4 @@
+pub mod expr_builder;
+
+#[cfg(test)]
+mod tests;

kclvm/tools/src/vet/test_datas/invalid/test_invalid.json

@@ -0,0 +1,2 @@
+languages:
+  - Ruby

kclvm/tools/src/vet/test_datas/invalid/test_invalid.yaml

@@ -0,0 +1,5 @@
+{
+    "name": "John Doe",
+        "city": "London"
+invalid
+

kclvm/tools/src/vet/test_datas/invalid/unsupported/json_with_u64.ast.json

@@ -0,0 +1,3 @@
+{
+    "u64_value": 9223372036854775808
+}

kclvm/tools/src/vet/test_datas/invalid/unsupported/json_with_u64.json

@@ -0,0 +1,3 @@
+{
+    "u64_value": 9223372036854775808
+}

kclvm/tools/src/vet/test_datas/invalid/unsupported/yaml_with_tag.yaml

@@ -0,0 +1,4 @@
+!mytag
+a: 1
+b: 2
+c: 2022-05-01

kclvm/tools/src/vet/test_datas/invalid/unsupported/yaml_with_u64.ast.yaml

@@ -0,0 +1,3 @@
+{
+    "u64_value": 9223372036854775808
+}

kclvm/tools/src/vet/test_datas/invalid/unsupported/yaml_with_u64.yaml

@@ -0,0 +1,3 @@
+{
+    "u64_value": 9223372036854775808
+}