From 35c31c792a7f4ea720484af07e98d80374cbf328 Mon Sep 17 00:00:00 2001 From: Kevin Date: Mon, 21 Mar 2022 18:55:53 -0500 Subject: [PATCH] Initial commit --- .gitignore | 1 + tesql-cli/.gitignore | 3 + tesql-cli/Cargo.lock | 247 ++++++++++++++++++++++++++++++++ tesql-cli/Cargo.toml | 10 ++ tesql-cli/src/error.rs | 3 + tesql-cli/src/main.rs | 55 +++++++ tesql-cli/testdata/data.json | 51 +++++++ tesql/.gitignore | 2 + tesql/.vscode/launch.json | 64 +++++++++ tesql/Cargo.lock | 132 +++++++++++++++++ tesql/Cargo.toml | 11 ++ tesql/src/bin/main.rs | 19 +++ tesql/src/data/mod.rs | 2 + tesql/src/data/parser.rs | 77 ++++++++++ tesql/src/data/types.rs | 47 ++++++ tesql/src/json.rs | 10 ++ tesql/src/lib.rs | 158 ++++++++++++++++++++ tesql/src/mapper.rs | 68 +++++++++ tesql/src/sqlizer.rs | 154 ++++++++++++++++++++ tesql/testdata/data-test.json | 51 +++++++ tesql/tests/gen-inserts-test.rs | 22 +++ 21 files changed, 1187 insertions(+) create mode 100644 .gitignore create mode 100644 tesql-cli/.gitignore create mode 100644 tesql-cli/Cargo.lock create mode 100644 tesql-cli/Cargo.toml create mode 100644 tesql-cli/src/error.rs create mode 100644 tesql-cli/src/main.rs create mode 100644 tesql-cli/testdata/data.json create mode 100644 tesql/.gitignore create mode 100644 tesql/.vscode/launch.json create mode 100644 tesql/Cargo.lock create mode 100644 tesql/Cargo.toml create mode 100644 tesql/src/bin/main.rs create mode 100644 tesql/src/data/mod.rs create mode 100644 tesql/src/data/parser.rs create mode 100644 tesql/src/data/types.rs create mode 100644 tesql/src/json.rs create mode 100644 tesql/src/lib.rs create mode 100644 tesql/src/mapper.rs create mode 100644 tesql/src/sqlizer.rs create mode 100644 tesql/testdata/data-test.json create mode 100644 tesql/tests/gen-inserts-test.rs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f2a4093 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +**/target \ No newline at end of file diff --git a/tesql-cli/.gitignore b/tesql-cli/.gitignore new file mode 100644 index 0000000..2496ead --- /dev/null +++ b/tesql-cli/.gitignore @@ -0,0 +1,3 @@ +tesql_out +tesql_sql +tesql_json \ No newline at end of file diff --git a/tesql-cli/Cargo.lock b/tesql-cli/Cargo.lock new file mode 100644 index 0000000..77cd28b --- /dev/null +++ b/tesql-cli/Cargo.lock @@ -0,0 +1,247 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "clap" +version = "2.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0610544180c38b88101fecf2dd634b174a62eef6946f84dfc6a7127512b381c" +dependencies = [ + "bitflags", + "textwrap", + "unicode-width", +] + +[[package]] +name = "hashbrown" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" + +[[package]] +name = "heck" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "indexmap" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282a6247722caba404c065016bbfa522806e51714c34f5dfc3e4a3a46fcb4223" +dependencies = [ + "autocfg", + "hashbrown", +] + +[[package]] +name = "itoa" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "linked-hash-map" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" +dependencies = [ + "serde", + "serde_test", +] + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + +[[package]] +name = "proc-macro2" +version = "1.0.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7342d5883fbccae1cc37a2353b09c87c9b0f3afd73f5fb9bba687a1f733b029" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "quote" +version = "1.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4af2ec4714533fcdf07e886f17025ace8b997b9ce51204ee69b6da831c3da57" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "ryu" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f" + +[[package]] +name = "serde" +version = "1.0.136" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce31e24b01e1e524df96f1c2fdd054405f8d7376249a5110886fb4b658484789" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.136" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08597e7152fcd306f41838ed3e37be9eaeed2b61c42e2117266a554fab4662f9" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e8d9fa5c3b304765ce1fd9c4c8a3de2c8db365a5b91be52f186efc675681d95" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_test" +version = "1.0.136" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21675ba6f9d97711cc00eee79d8dd7d0a31e571c350fb4d8a7c78f70c0e7b0e9" +dependencies = [ + "serde", +] + +[[package]] +name = "structopt" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c6b5c64445ba8094a6ab0c3cd2ad323e07171012d9c98b0b15651daf1787a10" +dependencies = [ + "clap", + "lazy_static", + "structopt-derive", +] + +[[package]] +name = "structopt-derive" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0" +dependencies = [ + "heck", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "syn" +version = "1.0.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea297be220d52398dcc07ce15a209fce436d361735ac1db700cab3b6cdfb9f54" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "tesql" +version = "0.1.0" +dependencies = [ + "linked-hash-map", + "serde", + "serde_json", +] + +[[package]] +name = "tesql-cli" +version = "0.1.0" +dependencies = [ + "structopt", + "tesql", +] + +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "unicode-segmentation" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e8820f5d777f6224dc4be3632222971ac30164d4a258d595640799554ebfd99" + +[[package]] +name = "unicode-width" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973" + +[[package]] +name = "unicode-xid" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" diff --git a/tesql-cli/Cargo.toml b/tesql-cli/Cargo.toml new file mode 100644 index 0000000..222491e --- /dev/null +++ b/tesql-cli/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "tesql-cli" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +tesql = { path = "../tesql", version = "0.1.0" } +structopt = { version = "0.3", default-features = false } diff --git a/tesql-cli/src/error.rs b/tesql-cli/src/error.rs new file mode 100644 index 0000000..f6546f7 --- /dev/null +++ b/tesql-cli/src/error.rs @@ -0,0 +1,3 @@ +pub fn print_error(msg: impl Into) { + eprintln!("[tesql error]: {}", msg.into()); +} diff --git a/tesql-cli/src/main.rs b/tesql-cli/src/main.rs new file mode 100644 index 0000000..e6418de --- /dev/null +++ b/tesql-cli/src/main.rs @@ -0,0 +1,55 @@ +use structopt::StructOpt; +use tesql; + +mod error; +use crate::error::print_error; + +#[derive(StructOpt, Debug)] +struct Cli { + #[structopt(name = "in_path", long = "in", short = "i", default_value = ".")] + in_path: String, + #[structopt( + name = "json_out_path", + long = "json-out", + default_value = "./tesql_out/out.json" + )] + json_out_path: String, + #[structopt( + name = "sql_out_path", + long = "sql-out", + default_value = "./tesql_out/out.sql" + )] + sql_out_path: String, + #[structopt( + name = "json_out_dir", + long = "json-dir", + default_value = "./tesql_json" + )] + json_dir_path: String, + #[structopt(name = "sql_out_dir", long = "sql-dir", default_value = "./tesql_sql")] + sql_dir_path: String, + #[structopt(name = "split_files", long = "split", short = "sp")] + split_files: bool, +} + +#[cfg(windows)] +fn main() { + let args = Cli::from_args(); + let gen = tesql::gen_inserts_from_file( + args.in_path.as_str(), + tesql::GenInsertOptions { + split_data_files: args.split_files, + json_out_dir_path: args.json_dir_path.as_str(), + json_out_file_path: args.json_out_path.as_str(), + sql_out_dir_path: args.sql_dir_path.as_str(), + sql_out_file_path: args.sql_out_path.as_str(), + }, + ); + + match gen { + Ok(_) => { + println!("[tesql]: successfully generated sql and json files"); + } + Err(err) => print_error(format!("{:?}", err)), + } +} diff --git a/tesql-cli/testdata/data.json b/tesql-cli/testdata/data.json new file mode 100644 index 0000000..f6a309f --- /dev/null +++ b/tesql-cli/testdata/data.json @@ -0,0 +1,51 @@ +{ + "data": { + "user": { + "columns": [ + { "name": "id", "type": "uuid" }, + { "name": "username", "type": "text" }, + { "name": "is_student", "type": "text" }, + { "name": "friends_ids", "type": "array" }, + { "name": "nested_arrays", "type": "array" } + ], + "inserts": [ + { + "id": "0101111111111", + "username": "Bob", + "is_student": true, + "friends_ids": ["hello", "world", "hi!"], + "nested_arrays": [ + ["hey", "man"], + ["the", "world", "goes", "round"] + ] + }, + { + "id": "2", + "username": "Bob2", + "is_student": false, + "friends_ids": ["hi", "mom"], + "nested_arrays": [["hello", "there"], ["hello", "world"], ["hi!"]] + } + ] + }, + "user1": { + "columns": [ + { "name": "id", "type": "uuid" }, + { "name": "username", "type": "text" }, + { "name": "bool", "type": "text" } + ], + "inserts": [ + { + "id": "0101", + "username": "Bob", + "bool": true + }, + { + "id": "2", + "username": "Bob2", + "bool": false + } + ] + } + } +} diff --git a/tesql/.gitignore b/tesql/.gitignore new file mode 100644 index 0000000..7e0aa5c --- /dev/null +++ b/tesql/.gitignore @@ -0,0 +1,2 @@ +/testgen* +/out* \ No newline at end of file diff --git a/tesql/.vscode/launch.json b/tesql/.vscode/launch.json new file mode 100644 index 0000000..7a6b932 --- /dev/null +++ b/tesql/.vscode/launch.json @@ -0,0 +1,64 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "type": "lldb", + "request": "launch", + "name": "Debug unit tests in library 'tesql'", + "cargo": { + "args": [ + "test", + "--no-run", + "--lib", + "--package=tesql" + ], + "filter": { + "name": "tesql", + "kind": "lib" + } + }, + "args": [], + "cwd": "${workspaceFolder}" + }, + { + "type": "lldb", + "request": "launch", + "name": "Debug executable 'main'", + "cargo": { + "args": [ + "build", + "--bin=main", + "--package=tesql" + ], + "filter": { + "name": "main", + "kind": "bin" + } + }, + "args": [], + "cwd": "${workspaceFolder}" + }, + { + "type": "lldb", + "request": "launch", + "name": "Debug unit tests in executable 'main'", + "cargo": { + "args": [ + "test", + "--no-run", + "--bin=main", + "--package=tesql" + ], + "filter": { + "name": "main", + "kind": "bin" + } + }, + "args": [], + "cwd": "${workspaceFolder}" + } + ] +} \ No newline at end of file diff --git a/tesql/Cargo.lock b/tesql/Cargo.lock new file mode 100644 index 0000000..e0fbcc2 --- /dev/null +++ b/tesql/Cargo.lock @@ -0,0 +1,132 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "hashbrown" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" + +[[package]] +name = "indexmap" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282a6247722caba404c065016bbfa522806e51714c34f5dfc3e4a3a46fcb4223" +dependencies = [ + "autocfg", + "hashbrown", +] + +[[package]] +name = "itoa" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" + +[[package]] +name = "linked-hash-map" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fb9b38af92608140b86b693604b9ffcc5824240a484d1ecd4795bacb2fe88f3" +dependencies = [ + "serde", + "serde_test", +] + +[[package]] +name = "proc-macro2" +version = "1.0.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7342d5883fbccae1cc37a2353b09c87c9b0f3afd73f5fb9bba687a1f733b029" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "quote" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "864d3e96a899863136fc6e99f3d7cae289dafe43bf2c5ac19b70df7210c0a145" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "ryu" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f" + +[[package]] +name = "serde" +version = "1.0.136" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce31e24b01e1e524df96f1c2fdd054405f8d7376249a5110886fb4b658484789" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.136" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08597e7152fcd306f41838ed3e37be9eaeed2b61c42e2117266a554fab4662f9" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e8d9fa5c3b304765ce1fd9c4c8a3de2c8db365a5b91be52f186efc675681d95" +dependencies = [ + "indexmap", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_test" +version = "1.0.136" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21675ba6f9d97711cc00eee79d8dd7d0a31e571c350fb4d8a7c78f70c0e7b0e9" +dependencies = [ + "serde", +] + +[[package]] +name = "syn" +version = "1.0.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a65b3f4ffa0092e9887669db0eae07941f023991ab58ea44da8fe8e2d511c6b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "tesql" +version = "0.1.0" +dependencies = [ + "linked-hash-map", + "serde", + "serde_json", +] + +[[package]] +name = "unicode-xid" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" diff --git a/tesql/Cargo.toml b/tesql/Cargo.toml new file mode 100644 index 0000000..0c17b45 --- /dev/null +++ b/tesql/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "tesql" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +serde_json = { version = "1.0", features = ["preserve_order"] } +serde = { version = "1.0", features = ["derive"] } +linked-hash-map = { version = "0.5.4", features = ["serde_impl"] } diff --git a/tesql/src/bin/main.rs b/tesql/src/bin/main.rs new file mode 100644 index 0000000..5739697 --- /dev/null +++ b/tesql/src/bin/main.rs @@ -0,0 +1,19 @@ +use tesql::{gen_inserts_from_file, GenInsertOptions}; + +fn main() { + match gen_inserts_from_file( + "./testdata/data-test.json", + GenInsertOptions { + split_data_files: false, + sql_out_dir_path: "./out", + sql_out_file_path: "./out/out.sql", + json_out_dir_path: "./out", + json_out_file_path: "./out/out.json", + }, + ) { + Ok(_) => { + println!("Succesfully generated sql"); + } + Err(e) => println!("{:?}", e), + } +} diff --git a/tesql/src/data/mod.rs b/tesql/src/data/mod.rs new file mode 100644 index 0000000..07295a2 --- /dev/null +++ b/tesql/src/data/mod.rs @@ -0,0 +1,2 @@ +pub mod parser; +pub mod types; diff --git a/tesql/src/data/parser.rs b/tesql/src/data/parser.rs new file mode 100644 index 0000000..b1be40d --- /dev/null +++ b/tesql/src/data/parser.rs @@ -0,0 +1,77 @@ +use crate::data::types::DataSqlType; +use linked_hash_map::LinkedHashMap; +use serde::Deserialize; +use serde_json::Value; +use std::{fs::File, io::Read}; + +#[derive(Deserialize, Debug, Clone)] +pub struct DataColumnModel { + pub name: String, + pub r#type: DataSqlType, +} + +#[derive(Deserialize, Debug, Clone)] +pub struct DataInsertModel { + pub columns: Vec, + pub inserts: Vec>, +} + +#[derive(Deserialize, Debug)] +pub struct RawDataModel { + pub data: LinkedHashMap, +} + +#[derive(Debug)] +pub enum ParserError { + OpenFileError, + ReadFileError, + ParseJsonError, +} + +pub fn from_file(path: &str) -> Result { + match File::open(path) { + Ok(mut file) => { + let mut data = String::new(); + + if let Err(_) = file.read_to_string(&mut data) { + return Err(ParserError::ReadFileError); + } + + match serde_json::from_str::(&mut data) { + Ok(json) => Ok(json), + Err(a) => { + println!("{:?}", a); + return Err(ParserError::ParseJsonError); + } + } + } + Err(_) => return Err(ParserError::OpenFileError), + } +} + +#[cfg(test)] +mod data_tests { + use core::panic; + + use crate::data::{self, types}; + + #[test] + fn test_from_file() { + match data::parser::from_file("./testdata/data-test.json") { + Ok(json) => match json.data.get("user") { + Some(user) => { + assert_eq!("id", user.columns[0].name); + assert_eq!(types::DataSqlType::UUID, user.columns[0].r#type); + assert_eq!("username", user.columns[1].name); + assert_eq!("is_student", user.columns[2].name); + } + None => { + panic!("Invalid key"); + } + }, + Err(e) => { + panic!("{:?}", e); + } + } + } +} diff --git a/tesql/src/data/types.rs b/tesql/src/data/types.rs new file mode 100644 index 0000000..6faad59 --- /dev/null +++ b/tesql/src/data/types.rs @@ -0,0 +1,47 @@ +use serde::{de::IntoDeserializer, Deserialize, Deserializer}; + +#[derive(Deserialize, Debug)] +#[serde(remote = "DataSqlKey")] +pub enum DataSqlKey { + PKey, + FKey, +} + +impl<'de> Deserialize<'de> for DataSqlKey { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let s = String::deserialize(deserializer)?; + match s.as_str() { + "pKey" => Ok(DataSqlKey::PKey), + "fKey" => Ok(DataSqlKey::FKey), + _ => DataSqlKey::deserialize(s.into_deserializer()), + } + } +} + +#[derive(Deserialize, Debug, Clone, PartialEq)] +#[serde(remote = "DataSqlType")] +pub enum DataSqlType { + UUID, + Text, + Array, +} + +impl<'de> Deserialize<'de> for DataSqlType { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let s = String::deserialize(deserializer)?; + match s.as_str() { + "uuid" => { + return Ok(DataSqlType::UUID); + } + "text" => Ok(DataSqlType::Text), + "array" => Ok(DataSqlType::Text), + _ => DataSqlType::deserialize(s.into_deserializer()), + } + } +} diff --git a/tesql/src/json.rs b/tesql/src/json.rs new file mode 100644 index 0000000..e731eab --- /dev/null +++ b/tesql/src/json.rs @@ -0,0 +1,10 @@ +use linked_hash_map::LinkedHashMap; +use serde::Serialize; +use serde_json::Value; + +#[derive(Serialize)] +pub struct JsonRawModel { + pub data: LinkedHashMap, +} + +pub type JsonDataArray = Vec>; diff --git a/tesql/src/lib.rs b/tesql/src/lib.rs new file mode 100644 index 0000000..b2ebabc --- /dev/null +++ b/tesql/src/lib.rs @@ -0,0 +1,158 @@ +use std::{ + collections::HashMap, + fs::{self, File}, + io::Write, + path::Path, +}; + +use data::parser::ParserError; +use json::{JsonDataArray, JsonRawModel}; +use sqlizer::StringifyInsertError; + +mod data; +mod json; +mod mapper; +mod sqlizer; + +pub struct GenInsertOptions<'a> { + pub split_data_files: bool, + pub sql_out_dir_path: &'a str, + pub sql_out_file_path: &'a str, + pub json_out_dir_path: &'a str, + pub json_out_file_path: &'a str, +} + +impl<'a> Default for GenInsertOptions<'a> { + fn default() -> GenInsertOptions<'a> { + GenInsertOptions { + split_data_files: false, + sql_out_dir_path: "./out", + json_out_dir_path: "./out/json", + sql_out_file_path: "./out.sql", + json_out_file_path: "./out/out.json", + } + } +} + +#[derive(Debug)] +pub enum GenInsertsFromFileError<'a> { + SqlOutFileWriteError(ForceWriteAllError), + StringifyError(StringifyInsertError), + ParserError(ParserError), + OutDirRequired(&'a str), + JsonOutFileWriteError(ForceWriteAllError), + JsonSerializeError, +} + +pub fn gen_inserts_from_file<'a>( + path: &str, + opts: GenInsertOptions, +) -> Result<(), GenInsertsFromFileError<'a>> { + match data::parser::from_file(path) { + Ok(model) => { + let sql_models = mapper::map_data_to_inserts(&model); + let json_models = mapper::map_data_to_json(&model); + + if opts.split_data_files { + // map to sql files + for sm in sql_models.iter() { + match sqlizer::sqlize_insert(sm) { + Ok(sql) => { + let insert_str = format!("{}\n\n", sql.to_owned()); + let file_path = + format!("{}/{}.sql", opts.sql_out_dir_path, sm.table_name); + if let Err(e) = force_write_all(&file_path, insert_str.as_bytes()) { + return Err(GenInsertsFromFileError::SqlOutFileWriteError(e)); + } + } + Err(e) => { + return Err(GenInsertsFromFileError::StringifyError(e)); + } + } + } + + // map to json files + for (name, insert_model) in model.data.iter() { + let json = mapper::map_insert_to_json(name, insert_model); + + let json = match serde_json::to_string::>(&json) + { + Ok(j) => j, + Err(_) => { + return Err(GenInsertsFromFileError::JsonSerializeError); + } + }; + + let file_path = format!("{}/{}.json", opts.json_out_dir_path, name); + if let Err(e) = force_write_all(&file_path, json.as_bytes()) { + return Err(GenInsertsFromFileError::JsonOutFileWriteError(e)); + } + } + + return Ok(()); + } + + // one file + let sql_str = match sqlizer::compose_inserts(sql_models) { + Ok(sql) => sql, + Err(e) => return Err(GenInsertsFromFileError::StringifyError(e)), + }; + + if let Err(e) = force_write_all(opts.sql_out_file_path, sql_str.as_bytes()) { + return Err(GenInsertsFromFileError::SqlOutFileWriteError(e)); + } + + let json = match serde_json::to_string::(&json_models) { + Ok(j) => j, + Err(_) => { + return Err(GenInsertsFromFileError::JsonSerializeError); + } + }; + + if let Err(e) = force_write_all(opts.json_out_file_path, json.as_bytes()) { + return Err(GenInsertsFromFileError::SqlOutFileWriteError(e)); + } + + return Ok(()); + } + Err(e) => { + return Err(GenInsertsFromFileError::ParserError(e)); + } + } +} + +#[derive(Debug)] +pub enum ForceWriteAllError { + InvalidPath, + WriteError, + CreateFileError, + CreateDirectoryError, +} + +fn force_write_all(path: &str, data: &[u8]) -> Result<(), ForceWriteAllError> { + let full_path = Path::new(path); + + let dir_path = full_path.parent(); + let dir_path = match dir_path { + Some(p) => p, + None => return Err(ForceWriteAllError::InvalidPath), + }; + + let create_dir = fs::create_dir_all(dir_path); + if let Err(_) = create_dir { + return Err(ForceWriteAllError::CreateDirectoryError); + } + + let file = File::create(full_path); + let mut file = match file { + Ok(f) => f, + Err(_) => return Err(ForceWriteAllError::CreateFileError), + }; + + let write_res = file.write_all(data); + if let Err(_) = write_res { + return Err(ForceWriteAllError::WriteError); + } + + Ok(()) +} diff --git a/tesql/src/mapper.rs b/tesql/src/mapper.rs new file mode 100644 index 0000000..e912bc3 --- /dev/null +++ b/tesql/src/mapper.rs @@ -0,0 +1,68 @@ +use std::collections::HashMap; + +use linked_hash_map::LinkedHashMap; +use serde_json::Value; + +use crate::{ + data::{ + parser::{DataInsertModel, RawDataModel}, + types::DataSqlType, + }, + json::{JsonDataArray, JsonRawModel}, + sqlizer::{SqlColumn, SqlInsertModel, SqlType}, +}; + +pub fn map_data_to_inserts(model: &RawDataModel) -> Vec { + let mut sql_models: Vec = vec![]; + + for (model_name, data_insert_model) in model.data.iter() { + sql_models.push(SqlInsertModel { + table_name: model_name.clone(), + columns: data_insert_model + .clone() + .columns + .into_iter() + .map(|x| SqlColumn { + name: x.name.clone(), + r#type: match x.r#type { + DataSqlType::Text => SqlType::Text, + DataSqlType::UUID => SqlType::UUID, + DataSqlType::Array => SqlType::Array, + }, + }) + .collect::>(), + inserts: data_insert_model.inserts.clone(), + }) + } + + return sql_models; +} + +pub fn map_insert_to_json( + key: &String, + insert_model: &DataInsertModel, +) -> HashMap { + let mut inserts: JsonDataArray = vec![]; + + for map in insert_model.inserts.iter() { + inserts.push(map.clone()); + } + + HashMap::from([(key.clone(), inserts)]) +} + +pub fn map_data_to_json(data_model: &RawDataModel) -> JsonRawModel { + let mut data = LinkedHashMap::::new(); + + for (k, v) in data_model.data.iter() { + let mut inserts: Vec> = vec![]; + + for i in &v.inserts { + inserts.push(i.clone()); + } + + data.insert(k.clone(), inserts); + } + + JsonRawModel { data: data } +} diff --git a/tesql/src/sqlizer.rs b/tesql/src/sqlizer.rs new file mode 100644 index 0000000..876f042 --- /dev/null +++ b/tesql/src/sqlizer.rs @@ -0,0 +1,154 @@ +use linked_hash_map::LinkedHashMap; +use serde_json::Value; + +#[derive(Debug)] +pub enum SqlType { + UUID, + Text, + Array, +} + +#[derive(Debug)] +pub struct SqlColumn { + pub name: String, + pub r#type: SqlType, +} + +#[derive(Debug)] +pub struct SqlInsertModel { + pub table_name: String, + pub columns: Vec, + pub inserts: Vec>, +} + +#[derive(Debug)] +pub enum StringifyInsertError { + ValueNotFoundForColumn(String), + Invalid, +} + +pub fn sqlize_insert(m: &SqlInsertModel) -> Result { + let mut sql = format!( + "INSERT INTO \"{}\" ({})\nVALUES ", + m.table_name, + m.columns + .iter() + .enumerate() + .fold(String::new(), |acc, (i, x)| { + if i == 0 { + x.name.clone() + } else { + format!("{}, {}", acc, x.name) + } + }) + ) + .to_string(); + + for (i, rows) in m.inserts.iter().enumerate() { + let mut values = String::from("("); + for (i, c) in m.columns.iter().enumerate() { + let column_name = &c.name; + let row_val = rows.get(column_name); + + match row_val { + Some(val) => match val { + Value::Bool(val) => { + values.push_str(&val.to_string()); + } + Value::Array(val) => values.push_str(&format!("'{}'", &stringify_vec(val))), + Value::String(val) => { + values.push_str(&format!("'{}'", val)); + } + Value::Null => {} + _ => return Err(StringifyInsertError::Invalid), + }, + None => { + return Err(StringifyInsertError::ValueNotFoundForColumn( + column_name.clone(), + )) + } + }; + + if i != m.columns.len() - 1 { + values.push_str(", "); + } + } + values.push_str(if i == m.inserts.len() - 1 { + ");" + } else { + "), " + }); + sql.push_str(&values); + } + + Ok(sql) +} + +pub fn compose_inserts(sql_models: Vec) -> Result { + let mut sql_str = String::new(); + let is_multiline = sql_models.len() > 1; + + for sm in sql_models.iter() { + match sqlize_insert(sm) { + Ok(sql) => { + let mut insert_str = if is_multiline { + format!("{}\n\n", sql.to_owned()) + } else { + format!("{}", sql.to_owned()) + }; + sql_str.push_str(insert_str.as_mut_str()); + } + Err(e) => return Err(e), + } + } + Ok(sql_str) +} + +fn stringify_vec(v: &Vec) -> String { + let mut str = String::from("{"); + for (i, x) in v.iter().enumerate() { + let is_last = i == v.len() - 1; + match x { + Value::Array(ar) => { + let s = if is_last { + stringify_vec(ar) + } else { + format!("{}, ", stringify_vec(ar)) + }; + str.push_str(&s); + } + _ => { + let s = if is_last { + format!("{}", &x.to_string()) + } else { + format!("{}, ", &x.to_string()) + }; + str.push_str(&s.to_string()); + } + } + } + str.push_str("}"); + str +} + +#[cfg(test)] +mod sqlizer_tests { + use serde_json::{json, Value}; + + use crate::sqlizer::{self}; + + #[test] + fn test_stringify_vec() { + let vec: Vec = vec![json!(1), json!(2), json!(3)]; + let str = sqlizer::stringify_vec(&vec); + assert_eq!(String::from("{1, 2, 3}"), str); + + // nested arrays + let vec: Vec = vec![ + json!(vec![String::from("hello")]), + json!(vec![String::from("world")]), + ]; + let str = sqlizer::stringify_vec(&vec); + assert_eq!(String::from("{{\"hello\"}, {\"world\"}}"), str); + } +} diff --git a/tesql/testdata/data-test.json b/tesql/testdata/data-test.json new file mode 100644 index 0000000..f6a309f --- /dev/null +++ b/tesql/testdata/data-test.json @@ -0,0 +1,51 @@ +{ + "data": { + "user": { + "columns": [ + { "name": "id", "type": "uuid" }, + { "name": "username", "type": "text" }, + { "name": "is_student", "type": "text" }, + { "name": "friends_ids", "type": "array" }, + { "name": "nested_arrays", "type": "array" } + ], + "inserts": [ + { + "id": "0101111111111", + "username": "Bob", + "is_student": true, + "friends_ids": ["hello", "world", "hi!"], + "nested_arrays": [ + ["hey", "man"], + ["the", "world", "goes", "round"] + ] + }, + { + "id": "2", + "username": "Bob2", + "is_student": false, + "friends_ids": ["hi", "mom"], + "nested_arrays": [["hello", "there"], ["hello", "world"], ["hi!"]] + } + ] + }, + "user1": { + "columns": [ + { "name": "id", "type": "uuid" }, + { "name": "username", "type": "text" }, + { "name": "bool", "type": "text" } + ], + "inserts": [ + { + "id": "0101", + "username": "Bob", + "bool": true + }, + { + "id": "2", + "username": "Bob2", + "bool": false + } + ] + } + } +} diff --git a/tesql/tests/gen-inserts-test.rs b/tesql/tests/gen-inserts-test.rs new file mode 100644 index 0000000..36578d8 --- /dev/null +++ b/tesql/tests/gen-inserts-test.rs @@ -0,0 +1,22 @@ +#[cfg(test)] +mod integration { + use std::fs::{self}; + + use tesql::{gen_inserts_from_file, GenInsertOptions}; + + #[test] + fn test_gen_inserts_from_file_single_file() { + let opts = GenInsertOptions { + split_data_files: false, + sql_out_file_path: "./testgen-out/data-test.sql", + ..Default::default() + }; + match gen_inserts_from_file("./testdata/data-test.json", opts) { + Ok(_) => assert!(true), + Err(_) => assert!(false), + } + + let str = fs::read_to_string("./testgen-out/data-test.sql").unwrap(); + assert_eq!(true, str.starts_with("INSERT INTO \"user\"")); + } +}