refactor(benchmarks): evaluation pipeline (#71)

deepsense-ai · Aug 6, 2024 · cd2cece · cd2cece
1 parent 90b0e66
commit cd2cece
Show file tree

Hide file tree

Showing 65 changed files with 2,784 additions and 1,600 deletions.
diff --git a/.gitignore b/.gitignore
@@ -55,6 +55,7 @@ licenses.txt
 **/dist/
 **/checkpoints/
 **/outputs/
+**/multirun/
 
 # Other env files
 .python-version
@@ -74,7 +75,6 @@ coverage.xml
 
 # dotenv
 .env
-src/dbally_benchmark/.env
 
 # coverage and pytest reports
 coverage.xml
@@ -87,8 +87,8 @@ cmake-build-*/
 **/.terraform.lock.hcl
 **/.terraform
 
-# experiments results
-experiments/
+# benchmarks
+benchmarks/sql/data/
 
 # mkdocs generated files
 site/
diff --git a/benchmark/dbally_benchmark/config.py b/benchmark/dbally_benchmark/config.py
diff --git a/benchmark/dbally_benchmark/constants.py b/benchmark/dbally_benchmark/constants.py
diff --git a/benchmark/dbally_benchmark/dataset/bird_dataset.py b/benchmark/dbally_benchmark/dataset/bird_dataset.py
diff --git a/benchmark/dbally_benchmark/e2e_benchmark.py b/benchmark/dbally_benchmark/e2e_benchmark.py
diff --git a/benchmark/dbally_benchmark/evaluate.py b/benchmark/dbally_benchmark/evaluate.py
diff --git a/benchmark/dbally_benchmark/experiment_config/config.yaml b/benchmark/dbally_benchmark/experiment_config/config.yaml
diff --git a/benchmark/dbally_benchmark/experiment_config/e2e/superhero.yaml b/benchmark/dbally_benchmark/experiment_config/e2e/superhero.yaml
diff --git a/benchmark/dbally_benchmark/experiment_config/iql/superhero.yaml b/benchmark/dbally_benchmark/experiment_config/iql/superhero.yaml
diff --git a/benchmark/dbally_benchmark/experiment_config/text2sql/superhero.yaml b/benchmark/dbally_benchmark/experiment_config/text2sql/superhero.yaml
diff --git a/benchmark/dbally_benchmark/iql/iql_result.py b/benchmark/dbally_benchmark/iql/iql_result.py