Add e2e tests (#6)

* Add e2e tests

Author: AndrewNepogoda

.github/workflows/run-tests.yml

@@ -36,3 +36,27 @@ jobs:
         run: |
           cd $GITHUB_WORKSPACE/test
           python -m unittest discover
+
+      # Setup config required for Python CLI run
+      - name: Config setup
+        run: |
+          mkdir ~/.rai
+          cd ~/.rai
+          echo "[default]" > config
+          echo "region = us-east" >> config
+          echo "host = azure.relationalai.com" >> config
+          echo "port = 443" >> config
+          echo "client_id = ${{ env.RAI_CLIENT_ID }}" >> config
+          echo "client_secret = ${{ env.RAI_CLIENT_SECRET }}" >> config
+          # Create empty toml
+          cd $GITHUB_WORKSPACE/cli-e2e-test/config
+          touch loader.toml
+        env:
+          RAI_CLIENT_ID: ${{ secrets.client_id }}
+          RAI_CLIENT_SECRET: ${{ secrets.client_secret }}
+
+      # Run e2e Python CLI test
+      - name: Python CLI tests
+        run: |
+          cd $GITHUB_WORKSPACE/cli-e2e-test
+          python -m unittest discover

.gitignore

@@ -235,9 +235,6 @@ tramp
 # elpa packages
 /elpa/
 
-# reftex files
-*.rel
-
 # AUCTeX auto folder
 /auto/
 

cli-e2e-test/__init__.py

@@ -0,0 +1,85 @@
+{
+  "workflow": [
+    {
+      "type": "ConfigureSources",
+      "name": "ConfigureSources",
+      "configFiles": [
+        "config/scenario1.rel"
+      ],
+      "sources": [
+        {
+          "relation": "zip_city_state_master_data",
+          "isPartitioned": false,
+          "relativePath": "master_source/zip_city_state",
+          "inputFormat": "csv",
+          "isMaster": true
+        },
+        {
+          "relation": "device_data",
+          "isPartitioned": true,
+          "relativePath": "device",
+          "inputFormat": "csv",
+          "loadsNumberOfDays": 60
+        },
+        {
+          "relation": "store_data",
+          "isPartitioned": true,
+          "relativePath": "store",
+          "inputFormat": "jsonl",
+          "extensions": [
+            "json",
+            "jsonl"
+          ],
+          "loadsNumberOfDays": 60
+        }
+      ]
+    },
+    {
+      "type": "InstallModels",
+      "name": "InstallModels1",
+      "modelFiles": [
+        "device.rel",
+        "zip.rel",
+        "store.rel",
+        "json_schema_mapping.rel"
+      ]
+    },
+    {
+      "type": "LoadData",
+      "name": "LoadData"
+    },
+    {
+      "type": "Materialize",
+      "name": "Materialize",
+      "relations": [
+        "city:name",
+        "device:name",
+        "store:name"
+      ],
+      "materializeJointly": true
+    },
+    {
+      "type": "Export",
+      "name": "Export",
+      "exportJointly": false,
+      "dateFormat": "%Y%m%d",
+      "exports": [
+        {
+          "type": "csv",
+          "configRelName": "cities_csv",
+          "relativePath": "cities"
+        },
+        {
+          "type": "csv",
+          "configRelName": "devices_csv",
+          "relativePath": "devices"
+        },
+        {
+          "type": "csv",
+          "configRelName": "stores_csv",
+          "relativePath": "stores"
+        }
+      ]
+    }
+  ]
+}

cli-e2e-test/config/model/scenario1.json

@@ -0,0 +1,3 @@
+device,IMEI
+IPhone1,123
+IPhone2,124

cli-e2e-test/data/device/data_dt=20220105/part-1.csv

@@ -0,0 +1,3 @@
+ZIP_CODE,CITY_NAME,STATE_CODE,STATE_NAME
+90005,Los Angeles,CA,California
+90401,Santa Monica,CA,California

cli-e2e-test/data/master_source/zip_city_state/zip_city_state.csv

@@ -0,0 +1,2 @@
+{ "storeId": "0ffc-0909", "storeName": "Store1"}
+{ "storeId": "0567-aa98", "storeName": "Store2"}

cli-e2e-test/data/store/data_dt=20220105/part-1.jsonl

@@ -0,0 +1,3 @@
+name
+Los Angeles
+Santa Monica

cli-e2e-test/expected_results/cities_csv.csv

@@ -0,0 +1,3 @@
+name
+IPhone1
+IPhone2

cli-e2e-test/expected_results/devices_csv.csv

@@ -0,0 +1,3 @@
+name
+Store1
+Store2

cli-e2e-test/expected_results/stores_csv.csv

@@ -0,0 +1,4 @@
+import cli.runner
+
+if __name__ == "__main__":
+    cli.runner.start()

cli-e2e-test/main.py

@@ -0,0 +1 @@
+This dir is used in dev mode to store the output csv of the test data.

cli-e2e-test/output/README.md

@@ -0,0 +1,30 @@
+@inline
+module export_config
+    module cities_csv
+        def data = city
+
+        def syntax:header = {
+            1, :name
+        }
+    end
+
+    module devices_csv
+        def data = device
+
+        def syntax:header = {
+            1, :name
+        }
+    end
+
+    module stores_csv
+        def data = store
+
+        def syntax:header = {
+            1, :name
+        }
+    end
+end
+
+def part_resource_date_pattern = "^(.+)/data_dt=(?<date>[0-9]+)/(.+).(csv|json|jsonl)$"
+def part_resource_index_pattern = "^(.+)/data_dt=(.+)/part-(?<shard>[0-9]).(csv|json|jsonl)$"
+def part_resource_index_multiplier = 100000

cli-e2e-test/rel/config/scenario1.rel

@@ -0,0 +1,10 @@
+bound import_config:device_data:schema
+bound syntax:header
+
+module device_data
+    def DEVICE_NAME[idx, row] = source_catalog:device_data[idx, :device, row]
+end
+
+module device
+    def name = device_data:DEVICE_NAME[_, _]
+end

cli-e2e-test/rel/device.rel

@@ -0,0 +1,64 @@
+/**
+ * The modules in this file automate the schema mapping of JSON or JSONL sources
+ * that are configured as part of the json_source_config module.
+ *
+ * To use this framework, simply populate the json_source_config relation for
+ * some *source* with the string values of keys for any one or more of the
+ * following types: (string, int, decimal, and object).
+ */
+
+def JsonSourceConfigKey(k) { json_source_config(k, x...) from x... }
+
+def json_source:value[src in JsonSourceConfigKey] = source_catalog[src, _, :value]
+def json_source:child[src in JsonSourceConfigKey] = source_catalog[src, _, :child]
+def json_source:root[src in JsonSourceConfigKey] = source_catalog[src, _, :root]
+def json_source:array[src in JsonSourceConfigKey] = source_catalog[src, _, :array]
+
+def json_source:value[src in JsonSourceConfigKey] = simple_source_catalog[src, :value]
+def json_source:child[src in JsonSourceConfigKey] = simple_source_catalog[src, :child]
+def json_source:root[src in JsonSourceConfigKey] = simple_source_catalog[src, :root]
+def json_source:array[src in JsonSourceConfigKey] = simple_source_catalog[src, :array]
+
+/**
+ * [REKS] Note that json_source, json_hash, json_name_sym, and json_schema cannot
+ *        be further consolidated into a container module because of limits on the
+ *        use of metafication (i.e., the #(...) operator) inside recursive
+ *        definitions.
+ */
+def json_hash(src, t, h, s) =
+    murmurhash3f(s, x) and
+    h = uint128_hash_value_convert[x] and
+    json_source_config(src, t, s)
+    from x
+
+def json_name_sym[src, x] = #(json_hash[src, _, x])
+
+module json_schema
+
+    def parent(src, y, x, z) { json_source:child(src, x, y, z) }
+
+    def value(src, r, o, val) {
+        parent(src, r, o, t) and
+        json_hash(src, :string, r, _) and
+        json_source:value(src, t, val) and
+        String(val)
+        from t
+    }
+
+    def value(src, r, o, v) {
+        parent(src, r, o, t) and
+        json_hash(src, :decimal, r, _) and
+        json_source:value(src, t, v) and
+        Float(v)
+        from t
+    }
+
+    def value(src, r, o, v) {
+        parent(src, r, o, t) and
+        json_hash(src, :int, r, _) and
+        json_source:value(src, t, v) and
+        Int(v)
+        from t
+    }
+
+end

cli-e2e-test/rel/json_schema_mapping.rel

@@ -0,0 +1,34 @@
+def json_source_config:store_data:string = {
+    "storeId" ;
+    "storeName"
+}
+
+def json_source_config:store_data:object(k) {
+    store_data_metadata:object_or_value_key(k) and
+    not json_source_config:store_data:string(k)
+}
+
+module store_data_metadata
+
+    def object_or_value_key = covers[_]
+
+    def covers = {
+        "_root_", "storeId" ;
+        "_root_", "storeName"
+    }
+
+end
+
+def store_data_value(t, o, v) =
+    json_schema:value:store_data(h, o, v) and
+    json_name_sym:store_data(h, t)
+    from h
+
+def store_data_object(t, o, v) =
+    json_schema:parent:store_data(h, o, v) and
+    json_name_sym:store_data(h, t)
+    from h
+
+module store
+    def name = store_data_value:storeName[_]
+end

cli-e2e-test/rel/store.rel

@@ -0,0 +1,7 @@
+module zip_city_state_master_data
+    def CITY_NAME[row] = simple_source_catalog:zip_city_state_master_data:CITY_NAME[row]
+end
+
+module city
+    def name = zip_city_state_master_data:CITY_NAME[_]
+end

cli-e2e-test/rel/zip.rel

@@ -0,0 +1,73 @@
+import unittest
+import os
+import logging
+import uuid
+
+import workflow.manager
+from csv_diff import load_csv, compare, human_text
+from subprocess import call
+
+
+class CliE2ETest(unittest.TestCase):
+    logger: logging.Logger
+    resource_manager: workflow.manager.ResourceManager
+    output = "./output"
+    env_config = "./config/loader.toml"
+    expected = "./expected_results"
+    resource_name = "wm-cli-e2e-test-" + str(uuid.uuid4())
+    cmd_with_common_arguments = ["python", "main.py",
+                                 "--run-mode", "local",
+                                 "--env-config", env_config,
+                                 "--engine", resource_name,
+                                 "--database", resource_name,
+                                 "--rel-config-dir", "./rel",
+                                 "--dev-data-dir", "./data",
+                                 "--output-root", output]
+
+    def test_scamp_aggregates_model(self):
+        # when
+        test_args = ["--batch-config", "./config/model/scenario1.json",
+                     "--start-date", "20220103",
+                     "--end-date", "20220105"]
+        command = self.cmd_with_common_arguments
+        command += test_args
+        rsp = call(command)
+        # then
+        self.assertNotEqual(rsp, 1)
+        self.assert_output_dir_files()
+
+    @classmethod
+    def setUpClass(cls) -> None:
+        # Make sure output folder is empty since the folder share across repository. Remove README.md, other files left.
+        cleanup_output(cls.output)
+        cls.logger = logging.getLogger("cli-e2e-test")
+        cls.resource_manager = workflow.manager.ResourceManager.init(cls.logger, cls.resource_name, cls.resource_name)
+        cls.logger.setLevel(logging.INFO)
+        cls.logger.addHandler(logging.StreamHandler())
+        cls.resource_manager.add_engine()
+
+    def tearDown(self):
+        cleanup_output(self.output)
+
+    @classmethod
+    def tearDownClass(cls) -> None:
+        cls.resource_manager.cleanup_resources()
+
+    def assert_output_dir_files(self):
+        for filename in os.listdir(self.output):
+            actual_path = os.path.join(self.output, filename)
+            expected_path = os.path.join( self.expected, filename)
+            with open(actual_path, 'r') as actual, open(expected_path, 'r') as expected:
+                diff = compare(
+                    load_csv(actual),
+                    load_csv(expected)
+                )
+                self.logger.info(f"Assert file `{filename}`")
+                self.assertEqual(human_text(diff), '')
+
+
+def cleanup_output(directory: str):
+    for filename in os.listdir(directory):
+        file_path = os.path.join(directory, filename)
+        if os.path.isfile(file_path):
+            os.remove(file_path)

cli-e2e-test/test_e2e.py

@@ -76,7 +76,7 @@ module part_resource
         uri:identifies(u, r) and
         d = parse_int[ uri:parse[u, "date"] ] and
         s = parse_int[ uri:parse[u, shard_alias] ] and
-        n =  ^PartIndex[ d * part_resource_index_multiplier + s ]
+        n = ^PartIndex[ d * part_resource_index_multiplier + s ]
         from u, d, s
     }
 end