MeltanoLabs · aaronsteers · Jan 29, 2022 · Jan 29, 2022 · Jan 29, 2022 · Jan 29, 2022
diff --git a/.github/workflows/ci_workflow.yml b/.github/workflows/ci_workflow.yml
@@ -0,0 +1,58 @@
+### A CI workflow template that runs linting and python testing
+### TODO: Modify as needed or as desired.
+
+name: Test 'singer-sqlite'
+
+on: [push]
+
+jobs:
+  linting:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        # Only lint using the primary version used for dev
+        python-version: [3.9]
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python '${{ matrix.python-version }}'
+      uses: actions/setup-python@v2
+      with:
+        python-version: '${{ matrix.python-version }}'
+    - name: Install Poetry
+      uses: snok/install-poetry@v1
+      with:
+        version: 1.1.12
+    - name: Install dependencies
+      run: |
+        poetry install
+    - name: Run lint command from tox.ini
+      run: |
+        poetry run tox -e lint
+
+  pytest:
+
+    runs-on: ubuntu-latest
+    env:
+      GITHUB_TOKEN: '${{secrets.GITHUB_TOKEN}} }}'
+    strategy:
+      matrix:
+        python-version: [3.7, 3.8, 3.9]
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python '${{ matrix.python-version }}'
+      uses: actions/setup-python@v2
+      with:
+        python-version: '${{ matrix.python-version }}'
+    - name: Install Poetry
+      uses: snok/install-poetry@v1
+      with:
+        version: 1.1.11
+    - name: Install dependencies
+      run: |
+        poetry install
+    - name: Test with pytest
+      run: |
+        poetry run pytest --capture=no
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
diff --git a/target_sqlite/sqlite_loader.py → bak/sqlite_loader.py b/target_sqlite/sqlite_loader.py → bak/sqlite_loader.py
@@ -9,7 +9,7 @@
 from sqlalchemy.schema import CreateSchema
 from sqlalchemy import exc
 
-from target_sqlite.utils.error import SchemaUpdateError
+from singer_sqlite.utils.error import SchemaUpdateError
 
 
 # Map sqlalchemy types to SQLite Types

diff --git a/target_sqlite/target_sqlite.py → bak/target_sqlite.py b/target_sqlite/target_sqlite.py → bak/target_sqlite.py
@@ -6,12 +6,12 @@
 from jsonschema import ValidationError, Draft4Validator, FormatChecker
 from typing import Dict, List, Iterator
 
-from target_sqlite.utils.singer_target_utils import (
+from singer_sqlite.utils.singer_target_utils import (
     flatten_record,
     flatten_key,
     generate_sqlalchemy_table,
 )
-from target_sqlite.sqlite_loader import SQLiteLoader
+from singer_sqlite.sqlite_loader import SQLiteLoader
 
 
 LOGGER = singer.get_logger()
@@ -350,7 +350,7 @@ def emit_state(self, state) -> None:
         """
         if state is not None:
             line = json.dumps(state)
-            LOGGER.debug('Emitting state {}'.format(line))
+            LOGGER.debug("Emitting state {}".format(line))
             sys.stdout.write("{}\n".format(line))
             sys.stdout.flush()
 

diff --git a/bak/tests-bak/bak-conftest.py b/bak/tests-bak/bak-conftest.py
@@ -0,0 +1,31 @@
+import json
+import pytest
+
+
+class SilencedDict(dict):
+    def __repr__(self):
+        return "Dict[ ... sensitive_data ... ]"
+
+    def __str__(self):
+        return "Dict[ ... sensitive_data ... ]"
+
+
+def pytest_addoption(parser):
+    parser.addoption("--config", help="Config file", required=True)
+
+
+@pytest.fixture(scope="session")
+def config(request):
+    config = SilencedDict()
+
+    config_file = request.config.getoption("--config")
+
+    if config_file:
+        with open(config_file) as input:
+            config.update(json.load(input))
+    else:
+        raise Exception("No config file provided")
+
+    config["timestamp_column"] = config.get("timestamp_column", "__loaded_at")
+
+    return config
diff --git a/tests/test_sqlite_loader.py → bak/tests-bak/bak-test_sqlite_loader.py b/tests/test_sqlite_loader.py → bak/tests-bak/bak-test_sqlite_loader.py
@@ -7,7 +7,7 @@
 from sqlalchemy import MetaData, Table, Column, inspect
 from sqlalchemy.types import TIMESTAMP, Float, String, BigInteger, Boolean
 
-from target_sqlite.sqlite_loader import SQLiteLoader
+from singer_sqlite.sqlite_loader import SQLiteLoader
 
 
 @pytest.fixture(scope="class")

diff --git a/tests/test_target_sqlite.py → bak/tests-bak/bak-test_target_sqlite.py b/tests/test_target_sqlite.py → bak/tests-bak/bak-test_target_sqlite.py
@@ -4,8 +4,7 @@
 from jsonschema import ValidationError
 from sqlalchemy import create_engine, inspect
 
-from target_sqlite.target_sqlite import TargetSQLite
-from target_sqlite.utils.error import SchemaUpdateError
+from singer_sqlite import TargetSQLite, TapSQLite
 
 
 def load_stream(filename):

diff --git a/tests/data_files/array_data.stream → bak/tests-bak/data_files/array_data.stream b/tests/data_files/array_data.stream → bak/tests-bak/data_files/array_data.stream
diff --git a/tests/data_files/camelcase.stream → bak/tests-bak/data_files/camelcase.stream b/tests/data_files/camelcase.stream → bak/tests-bak/data_files/camelcase.stream
diff --git a/...ata_files/camelcase_complex_schema.stream → ...ata_files/camelcase_complex_schema.stream b/...ata_files/camelcase_complex_schema.stream → ...ata_files/camelcase_complex_schema.stream
diff --git a/tests/data_files/duplicate_records.stream → ...s-bak/data_files/duplicate_records.stream b/tests/data_files/duplicate_records.stream → ...s-bak/data_files/duplicate_records.stream
diff --git a/tests/data_files/encoded_strings.stream → ...sts-bak/data_files/encoded_strings.stream b/tests/data_files/encoded_strings.stream → ...sts-bak/data_files/encoded_strings.stream
diff --git a/tests/data_files/invalid_schema.stream → ...ests-bak/data_files/invalid_schema.stream b/tests/data_files/invalid_schema.stream → ...ests-bak/data_files/invalid_schema.stream
diff --git a/...data_files/multiple_state_messages.stream → ...data_files/multiple_state_messages.stream b/...data_files/multiple_state_messages.stream → ...data_files/multiple_state_messages.stream
diff --git a/tests/data_files/no_primary_keys.stream → ...sts-bak/data_files/no_primary_keys.stream b/tests/data_files/no_primary_keys.stream → ...sts-bak/data_files/no_primary_keys.stream
diff --git a/.../data_files/no_primary_keys_append.stream → .../data_files/no_primary_keys_append.stream b/.../data_files/no_primary_keys_append.stream → .../data_files/no_primary_keys_append.stream
diff --git a/tests/data_files/optional_attributes.stream → ...bak/data_files/optional_attributes.stream b/tests/data_files/optional_attributes.stream → ...bak/data_files/optional_attributes.stream
diff --git a/tests/data_files/record_before_schema.stream → ...ak/data_files/record_before_schema.stream b/tests/data_files/record_before_schema.stream → ...ak/data_files/record_before_schema.stream
diff --git a/..._files/record_missing_key_property.stream → ..._files/record_missing_key_property.stream b/..._files/record_missing_key_property.stream → ..._files/record_missing_key_property.stream
diff --git a/...s/record_missing_required_property.stream → ...s/record_missing_required_property.stream b/...s/record_missing_required_property.stream → ...s/record_missing_required_property.stream
diff --git a/tests/data_files/schema_no_properties.stream → ...ak/data_files/schema_no_properties.stream b/tests/data_files/schema_no_properties.stream → ...ak/data_files/schema_no_properties.stream
diff --git a/tests/data_files/schema_updates.stream → ...ests-bak/data_files/schema_updates.stream b/tests/data_files/schema_updates.stream → ...ests-bak/data_files/schema_updates.stream
diff --git a/..._files/special_chars_in_attributes.stream → ..._files/special_chars_in_attributes.stream b/..._files/special_chars_in_attributes.stream → ..._files/special_chars_in_attributes.stream
diff --git a/tests/data_files/user_location_data.stream → ...-bak/data_files/user_location_data.stream b/tests/data_files/user_location_data.stream → ...-bak/data_files/user_location_data.stream
diff --git a/...ta_files/user_location_upsert_data.stream → ...ta_files/user_location_upsert_data.stream b/...ta_files/user_location_upsert_data.stream → ...ta_files/user_location_upsert_data.stream
diff --git a/target_sqlite/utils/__init__.py → bak/utils/__init__.py b/target_sqlite/utils/__init__.py → bak/utils/__init__.py
diff --git a/target_sqlite/utils/error.py → bak/utils/error.py b/target_sqlite/utils/error.py → bak/utils/error.py
diff --git a/target_sqlite/utils/singer_target_utils.py → bak/utils/singer_target_utils.py b/target_sqlite/utils/singer_target_utils.py → bak/utils/singer_target_utils.py