#174 replace language container stuff with pec and slc pytest plugin (#…

…182)

* #174 Replaced Language Container Stuff with PEC
* Fixed typeguard errors
* #183 Fixed warning on tests with __init__ constructor
* Added plugin pytest-exasol-slc and used it for creating the SLC for integration tests.
* disabled backend saas for pytest
* #180 Replaced ITDE by pytest plugin exasol-backend
* Added dependency to pytest-exasol-extension
* tempoarily replacing bucketfs_connection_factory with own implementation, still, preserving the old interface to current test implementation in query_handler_runner_udf.py, still using the olf bucketfs api for now. Will be fixed in the scope of github issue #176.
* Removed SLC deployer and tests
  - language_container_deployer_cli.py
  - language_container_deployer.py
  - test_language_container_deployer_cli.py
  - test_language_container_deployer.py
* #184 Updated micromamba to version 2.0.0

.github/workflows/tests-integration-db.yml

@@ -44,13 +44,7 @@ jobs:
       - name: Poetry install
         run: poetry run -- nox -s run_in_dev_env -- poetry install
 
-      - name: Build language container
-        run: poetry run -- nox -s build_language_container
-
-      - name: Start test environment
-        run: poetry run -- nox -s start_integration_test_environment
-
       - name: Run Python integration tests with db
-        run: poetry run -- nox -s run_python_integration_tests_with_db
+        run: poetry run -- nox -s run_python_integration_tests_with_db -- -- --setup-show --backend=onprem
         env:
           PYTEST_ADDOPTS: '-o log_cli=true -o log_cli_level=INFO ${{ steps.pytest-markers.outputs.slow-tests }}'

.gitignore

@@ -161,3 +161,6 @@ doc/_build
 
 # emacs
 TAGS
+
+# locally build script language container files
+.slc

doc/changes/changes_0.1.0.md

@@ -44,7 +44,27 @@ Code name:
 * #95: Remove setup.py
 * #114: Refactored BackgroundPeerState and introduced parameter objects
 * #173: Introduced Python Toolbox
+* #174: Replaced Language Container Stuff with PEC and SLC plugin
+* #183: Fixed warning on tests with `__init__` constructor
+* #184: Updated micromamba to the latest version 2.0.0
 
 ### Documentation
 
 * #9: Added README file
+
+## Dependency Updates
+
+Compared to `main` branch this release updates the following dependencies:
+
+### File `pyproject.toml`
+
+* Updated dependency `exasol-bucketfs:0.8.0` to `0.13.0`
+* Updated dependency `pyexasol:0.25.2` to `0.27.0`
+* Updated dependency `typeguard:2.13.3` to `4.3.0`
+* Updated dependency `exasol-integration-test-docker-environment:3.1.0` to `3.2.0`
+* Updated dependency `polyfactory:2.16.2` to `2.17.0`
+* Added dependency `exasol-python-extension-common:0.6.0`
+* Added dependency `exasol-script-languages-container-tool:1.0.0`
+* Added dependency `pytest-exasol-slc:0.3.0`
+* Added dependency `pytest-exasol-backend:0.3.0`
+* Added dependency `pytest-exasol-extension:0.1.0`

exasol_advanced_analytics_framework/deploy.py

@@ -1,18 +1,31 @@
 import logging
 import click
-from exasol_advanced_analytics_framework.deployment. \
-    language_container_deployer_cli import language_container_deployer_main
-from exasol_advanced_analytics_framework.deployment. \
-    scripts_deployer_cli import scripts_deployer_main
+from exasol_advanced_analytics_framework.slc import (
+    SLC_FILE_NAME,
+    SLC_URL_FORMATTER,
+)
+from exasol_advanced_analytics_framework.deployment import (
+    scripts_deployer_cli,
+    language_container_deployer_cli,
+)
+from exasol.python_extension_common.deployment.language_container_deployer_cli import (
+    language_container_deployer_main,
+    slc_parameter_formatters,
+    CustomizableParameters,
+)
 
 
 @click.group()
 def main():
     pass
 
 
-main.add_command(scripts_deployer_main)
-main.add_command(language_container_deployer_main)
+slc_parameter_formatters.set_formatter(CustomizableParameters.container_url, SLC_URL_FORMATTER)
+slc_parameter_formatters.set_formatter(CustomizableParameters.container_name, SLC_FILE_NAME)
+
+main.add_command(scripts_deployer_cli.scripts_deployer_main)
+main.add_command(language_container_deployer_cli.language_container_deployer_main)
+
 
 if __name__ == '__main__':
     logging.basicConfig(

exasol_advanced_analytics_framework/deployment/scripts_deployer.py

@@ -52,10 +52,8 @@ def deploy_scripts(self) -> None:
     @classmethod
     def run(cls, dsn: str, user: str, password: str,
             schema: str, language_alias: str, develop: bool):
-
         if develop:
             save_aaf_query_loop_lua_script()
-
         pyexasol_conn = pyexasol.connect(dsn=dsn, user=user, password=password)
         scripts_deployer = cls(language_alias, schema, pyexasol_conn)
         scripts_deployer.deploy_scripts()

exasol_advanced_analytics_framework/deployment/scripts_deployer_cli.py

@@ -2,28 +2,27 @@
 from exasol_advanced_analytics_framework.deployment import utils
 from exasol_advanced_analytics_framework.deployment.scripts_deployer import \
     ScriptsDeployer
-
+from exasol_advanced_analytics_framework.slc import LANGUAGE_ALIAS
 
 @click.command(name="scripts")
 @click.option('--dsn', type=str, required=True)
 @click.option('--user', type=str, required=True)
 @click.option('--pass', 'pwd', type=str)
 @click.option('--schema', type=str, required=True)
-@click.option('--language-alias', type=str, default="PYTHON3_AAF")
+@click.option('--language-alias', type=str, default=LANGUAGE_ALIAS)
 @click.option('--develop', type=bool, is_flag=True)
 def scripts_deployer_main(
         dsn: str, user: str, pwd: str, schema: str,
         language_alias: str, develop: bool):
     password = utils.get_password(
         pwd, user, utils.DB_PASSWORD_ENVIRONMENT_VARIABLE, "DB Password")
-
     ScriptsDeployer.run(
         dsn=dsn,
         user=user,
         password=password,
         schema=schema,
         language_alias=language_alias,
-        develop=develop
+        develop=develop,
     )
 
 

exasol_advanced_analytics_framework/slc.py

@@ -0,0 +1,18 @@
+from contextlib import contextmanager
+from exasol.python_extension_common.deployment.language_container_builder import (
+    LanguageContainerBuilder,
+    find_path_backwards
+)
+
+LANGUAGE_ALIAS = "PYTHON3_AAF"
+SLC_NAME = "exasol_advanced_analytics_framework_container"
+SLC_FILE_NAME = SLC_NAME + ".tar.gz"
+SLC_URL_FORMATTER = "https://github.com/exasol/advanced_analytics_framework/releases/download/{version}/" + SLC_NAME
+
+
+@contextmanager
+def custom_slc_builder() -> LanguageContainerBuilder:
+    project_directory = find_path_backwards("pyproject.toml", __file__).parent
+    with LanguageContainerBuilder(SLC_NAME) as builder:
+        builder.prepare_flavor(project_directory)
+        yield builder

exasol_data_science_utils_python/schema/column_name.py

@@ -9,7 +9,7 @@
 
 class ColumnName(ExasolIdentifierImpl):
     @typechecked
-    def __init__(self, name: str, table_like_name: TableLikeName = None):
+    def __init__(self, name: str, table_like_name: TableLikeName|None = None):
         super().__init__(name)
         self._table_like_name = table_like_name
 

exasol_data_science_utils_python/utils/data_classes_runtime_type_check.py

@@ -1,10 +1,12 @@
 from dataclasses import fields
 
 import typeguard
-
+from typeguard import TypeCheckError
 
 def check_dataclass_types(datacls):
     for field in fields(datacls):
-        typeguard.check_type(value=datacls.__dict__[field.name],
-                             expected_type=field.type,
-                             argname=field.name)
+        try:
+            typeguard.check_type(value=datacls.__dict__[field.name],
+                                 expected_type=field.type)
+        except TypeCheckError as e:
+            raise TypeCheckError(f"Field '{field.name}' has wrong type: {e}")

noxfile.py

@@ -1,6 +1,8 @@
 import json
 import os
 from pathlib import Path
+from exasol_advanced_analytics_framework.slc import custom_slc_builder
+from datetime import datetime
 
 import nox
 from nox import Session
@@ -108,11 +110,17 @@ def start_integration_test_environment(session: Session):
 
 @nox.session(python=False)
 def build_language_container(session: Session):
-    script_path = ROOT_DIR / "build_language_container.sh"
-    session.run(str(script_path))
+    export_path = ROOT_DIR / ".slc"
+    with custom_slc_builder() as builder:
+        builder.export(export_path)
 
 
 @nox.session(python=False)
 def run_python_integration_tests_with_db(session: Session):
     integration_test_directory = INTEGRATION_TEST_DIRECTORY / "with_db"
-    _run_in_dev_env_poetry_call(session, "pytest", str(integration_test_directory))
+    _run_in_dev_env_poetry_call(
+        session,
+        "pytest",
+        str(integration_test_directory),
+        *session.posargs,
+    )