Refactoring/#203 clean package names and directory structure (#210)

* #203: Cleaned-up package names and directory structure
* Fixed amalgation of lua scripts
* Fixed test_python_query_handler_runner.py
* removed duplicate declaration of class DropyQuery
* Applied additional renamings as suggested by code review

* query.handler -> exasol/analytics/query_handler
* query_result -> exasol/analytics/query/handler/result
* exasol/analytics/deployment/ -> exasol.analytics.query_handler/deployment
* exasol/analytics/deploy.py -> exasol.analytics.query_handler/deployment/deploy.py
* exasol/analytics/slc.py -> exasol.analytics.query_handler/deployment/slc.py

renamed dir  exasol.analytics.query_handler.result
to  	     exasol.analytics.query_handler.query.result
renamed file exasol.analytics.query_handler.query.result.impl.py
to 	     exasol.analytics.query_handler.result.py

Co-authored-by: Nicola Coretti <[email protected]>

.gitattributes

@@ -1,3 +1,3 @@
-exasol_advanced_analytics_framework/resources/outputs/create_query_loop.sql linguist-generated
+exasol/analytics/resources/outputs/create_query_loop.sql linguist-generated
 setup.py linguist-generated
 poetry.lock linguist-generated

advanced-analytics-framework-0.1.0-1.rockspec

@@ -23,10 +23,11 @@ dependencies = {
 build = {
    type = "builtin",
    modules = {
-      ["exasol.advanced_analytics_framework.exasol_script_tools"] = "exasol_advanced_analytics_framework/lua/src/exasol_script_tools.lua",
-      ["exasol.advanced_analytics_framework.query_loop"] = "exasol_advanced_analytics_framework/lua/src/query_loop.lua",
-      ["exasol.advanced_analytics_framework.query_handler_runner_main"] = "exasol_advanced_analytics_framework/lua/src/query_handler_runner_main.lua",
-      ["exasol.advanced_analytics_framework.query_handler_runner"] = "exasol_advanced_analytics_framework/lua/src/query_handler_runner.lua"
+      -- should the lua rock modules be renamed, too?
+      ["exasol.advanced_analytics_framework.exasol_script_tools"] = "exasol/analytics/lua/src/exasol_script_tools.lua",
+      ["exasol.advanced_analytics_framework.query_loop"] = "exasol/analytics/lua/src/query_loop.lua",
+      ["exasol.advanced_analytics_framework.query_handler_runner_main"] = "exasol/analytics/lua/src/query_handler_runner_main.lua",
+      ["exasol.advanced_analytics_framework.query_handler_runner"] = "exasol/analytics/lua/src/query_handler_runner.lua"
    },
    copy_directories = {
       "doc",

doc/changes/changes_0.1.1.md

@@ -12,3 +12,4 @@ Code name:
 
 * #188: Renamed global pytest fixtures to avoid name clashes
 * #208: Replaced access to private attribute by public
+* #203: Cleaned-up package names and directory structure

doc/developer_guide/developer_guide.md

@@ -16,7 +16,7 @@ Installing the SLC ins described in the [AAF User Guide](../user_guide/user_guid
 
 ## Update Generated Files
 
-AAF contains the amalgated Lua script [create_query_loop.sql](https://github.com/exasol/advanced-analytics-framework/blob/main/exasol_advanced_analytics_framework/resources/outputs/create_query_loop.sql) originating from the files in the directory [exasol_advanced_analytics_framework/lua/src](https://github.com/exasol/advanced-analytics-framework/blob/main/exasol_advanced_analytics_framework/lua/src/).
+AAF contains the amalgated Lua script [create_query_loop.sql](https://github.com/exasol/advanced-analytics-framework/blob/main/exasol/analytics/resources/outputs/create_query_loop.sql) originating from the files in the directory [exasol/analytics/lua/src](https://github.com/exasol/advanced-analytics-framework/blob/main/exasol/analytics/lua/src/).
 
 The following command updates the amalgated script:
 

doc/user_guide/example-udf-script/create.sql

@@ -3,17 +3,18 @@ CREATE OR REPLACE PYTHON3_AAF SET SCRIPT "EXAMPLE_SCHEMA"."EXAMPLE_QUERY_HANDLER
 EMITS (outputs VARCHAR(2000000)) AS
 
 from typing import Union
-from exasol_advanced_analytics_framework.udf_framework.udf_query_handler import UDFQueryHandler
-from exasol_advanced_analytics_framework.udf_framework.dynamic_modules import create_module
-from exasol_advanced_analytics_framework.query_handler.context.query_handler_context import QueryHandlerContext
-from exasol_advanced_analytics_framework.query_result.query_result import QueryResult
-from exasol_advanced_analytics_framework.query_handler.result import Result, Continue, Finish
-from exasol_advanced_analytics_framework.query_handler.query.select_query import SelectQuery, SelectQueryWithColumnDefinition
-from exasol_advanced_analytics_framework.query_handler.context.proxy.bucketfs_location_proxy import \
-    BucketFSLocationProxy
-from exasol_data_science_utils_python.schema.column import Column
-from exasol_data_science_utils_python.schema.column_name import ColumnName
-from exasol_data_science_utils_python.schema.column_type import ColumnType
+from exasol.analytics.query_handler.udf.interface import UDFQueryHandler
+from exasol.analytics.utils.dynamic_modules import create_module
+from exasol.analytics.query_handler.context.query_handler_context import QueryHandlerContext
+from exasol.analytics.query_handler.query.result.interface import QueryResult
+from exasol.analytics.query_handler.result import Result, Continue, Finish
+from exasol.analytics.query_handler.query.select import SelectQuery, SelectQueryWithColumnDefinition
+from exasol.analytics.query_handler.context.proxy.bucketfs_location_proxy import     BucketFSLocationProxy
+from exasol.analytics.schema import (
+    Column,
+    ColumnType,
+    ColumnName,
+)
 from datetime import datetime
 from exasol.bucketfs import as_string
 
@@ -77,7 +78,7 @@ class ExampleQueryHandlerFactory:
 
 example_module.add_to_module(ExampleQueryHandlerFactory)
 
-from exasol_advanced_analytics_framework.udf_framework.query_handler_runner_udf \
+from exasol.analytics.query_handler.udf.runner.udf \
     import QueryHandlerRunnerUDF
 
 udf = QueryHandlerRunnerUDF(exa)

doc/user_guide/proxies.md

@@ -7,7 +7,7 @@ An _Object Proxy_
 * Provides a reference enabling using the object, i.e. its name incl. the database schema or the path in the BucketFS
 * Ensures the object is removed when leaving the current scope, e.g. the Query Handler.
 
-All Object Proxies are derived from class `exasol_advanced_analytics_framework.query_handler.context.proxy.object_proxy.ObjectProxy`:
+All Object Proxies are derived from class `exasol.analytics.query_handler.context.proxy.object_proxy.ObjectProxy`:
 * `BucketFSLocationProxy` encapsulates a location in the BucketFS
 * `DBObjectNameProxy` encapsulates a database object, e.g. a table
 

doc/user_guide/user_guide.md

@@ -79,7 +79,7 @@ Running the AAF requires an SLC. The following command
 * and registers it to the database.
 
 ```shell
-python -m exasol_advanced_analytics_framework.deploy language-container
+python -m exasol.analytics.query_handler.deployment.deploy language-container
 ```
 
 See the documentation in the Exasol Python Extension Common package for [options common to all Exasol extensions](https://github.com/exasol/python-extension-common/blob/0.8.0/doc/user_guide/user-guide).
@@ -91,7 +91,7 @@ Besides the BucketFS connection, the SLC, and the Python package AAF also requir
 The following command deploys the additional scripts to the specified database schema `$AAF_DB_SCHEMA` using the same language alias `$LANGUAGE_ALIAS` as for uploading the SLC before:
 
 ```shell
-python -m exasol_advanced_analytics_framework.deploy scripts \
+python -m exasol.analytics.query_handler.deployment.deploy scripts \
     --dsn "$DB_HOST:$DB_PORT" \
     --db-user "$DB_USER" \
     --db-pass "$DB_PASSWORD" \

exasol_advanced_analytics_framework/query_handler/context/connection_name.py → exasol/analytics/query_handler/context/connection_name.py

@@ -1,5 +1,8 @@
-from exasol_data_science_utils_python.schema.dbobject_name import DBObjectName
-from exasol_data_science_utils_python.schema.dbobject_name_impl import DBObjectNameImpl
+
+from exasol.analytics.schema import (
+    DBObjectNameImpl,
+    DBObjectName,
+)
 from typeguard import typechecked
 
 

exasol/analytics/query_handler/context/connection_name_proxy.py

@@ -0,0 +1,17 @@
+from exasol.analytics.query_handler.context.connection_name import ConnectionName, ConnectionNameImpl
+from exasol.analytics.query_handler.context.proxy.db_object_name_proxy import DBObjectNameProxy
+from exasol.analytics.query_handler.query.drop.connection import DropConnectionQuery
+from exasol.analytics.query_handler.query.interface import Query
+
+
+class ConnectionNameProxy(DBObjectNameProxy[ConnectionName], ConnectionName):
+
+    @property
+    def fully_qualified(self) -> str:
+        return self.quoted_name
+
+    def get_cleanup_query(self) -> Query:
+        return DropConnectionQuery(self._db_object_name)
+
+    def __init__(self, connection_name: ConnectionName, global_counter_value: int):
+        super().__init__(connection_name, global_counter_value)

exasol_advanced_analytics_framework/query_handler/context/proxy/bucketfs_location_proxy.py → exasol/analytics/query_handler/context/proxy/bucketfs_location_proxy.py

@@ -1,6 +1,6 @@
 import logging
 
-from exasol_advanced_analytics_framework.query_handler.context.proxy.object_proxy import ObjectProxy
+from exasol.analytics.query_handler.context.proxy.object_proxy import ObjectProxy
 import exasol.bucketfs as bfs
 
 LOGGER = logging.getLogger(__file__)

exasol_advanced_analytics_framework/query_handler/context/proxy/db_object_name_proxy.py → exasol/analytics/query_handler/context/proxy/db_object_name_proxy.py

@@ -1,11 +1,12 @@
 from abc import abstractmethod
 from typing import TypeVar, Generic
 
-from exasol_data_science_utils_python.schema.dbobject_name import DBObjectName
-from exasol_data_science_utils_python.utils.repr_generation_for_object import generate_repr_for_object
+from exasol.analytics.schema import DBObjectName
 
-from exasol_advanced_analytics_framework.query_handler.context.proxy.object_proxy import ObjectProxy
-from exasol_advanced_analytics_framework.query_handler.query.query import Query
+from exasol.analytics.utils.repr_generation_for_object import generate_repr_for_object
+
+from exasol.analytics.query_handler.context.proxy.object_proxy import ObjectProxy
+from exasol.analytics.query_handler.query.interface import Query
 
 NameType = TypeVar('NameType', bound=DBObjectName)
 

exasol_advanced_analytics_framework/query_handler/context/proxy/db_object_name_with_schema_proxy.py → exasol/analytics/query_handler/context/proxy/db_object_name_with_schema_proxy.py

@@ -1,9 +1,11 @@
 from typing import TypeVar, Generic
 
-from exasol_data_science_utils_python.schema.dbobject_name_with_schema import DBObjectNameWithSchema
-from exasol_data_science_utils_python.schema.schema_name import SchemaName
+from exasol.analytics.schema import (
+    DBObjectNameWithSchema,
+    SchemaName,
+)
 
-from exasol_advanced_analytics_framework.query_handler.context.proxy.db_object_name_proxy import DBObjectNameProxy
+from exasol.analytics.query_handler.context.proxy.db_object_name_proxy import DBObjectNameProxy
 
 NameType = TypeVar('NameType', bound=DBObjectNameWithSchema)
 

exasol_advanced_analytics_framework/query_handler/context/proxy/drop_udf_query.py → exasol/analytics/query_handler/context/proxy/drop_udf_query.py

@@ -1,6 +1,7 @@
-from exasol_data_science_utils_python.schema.udf_name import UDFName
+from exasol.analytics.schema import UDFName
 
-from exasol_advanced_analytics_framework.query_handler.query.drop_query import DropQuery
+
+from exasol.analytics.query_handler.query.drop.interface import DropQuery
 
 
 class DropUDFQuery(DropQuery):

exasol_advanced_analytics_framework/query_handler/context/proxy/table_like_name_proxy.py → exasol/analytics/query_handler/context/proxy/table_like_name_proxy.py

@@ -1,9 +1,9 @@
 from typing import Generic, TypeVar
 
-from exasol_data_science_utils_python.schema.table_like_name import TableLikeName
+from exasol.analytics.schema import TableLikeName
 
-from exasol_advanced_analytics_framework.query_handler.context.proxy.db_object_name_with_schema_proxy import \
-    DBObjectNameWithSchemaProxy
+
+from exasol.analytics.query_handler.context.proxy.db_object_name_with_schema_proxy import     DBObjectNameWithSchemaProxy
 
 NameType = TypeVar('NameType', bound=TableLikeName)
 

exasol/analytics/query_handler/context/proxy/table_name.py

@@ -0,0 +1,15 @@
+from exasol.analytics.schema import TableName
+
+
+from exasol.analytics.query_handler.context.proxy.table_like_name_proxy import TableLikeNameProxy
+from exasol.analytics.query_handler.query.drop.table import DropTableQuery
+from exasol.analytics.query_handler.query.interface import Query
+
+
+class TableNameProxy(TableLikeNameProxy[TableName], TableName):
+
+    def __init__(self, table_like_name: TableName, global_counter_value: int):
+        super().__init__(table_like_name, global_counter_value)
+
+    def get_cleanup_query(self) -> Query:
+        return DropTableQuery(self._db_object_name)

exasol/analytics/query_handler/context/proxy/udf_name.py

@@ -0,0 +1,15 @@
+from exasol.analytics.schema import UDFName
+
+
+from exasol.analytics.query_handler.context.proxy.db_object_name_with_schema_proxy import     DBObjectNameWithSchemaProxy
+from exasol.analytics.query_handler.context.proxy.drop_udf_query import DropUDFQuery
+from exasol.analytics.query_handler.query.interface import Query
+
+
+class UDFNameProxy(DBObjectNameWithSchemaProxy[UDFName], UDFName):
+
+    def get_cleanup_query(self) -> Query:
+        return DropUDFQuery(self._db_object_name)
+
+    def __init__(self, script_name: UDFName, global_counter_value: int):
+        super().__init__(script_name, global_counter_value)

exasol/analytics/query_handler/context/proxy/view_name_proxy.py

@@ -0,0 +1,15 @@
+from exasol.analytics.schema import ViewName
+
+
+from exasol.analytics.query_handler.context.proxy.table_like_name_proxy import TableLikeNameProxy
+from exasol.analytics.query_handler.query.drop.view import DropViewQuery
+from exasol.analytics.query_handler.query.interface import Query
+
+
+class ViewNameProxy(TableLikeNameProxy[ViewName], ViewName):
+
+    def __init__(self, table_like_name: ViewName, global_counter_value: int):
+        super().__init__(table_like_name, global_counter_value)
+
+    def get_cleanup_query(self) -> Query:
+        return DropViewQuery(self._db_object_name)

exasol_advanced_analytics_framework/query_handler/context/query_handler_context.py → exasol/analytics/query_handler/context/query_handler_context.py

@@ -1,13 +1,14 @@
 import abc
 from abc import ABC
 
-from exasol_data_science_utils_python.schema.table_name import TableName
-from exasol_data_science_utils_python.schema.udf_name import UDFName
-from exasol_data_science_utils_python.schema.view_name import ViewName
+from exasol.analytics.schema import (
+    TableName,
+    UDFName,
+    ViewName,
+)
 
-from exasol_advanced_analytics_framework.query_handler.context.connection_name import ConnectionName
-from exasol_advanced_analytics_framework.query_handler.context.proxy.bucketfs_location_proxy import \
-    BucketFSLocationProxy
+from exasol.analytics.query_handler.context.connection_name import ConnectionName
+from exasol.analytics.query_handler.context.proxy.bucketfs_location_proxy import     BucketFSLocationProxy
 
 
 class QueryHandlerContext(ABC):

exasol_advanced_analytics_framework/query_handler/context/scope_query_handler_context.py → exasol/analytics/query_handler/context/scope.py

@@ -1,8 +1,8 @@
 import enum
 from abc import abstractmethod, ABC
 
-from exasol_advanced_analytics_framework.query_handler.context.query_handler_context import QueryHandlerContext
-from exasol_advanced_analytics_framework.query_handler.context.proxy.object_proxy import ObjectProxy
+from exasol.analytics.query_handler.context.query_handler_context import QueryHandlerContext
+from exasol.analytics.query_handler.context.proxy.object_proxy import ObjectProxy
 
 
 class Connection(ABC):

exasol_advanced_analytics_framework/query_handler/context/top_level_query_handler_context.py → exasol/analytics/query_handler/context/top_level_query_handler_context.py

@@ -4,26 +4,26 @@
 from typing import Set, List, Callable
 
 import exasol.bucketfs as bfs
-from exasol_data_science_utils_python.schema.schema_name import SchemaName
-from exasol_data_science_utils_python.schema.table_name import TableName
-from exasol_data_science_utils_python.schema.table_name_builder import TableNameBuilder
-from exasol_data_science_utils_python.schema.udf_name import UDFName
-from exasol_data_science_utils_python.schema.udf_name_builder import UDFNameBuilder
-from exasol_data_science_utils_python.schema.view_name import ViewName
-from exasol_data_science_utils_python.schema.view_name_builder import ViewNameBuilder
-
-from exasol_advanced_analytics_framework.query_handler.context.connection_name_proxy import ConnectionNameProxy
-from exasol_advanced_analytics_framework.query_handler.context.connection_name import ConnectionName, ConnectionNameImpl
-from exasol_advanced_analytics_framework.query_handler.context.proxy.bucketfs_location_proxy import \
-    BucketFSLocationProxy
-from exasol_advanced_analytics_framework.query_handler.context.proxy.db_object_name_proxy import DBObjectNameProxy
-from exasol_advanced_analytics_framework.query_handler.context.proxy.object_proxy import ObjectProxy
-from exasol_advanced_analytics_framework.query_handler.context.proxy.table_name_proxy import TableNameProxy
-from exasol_advanced_analytics_framework.query_handler.context.proxy.udf_name_proxy import UDFNameProxy
-from exasol_advanced_analytics_framework.query_handler.context.proxy.view_name_proxy import ViewNameProxy
-from exasol_advanced_analytics_framework.query_handler.context.scope_query_handler_context import \
-    ScopeQueryHandlerContext, Connection
-from exasol_advanced_analytics_framework.query_handler.query.query import Query
+from exasol.analytics.schema import (
+    SchemaName,
+    TableNameBuilder,
+    UDFNameBuilder,
+    UDFName,
+    ViewNameBuilder,
+    TableName,
+    ViewName,
+)
+
+from exasol.analytics.query_handler.context.connection_name_proxy import ConnectionNameProxy
+from exasol.analytics.query_handler.context.connection_name import ConnectionName, ConnectionNameImpl
+from exasol.analytics.query_handler.context.proxy.bucketfs_location_proxy import     BucketFSLocationProxy
+from exasol.analytics.query_handler.context.proxy.db_object_name_proxy import DBObjectNameProxy
+from exasol.analytics.query_handler.context.proxy.object_proxy import ObjectProxy
+from exasol.analytics.query_handler.context.proxy.table_name import TableNameProxy
+from exasol.analytics.query_handler.context.proxy.udf_name import UDFNameProxy
+from exasol.analytics.query_handler.context.proxy.view_name_proxy import ViewNameProxy
+from exasol.analytics.query_handler.context.scope import     ScopeQueryHandlerContext, Connection
+from exasol.analytics.query_handler.query.interface import Query
 
 
 class TemporaryObjectCounter:

exasol_advanced_analytics_framework/deployment/aaf_exasol_lua_script_generator.py → exasol/analytics/query_handler/deployment/aaf_exasol_lua_script_generator.py

@@ -1,14 +1,14 @@
 import importlib_resources
 
-from exasol_advanced_analytics_framework.deployment import constants
-from exasol_advanced_analytics_framework.deployment.exasol_lua_script_generator import ExasolLuaScriptGenerator
-from exasol_advanced_analytics_framework.deployment.jinja_template_location import JinjaTemplateLocation
-from exasol_advanced_analytics_framework.deployment.lua_script_bundle import LuaScriptBundle, logger
+from exasol.analytics.query_handler.deployment import constants
+from exasol.analytics.query_handler.deployment.exasol_lua_script_generator import ExasolLuaScriptGenerator
+from exasol.analytics.query_handler.deployment.jinja_template_location import JinjaTemplateLocation
+from exasol.analytics.query_handler.deployment.lua_script_bundle import LuaScriptBundle, logger
 
 
 def get_aaf_query_loop_lua_script_generator() -> ExasolLuaScriptGenerator:
     base_dir = importlib_resources.files(
-        constants.BASE_DIR)
+        constants.BASE_PACKAGE)
     lua_src_dir = base_dir / "lua" / "src"
     lua_source_files = [
         lua_src_dir.joinpath("query_handler_runner_main.lua"),
@@ -25,7 +25,7 @@ def get_aaf_query_loop_lua_script_generator() -> ExasolLuaScriptGenerator:
         "MessageExpander"
     ]
     jinja_template_location = JinjaTemplateLocation(
-        package_name=constants.BASE_DIR,
+        package_name=constants.BASE_PACKAGE,
         package_path=constants.TEMPLATES_DIR,
         template_file_name=constants.LUA_SCRIPT_TEMPLATE)
     generator = ExasolLuaScriptGenerator(

exasol_advanced_analytics_framework/deployment/constants.py → exasol/analytics/query_handler/deployment/constants.py

@@ -1,13 +1,15 @@
 import pathlib
 from importlib_resources import files
 
-BASE_DIR = "exasol_advanced_analytics_framework"
+
+BASE_PACKAGE = "exasol.analytics"
+BASE_DIR = BASE_PACKAGE.replace(".", "/")
 TEMPLATES_DIR = pathlib.Path("resources", "templates")
 OUTPUTS_DIR = pathlib.Path("resources", "outputs")
-SOURCE_DIR = files(f"{BASE_DIR}.udf_framework")
+SOURCE_DIR = files(f"{BASE_PACKAGE}.query_handler.udf.runner")
 
 UDF_CALL_TEMPLATES = {
-    "query_handler_runner_udf_call.py": "create_query_handler.jinja.sql"
+    "call_udf.py": "create_query_handler.jinja.sql"
 }
 LUA_SCRIPT_TEMPLATE = "create_query_loop.jinja.sql"
 LUA_SCRIPT_OUTPUT = pathlib.Path(BASE_DIR, OUTPUTS_DIR, "create_query_loop.sql")

exasol_advanced_analytics_framework/deploy.py → exasol/analytics/query_handler/deployment/deploy.py

@@ -1,10 +1,10 @@
 import logging
 import click
-from exasol_advanced_analytics_framework.slc import (
+from exasol.analytics.query_handler.deployment.slc import (
     SLC_FILE_NAME,
     SLC_URL_FORMATTER,
 )
-from exasol_advanced_analytics_framework.deployment import (
+from exasol.analytics.query_handler.deployment import (
     scripts_deployer_cli,
 )
 from exasol.python_extension_common.deployment.language_container_deployer_cli import (

exasol_advanced_analytics_framework/deployment/exasol_lua_script_generator.py → exasol/analytics/query_handler/deployment/exasol_lua_script_generator.py

@@ -3,7 +3,7 @@
 
 from jinja2 import Template
 
-from exasol_advanced_analytics_framework.deployment.lua_script_bundle import LuaScriptBundle
+from exasol.analytics.query_handler.deployment.lua_script_bundle import LuaScriptBundle
 
 
 class ExasolLuaScriptGenerator: