From 84319ccf74f0d7c37fbb1b5e4259e1a2579f4ac1 Mon Sep 17 00:00:00 2001 From: Kurt Hutchison Date: Mon, 4 Oct 2021 10:24:19 -0700 Subject: [PATCH] Add support for virtualCleanup decorator (#400) --- .bumpversion.cfg | 2 +- README-dev.md | 6 +- common/setup.py | 2 +- .../python/dlpx/virtualization/common/VERSION | 2 +- docs/docs/References/Decorators.md | 1 + docs/docs/References/Plugin_Operations.md | 45 +++++++++++++ .../main/python/dlpx/virtualization/VERSION | 2 +- libs/setup.py | 2 +- .../python/dlpx/virtualization/libs/VERSION | 2 +- platform/setup.py | 2 +- .../dlpx/virtualization/platform/VERSION | 2 +- .../dlpx/virtualization/platform/_virtual.py | 63 +++++++++++++++++++ .../dlpx/virtualization/platform/operation.py | 1 + .../python/dlpx/virtualization/test_plugin.py | 26 ++++++++ .../dlpx/virtualization/_internal/VERSION | 2 +- .../_internal/plugin_importer.py | 2 + .../virtualization/_internal/settings.cfg | 2 +- .../validation_schemas/plugin_importer.yaml | 8 +++ .../dlpx/virtualization/_internal/conftest.py | 6 +- .../fake_plugin/direct/multiple_warnings.py | 5 ++ .../fake_plugin/direct/successful.py | 5 ++ .../fake_plugin/direct/upgrade_warnings.py | 5 ++ .../_internal/test_package_util.py | 10 +-- .../_internal/test_plugin_importer.py | 4 +- 24 files changed, 188 insertions(+), 19 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 8213c309..6af00a01 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 3.1.0 +current_version = 3.2.0.dev0 commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\.(?P[a-z]+)(?P\d+))? diff --git a/README-dev.md b/README-dev.md index 34540f17..65f3ce74 100644 --- a/README-dev.md +++ b/README-dev.md @@ -99,6 +99,8 @@ If you want to bump the major/minor/patch version, run `bumpversion [major|minor If you want to get rid of the dev label (bump from `1.1.0.dev7` to `1.1.0`), run `bumpversion release`. +Note: After bumpversion the tools unit test will need to be manually updated to test for the new version. + ## Testing Currently, there are three types of SDK testing: unit, manual, and functional (blackbox). @@ -122,7 +124,9 @@ all the standard workflows. The same workflows will be exercised by functional ( ### Functional (blackbox) testing To run blackbox tests, follow these steps: 1. Push your code to a branch in the forked repository on Github. Let's say the branch is called `my-feature` in repository called `/virtualization-sdk`. -2. Navigate to the app-gate directory and start tests using `git blackbox`. For the guide on which test suite to use, +2. If you bumped the version (one of major, minor, or micro, not the dev version part), then QA will have to createa a new branch (qa-appdata-toolkits branch sdk-3-2-0 for example with version 3.2.0) and update their map before you can run the blackbox tests: +* automation/regression/BlackBox/blackbox/appdata/virtualization_sdk/dvp_settings.py +3. Navigate to the app-gate directory and start tests using `git blackbox`. For the guide on which test suite to use, see the next sections. At a minimum, each pull request should pass `appdata_python_samples` and `appdata_basic` tests with a direct or staged plugin. diff --git a/common/setup.py b/common/setup.py index 56d61c75..4e6b897e 100644 --- a/common/setup.py +++ b/common/setup.py @@ -4,7 +4,7 @@ PYTHON_SRC = 'src/main/python' install_requires = [ - "dvp-api == 1.5.0", + "dvp-api == 1.6.0dev2", ] with open(os.path.join(PYTHON_SRC, 'dlpx/virtualization/common/VERSION')) as version_file: diff --git a/common/src/main/python/dlpx/virtualization/common/VERSION b/common/src/main/python/dlpx/virtualization/common/VERSION index fd2a0186..745e2a57 100644 --- a/common/src/main/python/dlpx/virtualization/common/VERSION +++ b/common/src/main/python/dlpx/virtualization/common/VERSION @@ -1 +1 @@ -3.1.0 +3.2.0.dev0 diff --git a/docs/docs/References/Decorators.md b/docs/docs/References/Decorators.md index 038cf01c..8dbf9c56 100644 --- a/docs/docs/References/Decorators.md +++ b/docs/docs/References/Decorators.md @@ -38,6 +38,7 @@ Plugin Operation | Decorator [Virtual Source Initialize](Plugin_Operations.md#virtual-source-initialize) | `@plugin.virtual.initialize()` [Virtual Source Unconfigure](Plugin_Operations.md#virtual-source-unconfigure) | `@plugin.virtual.unconfigure()` [Virtual Source Reconfigure](Plugin_Operations.md#virtual-source-reconfigure) | `@plugin.virtual.reconfigure()` +[Virtual Source Cleanup](Plugin_Operations.md#virtual-source-cleanup) | `@plugin.virtual.cleanup()` [Virtual Source Start](Plugin_Operations.md#virtual-source-start) | `@plugin.virtual.start()` [Virtual Source Stop](Plugin_Operations.md#virtual-source-stop) | `@plugin.virtual.stop()` [VirtualSource Pre-Snapshot](Plugin_Operations.md#virtualsource-pre-snapshot) | `@plugin.virtual.pre_snapshot()` diff --git a/docs/docs/References/Plugin_Operations.md b/docs/docs/References/Plugin_Operations.md index 9dc1b682..a66e1590 100644 --- a/docs/docs/References/Plugin_Operations.md +++ b/docs/docs/References/Plugin_Operations.md @@ -25,6 +25,7 @@ Plugin Operation | **Required** | Decorator | Delphix Engine Operations [Virtual Source
Configure](#virtual-source-configure) | **Yes** | `virtual.configure()` | [Virtual Source Provision](Workflows.md#virtual-source-provision)
[Virtual Source Refresh](Workflows.md#virtual-source-refresh) [Virtual Source
Unconfigure](#virtual-source-unconfigure) | **No** | `virtual.unconfigure()` | [Virtual Source Refresh](Workflows.md#virtual-source-refresh)
[Virtual Source Delete](Workflows.md#virtual-source-delete) [Virtual Source
Reconfigure](#virtual-source-reconfigure) | **Yes** | `virtual.reconfigure()` | [Virtual Source Rollback](Workflows.md#virtual-source-rollback)
[Virtual Source Enable](Workflows.md#virtual-source-enable) +[Virtual Source
Cleanup](#virtual-source-cleanup) | **No** | `virtual.cleanup()` | [Virtual Source Delete](Workflows.md#virtual-source-delete) [Virtual Source
Start](#virtual-source-start) | **No** | `virtual.start()` | [Virtual Source Start](Workflows.md#virtual-source-start) [Virtual Source
Stop](#virtual-source-stop) | **No** | `virtual.stop()` | [Virtual Source Stop](Workflows.md#virtual-source-stop) [Virtual Source
Pre-Snapshot](#virtual-source-pre-snapshot) | **No** | `virtual.pre_snapshot()` | [Virtual Source Snapshot](Workflows.md#virtual-source-snapshot) @@ -823,6 +824,50 @@ def reconfigure(virtual_source, repository, source_config, snapshot): } ``` +## Virtual Source Cleanup + +Intended to allow a final cleanup during a delete operation, unlike unconfigure which can be used to signal a temporary dissassociation with a database. + +Cleanup is called during the delete flow after unconfigure. + +### Required / Optional +**Optional.** + +### Delphix Engine Operations + +* [Virtual Source Delete](Workflows.md#virtual-source-delete) + +### Signature + +`def cleanup(virtual_source, repository, source_config)` + +### Decorator + +`virtual.cleanup()` + +### Arguments + +Argument | Type | Description +-------- | ---- | ----------- +virtual_source | [VirtualSource](Classes.md#virtualsource) | The source associated with this operation. +repository | [RepositoryDefinition](Schemas_and_Autogenerated_Classes.md#repositorydefinition-class) | The repository associated with this source. +source_config | [SourceConfigDefinition](Schemas_and_Autogenerated_Classes.md#sourceconfigdefinition-class) | The source config associated with this source. + +### Returns +None + +### Example + +```python +from dlpx.virtualization.platform import Plugin + +plugin = Plugin() + +@plugin.virtual.cleanup() +def cleanup(virtual_source, repository, source_config): + pass +``` + ## Virtual Source Start Executed whenever the data should be placed in a "running" state. diff --git a/dvp/src/main/python/dlpx/virtualization/VERSION b/dvp/src/main/python/dlpx/virtualization/VERSION index fd2a0186..745e2a57 100644 --- a/dvp/src/main/python/dlpx/virtualization/VERSION +++ b/dvp/src/main/python/dlpx/virtualization/VERSION @@ -1 +1 @@ -3.1.0 +3.2.0.dev0 diff --git a/libs/setup.py b/libs/setup.py index 907b103d..4090d6be 100644 --- a/libs/setup.py +++ b/libs/setup.py @@ -7,7 +7,7 @@ version = version_file.read().strip() install_requires = [ - "dvp-api == 1.5.0", + "dvp-api == 1.6.0dev2", "dvp-common == {}".format(version) ] diff --git a/libs/src/main/python/dlpx/virtualization/libs/VERSION b/libs/src/main/python/dlpx/virtualization/libs/VERSION index fd2a0186..745e2a57 100644 --- a/libs/src/main/python/dlpx/virtualization/libs/VERSION +++ b/libs/src/main/python/dlpx/virtualization/libs/VERSION @@ -1 +1 @@ -3.1.0 +3.2.0.dev0 diff --git a/platform/setup.py b/platform/setup.py index c9ec3a5d..600fe22f 100644 --- a/platform/setup.py +++ b/platform/setup.py @@ -7,7 +7,7 @@ version = version_file.read().strip() install_requires = [ - "dvp-api == 1.5.0", + "dvp-api == 1.6.0dev2", "dvp-common == {}".format(version), "enum34;python_version < '3.4'", ] diff --git a/platform/src/main/python/dlpx/virtualization/platform/VERSION b/platform/src/main/python/dlpx/virtualization/platform/VERSION index fd2a0186..745e2a57 100644 --- a/platform/src/main/python/dlpx/virtualization/platform/VERSION +++ b/platform/src/main/python/dlpx/virtualization/platform/VERSION @@ -1 +1 @@ -3.1.0 +3.2.0.dev0 diff --git a/platform/src/main/python/dlpx/virtualization/platform/_virtual.py b/platform/src/main/python/dlpx/virtualization/platform/_virtual.py index 2b2318a9..18d9e1f8 100644 --- a/platform/src/main/python/dlpx/virtualization/platform/_virtual.py +++ b/platform/src/main/python/dlpx/virtualization/platform/_virtual.py @@ -25,6 +25,7 @@ class VirtualOperations(object): def __init__(self): self.configure_impl = None self.unconfigure_impl = None + self.cleanup_impl = None self.reconfigure_impl = None self.start_impl = None self.stop_impl = None @@ -54,6 +55,16 @@ def unconfigure_decorator(unconfigure_impl): return unconfigure_decorator + def cleanup(self): + def cleanup_decorator(cleanup_impl): + if self.cleanup_impl: + raise OperationAlreadyDefinedError(Op.VIRTUAL_CLEANUP) + self.cleanup_impl = v.check_function(cleanup_impl, + Op.VIRTUAL_CLEANUP) + return cleanup_impl + + return cleanup_decorator + def reconfigure(self): def reconfigure_decorator(reconfigure_impl): if self.reconfigure_impl: @@ -255,6 +266,58 @@ def _internal_unconfigure(self, request): platform_pb2.UnconfigureResult()) return unconfigure_response + def _internal_cleanup(self, request): + """Cleanup operation wrapper. + + Executed when deleting an existing virtual source. + This plugin operation is run after unconfigure. + + Args: + request (VirtualCleanupRequest): Cleanup operation arguments. + + Returns: + VirtualCleanupResponse: A response containing VirtualCleanupResult + if successful or PluginErrorResult in case of an error. + """ + # Reasoning for method imports are in this file's docstring. + from generated.definitions import VirtualSourceDefinition + from generated.definitions import RepositoryDefinition + from generated.definitions import SourceConfigDefinition + + # + # While virtual.cleanup() is not a required operation, this should + # not be called if it wasn't implemented. + # + if not self.cleanup_impl: + raise OperationNotDefinedError(Op.VIRTUAL_CLEANUP) + + virtual_source_definition = VirtualSourceDefinition.from_dict( + json.loads(request.virtual_source.parameters.json)) + mounts = [ + VirtualOperations._from_protobuf_single_subset_mount(m) + for m in request.virtual_source.mounts + ] + + virtual_source = VirtualSource(guid=request.virtual_source.guid, + connection=RemoteConnection.from_proto( + request.virtual_source.connection), + parameters=virtual_source_definition, + mounts=mounts) + + repository = RepositoryDefinition.from_dict( + json.loads(request.repository.parameters.json)) + source_config = SourceConfigDefinition.from_dict( + json.loads(request.source_config.parameters.json)) + + self.cleanup_impl(repository=repository, + source_config=source_config, + virtual_source=virtual_source) + + virtual_cleanup_response = platform_pb2.VirtualCleanupResponse() + virtual_cleanup_response.return_value.CopyFrom( + platform_pb2.VirtualCleanupResult()) + return virtual_cleanup_response + def _internal_reconfigure(self, request): """Reconfigure operation wrapper. diff --git a/platform/src/main/python/dlpx/virtualization/platform/operation.py b/platform/src/main/python/dlpx/virtualization/platform/operation.py index 12f1fb6d..87a06c31 100644 --- a/platform/src/main/python/dlpx/virtualization/platform/operation.py +++ b/platform/src/main/python/dlpx/virtualization/platform/operation.py @@ -21,6 +21,7 @@ class Operation(Enum): VIRTUAL_CONFIGURE = 'virtual.configure()' VIRTUAL_UNCONFIGURE = 'virtual.unconfigure()' VIRTUAL_RECONFIGURE = 'virtual.reconfigure()' + VIRTUAL_CLEANUP = 'virtual.cleanup()' VIRTUAL_START = 'virtual.start()' VIRTUAL_STOP = 'virtual.stop()' VIRTUAL_PRE_SNAPSHOT = 'virtual.pre_snapshot()' diff --git a/platform/src/test/python/dlpx/virtualization/test_plugin.py b/platform/src/test/python/dlpx/virtualization/test_plugin.py index c5bb2f87..10740fe9 100755 --- a/platform/src/test/python/dlpx/virtualization/test_plugin.py +++ b/platform/src/test/python/dlpx/virtualization/test_plugin.py @@ -559,6 +559,32 @@ def virtual_reconfigure_impl(virtual_source, repository, source_config, " type 'unicode' but should be of class 'dlpx.virtualization." "fake_generated_definitions.SourceConfigDefinition'.") + @staticmethod + def test_virtual_cleanup(my_plugin, virtual_source, repository, + source_config): + @my_plugin.virtual.cleanup() + def virtual_cleanup_impl(virtual_source, repository, + source_config): + TestPlugin.assert_plugin_args(virtual_source=virtual_source, + repository=repository, + source_config=source_config) + return + + virtual_cleanup_request = platform_pb2.VirtualCleanupRequest() + TestPlugin.setup_request(request=virtual_cleanup_request, + virtual_source=virtual_source, + repository=repository, + source_config=source_config) + + expected_result = platform_pb2.VirtualCleanupResult() + + virtual_cleanup_response = my_plugin.virtual._internal_cleanup( + virtual_cleanup_request) + + # Check that the response's oneof is set to return_value and not error + assert virtual_cleanup_response.WhichOneof('result') == 'return_value' + assert virtual_cleanup_response.return_value == expected_result + @staticmethod def test_virtual_start(my_plugin, virtual_source, repository, source_config): diff --git a/tools/src/main/python/dlpx/virtualization/_internal/VERSION b/tools/src/main/python/dlpx/virtualization/_internal/VERSION index fd2a0186..745e2a57 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/VERSION +++ b/tools/src/main/python/dlpx/virtualization/_internal/VERSION @@ -1 +1 @@ -3.1.0 +3.2.0.dev0 diff --git a/tools/src/main/python/dlpx/virtualization/_internal/plugin_importer.py b/tools/src/main/python/dlpx/virtualization/_internal/plugin_importer.py index f3ad2119..501708f0 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/plugin_importer.py +++ b/tools/src/main/python/dlpx/virtualization/_internal/plugin_importer.py @@ -330,6 +330,8 @@ def _prepare_manifest(entry_point, module_content): bool(plugin_object.virtual.unconfigure_impl), 'hasVirtualReconfigure': bool(plugin_object.virtual.reconfigure_impl), + 'hasVirtualCleanup': + bool(plugin_object.virtual.cleanup_impl), 'hasVirtualStart': bool(plugin_object.virtual.start_impl), 'hasVirtualStop': diff --git a/tools/src/main/python/dlpx/virtualization/_internal/settings.cfg b/tools/src/main/python/dlpx/virtualization/_internal/settings.cfg index 700afb91..d71b6a5d 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/settings.cfg +++ b/tools/src/main/python/dlpx/virtualization/_internal/settings.cfg @@ -20,7 +20,7 @@ # versions in those packages until they are shipped out of band. # [General] -engine_api_version = 1.11.6 +engine_api_version = 1.11.11 distribution_name = dvp-tools package_author = Delphix namespace_package = dlpx diff --git a/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_importer.yaml b/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_importer.yaml index d769cd62..b04e88fc 100644 --- a/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_importer.yaml +++ b/tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_importer.yaml @@ -52,6 +52,10 @@ EXPECTED_STAGED_ARGS_BY_OP: - repository - source_config - snapshot + cleanup_impl: + - virtual_source + - repository + - source_config start_impl: - virtual_source - repository @@ -112,6 +116,10 @@ EXPECTED_DIRECT_ARGS_BY_OP: - repository - source_config - snapshot + cleanup_impl: + - virtual_source + - repository + - source_config start_impl: - virtual_source - repository diff --git a/tools/src/test/python/dlpx/virtualization/_internal/conftest.py b/tools/src/test/python/dlpx/virtualization/_internal/conftest.py index 39a4460f..3b9a42ee 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/conftest.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/conftest.py @@ -404,6 +404,7 @@ def mount_specification(virtual_source, repository): virtual.mount_specification_impl = mount_specification virtual.status_impl = None virtual.initialize_impl = None + virtual.cleanup_impl = None return virtual @@ -447,7 +448,8 @@ def plugin_manifest(upgrade_operation): 'hasVirtualMountSpecification': True, 'hasVirtualStatus': False, 'hasInitialize': False, - 'migrationIdList': upgrade_operation.migration_id_list + 'migrationIdList': upgrade_operation.migration_id_list, + 'hasVirtualCleanup': False, } return manifest @@ -691,7 +693,7 @@ def artifact_content(engine_api, virtual_source_definition, @pytest.fixture def engine_api(): - return {'type': 'APIVersion', 'major': 1, 'minor': 11, 'micro': 6} + return {'type': 'APIVersion', 'major': 1, 'minor': 11, 'micro': 11} @pytest.fixture diff --git a/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/multiple_warnings.py b/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/multiple_warnings.py index 56f4d299..1ee0d307 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/multiple_warnings.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/multiple_warnings.py @@ -75,6 +75,11 @@ def unconfigure(repository, source_config, virtual_source): pass +@vfiles.virtual.cleanup() +def cleanup(repository, source_config, virtual_source, bad_arg): + pass + + @vfiles.upgrade.repository('2019.10.30') def repo_upgrade(old_repository): return old_repository diff --git a/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/successful.py b/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/successful.py index cb355933..da568732 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/successful.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/successful.py @@ -81,6 +81,11 @@ def unconfigure(repository, source_config, virtual_source): pass +@direct.virtual.cleanup() +def cleanup(repository, source_config, virtual_source): + pass + + @direct.upgrade.repository('1.3', MigrationType.LUA) def repo_upgrade(old_repository): return old_repository diff --git a/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/upgrade_warnings.py b/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/upgrade_warnings.py index 0046fe49..0f018bf2 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/upgrade_warnings.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/upgrade_warnings.py @@ -74,6 +74,11 @@ def unconfigure(repository, source_config, virtual_source): pass +@direct.virtual.cleanup() +def cleanup(repository, source_config, virtual_source): + pass + + @direct.upgrade.repository('2019.11.20') def repo_upgrade(old_repository): return old_repository diff --git a/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py b/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py index 87f63c08..143362b5 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py @@ -10,22 +10,22 @@ class TestPackageUtil: @staticmethod def test_get_version(): - assert package_util.get_version() == '3.1.0' + assert package_util.get_version() == '3.2.0.dev0' @staticmethod def test_get_virtualization_api_version(): - assert package_util.get_virtualization_api_version() == '1.5.0' + assert package_util.get_virtualization_api_version() == '1.6.0' @staticmethod def test_get_engine_api_version(): - assert package_util.get_engine_api_version_from_settings() == '1.11.6' + assert package_util.get_engine_api_version_from_settings() == '1.11.11' @staticmethod def test_get_build_api_version_json(): build_api_version = { 'type': 'APIVersion', 'major': 1, - 'minor': 5, + 'minor': 6, 'micro': 0 } assert package_util.get_build_api_version() == build_api_version @@ -36,7 +36,7 @@ def test_get_engine_api_version_json(): 'type': 'APIVersion', 'major': 1, 'minor': 11, - 'micro': 6 + 'micro': 11 } assert package_util.get_engine_api_version() == engine_api_version diff --git a/tools/src/test/python/dlpx/virtualization/_internal/test_plugin_importer.py b/tools/src/test/python/dlpx/virtualization/_internal/test_plugin_importer.py index a8795765..e54ad838 100644 --- a/tools/src/test/python/dlpx/virtualization/_internal/test_plugin_importer.py +++ b/tools/src/test/python/dlpx/virtualization/_internal/test_plugin_importer.py @@ -137,8 +137,10 @@ def test_successful_validation(mock_file_util, plugin_config_file, ('multiple_warnings:vfiles', 'DIRECT', [ 'Error: Number of arguments do not match in method status', 'Error: Named argument mismatch in method status', + 'Error: Number of arguments do not match in method cleanup', + 'Error: Named argument mismatch in method cleanup', 'Warning: Implementation missing for required method' - ' virtual.reconfigure().', '1 Warning(s). 2 Error(s).' + ' virtual.reconfigure().', '1 Warning(s). 4 Error(s).' ])]) @mock.patch('dlpx.virtualization._internal.file_util.get_src_dir_path') def test_multiple_warnings(mock_file_util, plugin_config_file,