Add support for virtualCleanup decorator (delphix#400)

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 3.1.0
+current_version = 3.2.0.dev0
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\.(?P<release>[a-z]+)(?P<dev>\d+))?

README-dev.md

@@ -99,6 +99,8 @@ If you want to bump the major/minor/patch version, run `bumpversion [major|minor
 
 If you want to get rid of the dev label (bump from `1.1.0.dev7` to `1.1.0`), run `bumpversion release`.
 
+Note: After bumpversion the tools unit test will need to be manually updated to test for the new version.
+
 ## Testing
 
 Currently, there are three types of SDK testing: unit, manual, and functional (blackbox).
@@ -122,7 +124,9 @@ all the standard workflows. The same workflows will be exercised by functional (
 ### Functional (blackbox) testing
 To run blackbox tests, follow these steps: 
 1. Push your code to a branch in the forked repository on Github. Let's say the branch is called `my-feature` in repository called `<username>/virtualization-sdk`.
-2. Navigate to the app-gate directory and start tests using `git blackbox`. For the guide on which test suite to use,
+2. If you bumped the version (one of major, minor, or micro, not the dev version part), then QA will have to createa a new branch (qa-appdata-toolkits branch sdk-3-2-0 for example with version 3.2.0) and update their map before you can run the blackbox tests:
+* automation/regression/BlackBox/blackbox/appdata/virtualization_sdk/dvp_settings.py
+3. Navigate to the app-gate directory and start tests using `git blackbox`. For the guide on which test suite to use,
 see the next sections.
 
 At a minimum, each pull request should pass `appdata_python_samples` and `appdata_basic` tests with a direct or staged plugin.

common/setup.py

@@ -4,7 +4,7 @@
 PYTHON_SRC = 'src/main/python'
 
 install_requires = [
-  "dvp-api == 1.5.0",
+  "dvp-api == 1.6.0dev2",
 ]
 
 with open(os.path.join(PYTHON_SRC, 'dlpx/virtualization/common/VERSION')) as version_file:

common/src/main/python/dlpx/virtualization/common/VERSION

@@ -1 +1 @@
-3.1.0
+3.2.0.dev0

docs/docs/References/Decorators.md

@@ -38,6 +38,7 @@ Plugin Operation | Decorator
 [Virtual Source Initialize](Plugin_Operations.md#virtual-source-initialize) | `@plugin.virtual.initialize()`
 [Virtual Source Unconfigure](Plugin_Operations.md#virtual-source-unconfigure) | `@plugin.virtual.unconfigure()`
 [Virtual Source Reconfigure](Plugin_Operations.md#virtual-source-reconfigure) | `@plugin.virtual.reconfigure()`
+[Virtual Source Cleanup](Plugin_Operations.md#virtual-source-cleanup) | `@plugin.virtual.cleanup()`
 [Virtual Source Start](Plugin_Operations.md#virtual-source-start) | `@plugin.virtual.start()`
 [Virtual Source Stop](Plugin_Operations.md#virtual-source-stop) | `@plugin.virtual.stop()`
 [VirtualSource Pre-Snapshot](Plugin_Operations.md#virtualsource-pre-snapshot) | `@plugin.virtual.pre_snapshot()`

docs/docs/References/Plugin_Operations.md

@@ -25,6 +25,7 @@ Plugin Operation | **Required** | Decorator | Delphix Engine Operations
 [Virtual Source<br/>Configure](#virtual-source-configure) | **Yes** | `virtual.configure()` | [Virtual Source Provision](Workflows.md#virtual-source-provision)<br/>[Virtual Source Refresh](Workflows.md#virtual-source-refresh)
 [Virtual Source<br/>Unconfigure](#virtual-source-unconfigure) | **No** | `virtual.unconfigure()` | [Virtual Source Refresh](Workflows.md#virtual-source-refresh)<br/>[Virtual Source Delete](Workflows.md#virtual-source-delete)
 [Virtual Source<br/>Reconfigure](#virtual-source-reconfigure) | **Yes** | `virtual.reconfigure()` | [Virtual Source Rollback](Workflows.md#virtual-source-rollback)<br/>[Virtual Source Enable](Workflows.md#virtual-source-enable)
+[Virtual Source<br/>Cleanup](#virtual-source-cleanup) | **No** | `virtual.cleanup()` | [Virtual Source Delete](Workflows.md#virtual-source-delete)
 [Virtual Source<br/>Start](#virtual-source-start) | **No** | `virtual.start()` | [Virtual Source Start](Workflows.md#virtual-source-start)
 [Virtual Source<br/>Stop](#virtual-source-stop) | **No** | `virtual.stop()` | [Virtual Source Stop](Workflows.md#virtual-source-stop)
 [Virtual Source<br/>Pre-Snapshot](#virtual-source-pre-snapshot) | **No** | `virtual.pre_snapshot()` | [Virtual Source Snapshot](Workflows.md#virtual-source-snapshot)
@@ -823,6 +824,50 @@ def reconfigure(virtual_source, repository, source_config, snapshot):
 }
 ```
 
+## Virtual Source Cleanup
+
+Intended to allow a final cleanup during a delete operation, unlike unconfigure which can be used to signal a temporary dissassociation with a database.
+
+Cleanup is called during the delete flow after unconfigure.
+
+### Required / Optional
+**Optional.**
+
+### Delphix Engine Operations
+
+* [Virtual Source Delete](Workflows.md#virtual-source-delete)
+
+### Signature
+
+`def cleanup(virtual_source, repository, source_config)`
+
+### Decorator
+
+`virtual.cleanup()`
+
+### Arguments
+
+Argument | Type | Description
+-------- | ---- | -----------
+virtual_source | [VirtualSource](Classes.md#virtualsource) | The source associated with this operation.
+repository | [RepositoryDefinition](Schemas_and_Autogenerated_Classes.md#repositorydefinition-class) | The repository associated with this source.
+source_config | [SourceConfigDefinition](Schemas_and_Autogenerated_Classes.md#sourceconfigdefinition-class) | The source config associated with this source.
+
+### Returns
+None
+
+### Example
+
+```python
+from dlpx.virtualization.platform import Plugin
+
+plugin = Plugin()
+
+@plugin.virtual.cleanup()
+def cleanup(virtual_source, repository, source_config):
+  pass
+```
+
 ## Virtual Source Start
 
 Executed whenever the data should be placed in a "running" state.

dvp/src/main/python/dlpx/virtualization/VERSION

@@ -1 +1 @@
-3.1.0
+3.2.0.dev0

libs/setup.py

@@ -7,7 +7,7 @@
     version = version_file.read().strip()
 
 install_requires = [
-  "dvp-api == 1.5.0",
+  "dvp-api == 1.6.0dev2",
   "dvp-common == {}".format(version)
 ]
 

libs/src/main/python/dlpx/virtualization/libs/VERSION

@@ -1 +1 @@
-3.1.0
+3.2.0.dev0

platform/setup.py

@@ -7,7 +7,7 @@
     version = version_file.read().strip()
 
 install_requires = [
-  "dvp-api == 1.5.0",
+  "dvp-api == 1.6.0dev2",
   "dvp-common == {}".format(version),
   "enum34;python_version < '3.4'",
 ]

platform/src/main/python/dlpx/virtualization/platform/VERSION

@@ -1 +1 @@
-3.1.0
+3.2.0.dev0

platform/src/main/python/dlpx/virtualization/platform/_virtual.py

@@ -25,6 +25,7 @@ class VirtualOperations(object):
     def __init__(self):
         self.configure_impl = None
         self.unconfigure_impl = None
+        self.cleanup_impl = None
         self.reconfigure_impl = None
         self.start_impl = None
         self.stop_impl = None
@@ -54,6 +55,16 @@ def unconfigure_decorator(unconfigure_impl):
 
         return unconfigure_decorator
 
+    def cleanup(self):
+        def cleanup_decorator(cleanup_impl):
+            if self.cleanup_impl:
+                raise OperationAlreadyDefinedError(Op.VIRTUAL_CLEANUP)
+            self.cleanup_impl = v.check_function(cleanup_impl,
+                                                     Op.VIRTUAL_CLEANUP)
+            return cleanup_impl
+
+        return cleanup_decorator
+
     def reconfigure(self):
         def reconfigure_decorator(reconfigure_impl):
             if self.reconfigure_impl:
@@ -255,6 +266,58 @@ def _internal_unconfigure(self, request):
             platform_pb2.UnconfigureResult())
         return unconfigure_response
 
+    def _internal_cleanup(self, request):
+        """Cleanup operation wrapper.
+
+        Executed when deleting an existing virtual source.
+        This plugin operation is run after unconfigure.
+
+        Args:
+          request (VirtualCleanupRequest): Cleanup operation arguments.
+
+        Returns:
+          VirtualCleanupResponse: A response containing VirtualCleanupResult
+           if successful or PluginErrorResult in case of an error.
+        """
+        # Reasoning for method imports are in this file's docstring.
+        from generated.definitions import VirtualSourceDefinition
+        from generated.definitions import RepositoryDefinition
+        from generated.definitions import SourceConfigDefinition
+
+        #
+        # While virtual.cleanup() is not a required operation, this should
+        # not be called if it wasn't implemented.
+        #
+        if not self.cleanup_impl:
+            raise OperationNotDefinedError(Op.VIRTUAL_CLEANUP)
+
+        virtual_source_definition = VirtualSourceDefinition.from_dict(
+            json.loads(request.virtual_source.parameters.json))
+        mounts = [
+            VirtualOperations._from_protobuf_single_subset_mount(m)
+            for m in request.virtual_source.mounts
+        ]
+
+        virtual_source = VirtualSource(guid=request.virtual_source.guid,
+                                       connection=RemoteConnection.from_proto(
+                                           request.virtual_source.connection),
+                                       parameters=virtual_source_definition,
+                                       mounts=mounts)
+
+        repository = RepositoryDefinition.from_dict(
+            json.loads(request.repository.parameters.json))
+        source_config = SourceConfigDefinition.from_dict(
+            json.loads(request.source_config.parameters.json))
+
+        self.cleanup_impl(repository=repository,
+                              source_config=source_config,
+                              virtual_source=virtual_source)
+
+        virtual_cleanup_response = platform_pb2.VirtualCleanupResponse()
+        virtual_cleanup_response.return_value.CopyFrom(
+            platform_pb2.VirtualCleanupResult())
+        return virtual_cleanup_response
+
     def _internal_reconfigure(self, request):
         """Reconfigure operation wrapper.
 

platform/src/main/python/dlpx/virtualization/platform/operation.py

@@ -21,6 +21,7 @@ class Operation(Enum):
     VIRTUAL_CONFIGURE = 'virtual.configure()'
     VIRTUAL_UNCONFIGURE = 'virtual.unconfigure()'
     VIRTUAL_RECONFIGURE = 'virtual.reconfigure()'
+    VIRTUAL_CLEANUP = 'virtual.cleanup()'
     VIRTUAL_START = 'virtual.start()'
     VIRTUAL_STOP = 'virtual.stop()'
     VIRTUAL_PRE_SNAPSHOT = 'virtual.pre_snapshot()'

platform/src/test/python/dlpx/virtualization/test_plugin.py

@@ -559,6 +559,32 @@ def virtual_reconfigure_impl(virtual_source, repository, source_config,
             " type 'unicode' but should be of class 'dlpx.virtualization."
             "fake_generated_definitions.SourceConfigDefinition'.")
 
+    @staticmethod
+    def test_virtual_cleanup(my_plugin, virtual_source, repository,
+                                 source_config):
+        @my_plugin.virtual.cleanup()
+        def virtual_cleanup_impl(virtual_source, repository,
+                                     source_config):
+            TestPlugin.assert_plugin_args(virtual_source=virtual_source,
+                                          repository=repository,
+                                          source_config=source_config)
+            return
+
+        virtual_cleanup_request = platform_pb2.VirtualCleanupRequest()
+        TestPlugin.setup_request(request=virtual_cleanup_request,
+                                 virtual_source=virtual_source,
+                                 repository=repository,
+                                 source_config=source_config)
+
+        expected_result = platform_pb2.VirtualCleanupResult()
+
+        virtual_cleanup_response = my_plugin.virtual._internal_cleanup(
+            virtual_cleanup_request)
+
+        # Check that the response's oneof is set to return_value and not error
+        assert virtual_cleanup_response.WhichOneof('result') == 'return_value'
+        assert virtual_cleanup_response.return_value == expected_result
+
     @staticmethod
     def test_virtual_start(my_plugin, virtual_source, repository,
                            source_config):

tools/src/main/python/dlpx/virtualization/_internal/VERSION

@@ -1 +1 @@
-3.1.0
+3.2.0.dev0

tools/src/main/python/dlpx/virtualization/_internal/plugin_importer.py

@@ -330,6 +330,8 @@ def _prepare_manifest(entry_point, module_content):
         bool(plugin_object.virtual.unconfigure_impl),
         'hasVirtualReconfigure':
         bool(plugin_object.virtual.reconfigure_impl),
+        'hasVirtualCleanup':
+        bool(plugin_object.virtual.cleanup_impl),
         'hasVirtualStart':
         bool(plugin_object.virtual.start_impl),
         'hasVirtualStop':

tools/src/main/python/dlpx/virtualization/_internal/settings.cfg

@@ -20,7 +20,7 @@
 # versions in those packages until they are shipped out of band.
 #
 [General]
-engine_api_version = 1.11.6
+engine_api_version = 1.11.11
 distribution_name = dvp-tools
 package_author = Delphix
 namespace_package = dlpx

tools/src/main/python/dlpx/virtualization/_internal/validation_schemas/plugin_importer.yaml

@@ -52,6 +52,10 @@ EXPECTED_STAGED_ARGS_BY_OP:
       - repository
       - source_config
       - snapshot
+    cleanup_impl:
+      - virtual_source
+      - repository
+      - source_config
     start_impl:
       - virtual_source
       - repository
@@ -112,6 +116,10 @@ EXPECTED_DIRECT_ARGS_BY_OP:
       - repository
       - source_config
       - snapshot
+    cleanup_impl:
+      - virtual_source
+      - repository
+      - source_config
     start_impl:
       - virtual_source
       - repository

tools/src/test/python/dlpx/virtualization/_internal/conftest.py

@@ -404,6 +404,7 @@ def mount_specification(virtual_source, repository):
     virtual.mount_specification_impl = mount_specification
     virtual.status_impl = None
     virtual.initialize_impl = None
+    virtual.cleanup_impl = None
 
     return virtual
 
@@ -447,7 +448,8 @@ def plugin_manifest(upgrade_operation):
         'hasVirtualMountSpecification': True,
         'hasVirtualStatus': False,
         'hasInitialize': False,
-        'migrationIdList': upgrade_operation.migration_id_list
+        'migrationIdList': upgrade_operation.migration_id_list,
+        'hasVirtualCleanup': False,
     }
     return manifest
 
@@ -691,7 +693,7 @@ def artifact_content(engine_api, virtual_source_definition,
 
 @pytest.fixture
 def engine_api():
-    return {'type': 'APIVersion', 'major': 1, 'minor': 11, 'micro': 6}
+    return {'type': 'APIVersion', 'major': 1, 'minor': 11, 'micro': 11}
 
 
 @pytest.fixture

tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/multiple_warnings.py

@@ -75,6 +75,11 @@ def unconfigure(repository, source_config, virtual_source):
     pass
 
 
+@vfiles.virtual.cleanup()
+def cleanup(repository, source_config, virtual_source, bad_arg):
+    pass
+
+
 @vfiles.upgrade.repository('2019.10.30')
 def repo_upgrade(old_repository):
     return old_repository

tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/successful.py

@@ -81,6 +81,11 @@ def unconfigure(repository, source_config, virtual_source):
     pass
 
 
+@direct.virtual.cleanup()
+def cleanup(repository, source_config, virtual_source):
+    pass
+
+
 @direct.upgrade.repository('1.3', MigrationType.LUA)
 def repo_upgrade(old_repository):
     return old_repository

tools/src/test/python/dlpx/virtualization/_internal/fake_plugin/direct/upgrade_warnings.py

@@ -74,6 +74,11 @@ def unconfigure(repository, source_config, virtual_source):
     pass
 
 
+@direct.virtual.cleanup()
+def cleanup(repository, source_config, virtual_source):
+    pass
+
+
 @direct.upgrade.repository('2019.11.20')
 def repo_upgrade(old_repository):
     return old_repository

tools/src/test/python/dlpx/virtualization/_internal/test_package_util.py

@@ -10,22 +10,22 @@
 class TestPackageUtil:
     @staticmethod
     def test_get_version():
-        assert package_util.get_version() == '3.1.0'
+        assert package_util.get_version() == '3.2.0.dev0'
 
     @staticmethod
     def test_get_virtualization_api_version():
-        assert package_util.get_virtualization_api_version() == '1.5.0'
+        assert package_util.get_virtualization_api_version() == '1.6.0'
 
     @staticmethod
     def test_get_engine_api_version():
-        assert package_util.get_engine_api_version_from_settings() == '1.11.6'
+        assert package_util.get_engine_api_version_from_settings() == '1.11.11'
 
     @staticmethod
     def test_get_build_api_version_json():
         build_api_version = {
             'type': 'APIVersion',
             'major': 1,
-            'minor': 5,
+            'minor': 6,
             'micro': 0
         }
         assert package_util.get_build_api_version() == build_api_version
@@ -36,7 +36,7 @@ def test_get_engine_api_version_json():
             'type': 'APIVersion',
             'major': 1,
             'minor': 11,
-            'micro': 6
+            'micro': 11
         }
         assert package_util.get_engine_api_version() == engine_api_version