Merge pull request #296 from cloudify-cosmo/CY-802-Openstack-Plugin-V3

Cy 802 openstack plugin v3

.cicd/resource_interface_mappings.py

@@ -0,0 +1,113 @@
+# #######
+# Copyright (c) 2019 Cloudify Platform Ltd. All rights reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from openstack import connect
+
+
+class InterfaceBase(object):
+
+    type_name = None
+    client_name = None
+    get_method_name = None
+    delete_method_name = None
+
+    def __init__(self, resource_id, client_config):
+        self.id = resource_id
+        self.client_config = client_config
+
+    @property
+    def client(self):
+        openstack_connecton = connect(**self.client_config)
+        return getattr(openstack_connecton, self.client_name)
+
+    def get(self):
+        get_method = getattr(self.client, self.get_method_name)
+        return get_method(self.id)
+
+    def delete(self):
+        delete_method = getattr(self.client, self.delete_method_name)
+        return delete_method(self.id)
+
+
+class Router(InterfaceBase):
+
+    type_name = 'cloudify.nodes.openstack.Router'
+    client_name = 'network'
+    get_method_name = 'get_router'
+    delete_method_name = 'delete_router'
+
+
+class Network(InterfaceBase):
+
+    type_name = 'cloudify.nodes.openstack.Network'
+    client_name = 'network'
+    get_method_name = 'get_network'
+    delete_method_name = 'delete_network'
+
+
+class Subnet(InterfaceBase):
+
+    type_name = 'cloudify.nodes.openstack.Subnet'
+    client_name = 'network'
+    get_method_name = 'get_subnet'
+    delete_method_name = 'delete_subnet'
+
+
+class SecurityGroup(InterfaceBase):
+
+    type_name = 'cloudify.nodes.openstack.SecurityGroup'
+    client_name = 'network'
+    get_method_name = 'get_security_group'
+    delete_method_name = 'delete_security_group'
+
+
+class Port(InterfaceBase):
+
+    type_name = 'cloudify.nodes.openstack.Port'
+    client_name = 'network'
+    get_method_name = 'get_port'
+    delete_method_name = 'delete_port'
+
+
+class KeyPair(InterfaceBase):
+
+    type_name = 'cloudify.nodes.openstack.KeyPair'
+    client_name = 'compute'
+    get_method_name = 'get_keypair'
+    delete_method_name = 'delete_keypair'
+
+
+class VolumeType(InterfaceBase):
+
+    type_name = 'cloudify.nodes.openstack.VolumeType'
+    client_name = 'block_storage'
+    get_method_name = 'get_type'
+    delete_method_name = 'delete_type'
+
+
+class Server(InterfaceBase):
+
+    type_name = 'cloudify.nodes.openstack.Server'
+    client_name = 'compute'
+    get_method_name = 'get_server'
+    delete_method_name = 'delete_server'
+
+
+class FloatingIP(InterfaceBase):
+
+    type_name = 'cloudify.nodes.openstack.FloatingIP'
+    client_name = 'network'
+    get_method_name = 'get_ip'
+    delete_method_name = 'delete_ip'

.cicd/test_local.py

@@ -0,0 +1,328 @@
+# #######
+# Copyright (c) 2019 Cloudify Platform Ltd. All rights reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Standard Imports
+from os import getenv
+import time
+import StringIO
+import unittest
+
+# Third party imports
+import openstack
+import requests
+import requests.exceptions
+from fabric.api import settings as fabric_settings, run as fabric_run
+from cloudify.workflows import local
+
+# Local imports
+import resource_interface_mappings
+
+
+IGNORED_LOCAL_WORKFLOW_MODULES = (
+    'worker_installer.tasks',
+    'plugin_installer.tasks',
+    'cloudify_agent.operations',
+    'cloudify_agent.installer.operations',
+)
+
+RETRY_MAX = 10
+RETRY_INT = 1
+
+
+class TestEnvironmentValidationError(Exception):
+    pass
+
+
+class LiveUseCaseTests(unittest.TestCase):
+    """ Test a use case using a "local" Cloudify Workflow.
+
+    Write a blueprint for a particular use case,
+    for example creating a port with allowed address pairs.
+
+    You need the client config in your inputs, e.g.:
+    ```yaml
+      auth_url:
+        type: string
+
+      username:
+        type: string
+
+      password:
+        type: string
+
+      region_name:
+        type: string
+
+      project_name:
+        type: string
+    ```
+
+    To setup your test environment in PyCharm,
+    add the following environment variables:
+
+    ```bash
+    openstack_username=.......;openstack_password=.........;
+    openstack_project_name=.....;openstack_region_name=RegionOne;
+    openstack_auth_url=https://....;=
+    ```
+
+    """
+
+    def setUp(self):
+        super(LiveUseCaseTests, self).setUp()
+
+    @property
+    def client_config(self):
+        return {
+            'auth_url': getenv('openstack_auth_url'),
+            'username': getenv('openstack_username'),
+            'password': getenv('openstack_password'),
+            'project_name': getenv('openstack_project_name'),
+            'region_name': getenv('openstack_region_name'),
+        }
+
+    @staticmethod
+    def resolve_resource_interface(node_type):
+        try:
+            return getattr(
+                resource_interface_mappings, node_type.split('.')[-1])
+        except AttributeError:
+            return None
+
+    def get_resource_interfaces(self):
+        resource_interface_list = []
+        for node_instance in self.cfy_local.storage.get_node_instances():
+            node_template = \
+                self.cfy_local.storage.get_node(node_instance.node_id)
+            resource_interface = \
+                self.resolve_resource_interface(
+                    node_template.type)
+            if not resource_interface:
+                continue
+            if node_template.properties['use_external_resource']:
+                continue
+            resource_identifier = \
+                node_template.properties['resource_config'].get('name') or \
+                node_template.properties['resource_config'].get('id')
+            if not resource_identifier:
+                raise Exception('Test blueprints must provide name or id.')
+            resource_interface_list.append(
+                resource_interface(resource_identifier, self.client_config))
+        return resource_interface_list
+
+    def verify_no_conflicting_resources(self):
+        """ This method checks that there are no conflicting resources in
+        Openstack before we run a test.
+        :return: Nothing.
+        :Raises Exception: Raises an exception if there are such resources.
+        """
+        for resource_interface in self.get_resource_interfaces():
+            try:
+                conflicting_resource = resource_interface.get()
+            except openstack.exceptions.HttpException:
+                continue
+            raise TestEnvironmentValidationError(
+                'Conflicting resource found {0}'.format(conflicting_resource))
+
+    def delete_all_resources(self):
+        """Deletes orphan resources in Openstack.
+        :return: Nothing.
+        """
+        for resource_interface in self.get_resource_interfaces():
+            try:
+                resource_interface.delete()
+            except openstack.exceptions.SDKException:
+                pass
+
+    def initialize_local_blueprint(self):
+        self.cfy_local = local.init_env(
+            self.blueprint_path,
+            self.test_name,
+            inputs=self.inputs,
+            ignored_modules=IGNORED_LOCAL_WORKFLOW_MODULES)
+        self.verify_no_conflicting_resources()
+
+    def install_blueprint(self,
+                          task_retries=RETRY_MAX,
+                          task_retry_interval=RETRY_INT):
+
+        self.cfy_local.execute(
+            'install',
+            task_retries=task_retries,
+            task_retry_interval=task_retry_interval)
+
+    def uninstall_blueprint(self,
+                            task_retries=RETRY_MAX,
+                            task_retry_interval=RETRY_INT,
+                            ignore_failure=False):
+
+        if ignore_failure:
+            self.cfy_local.execute(
+                'uninstall',
+                parameters={'ignore_failure': True},
+                task_retries=task_retries,
+                task_retry_interval=task_retry_interval)
+        else:
+            self.cfy_local.execute(
+                'uninstall',
+                task_retries=task_retries,
+                task_retry_interval=task_retry_interval)
+
+    def cleanup_uninstall(self):
+        self.uninstall_blueprint(ignore_failure=True)
+
+    def test_keypair_example(self, *_):
+        self.test_name = 'test_keypair_example'
+        self.blueprint_path = './examples/local/keypair.yaml'
+        self.inputs = dict(self.client_config)
+        self.initialize_local_blueprint()
+        self.install_blueprint()
+        self.uninstall_blueprint()
+
+    def test_server_group_example(self, *_):
+        self.test_name = 'test_server_group_example'
+        self.blueprint_path = './examples/local/server_group.yaml'
+        self.inputs = dict(self.client_config)
+        self.initialize_local_blueprint()
+        self.install_blueprint()
+        self.uninstall_blueprint()
+
+    # Requires Special Permissions
+    def test_volume_type_example(self, *_):
+        self.test_name = 'test_volume_type_example'
+        self.blueprint_path = './examples/local/volume_type.yaml'
+        self.inputs = dict(self.client_config)
+        self.initialize_local_blueprint()
+        # execute install workflow
+        self.cfy_local.execute(
+            'install',
+            task_retries=30,
+            task_retry_interval=1)
+        # execute uninstall workflow
+        self.cfy_local.execute(
+            'uninstall',
+            task_retries=30,
+            task_retry_interval=1)
+
+    def test_network_example(self, *_):
+        self.test_name = 'test_network_example'
+        self.blueprint_path = './examples/local/network.yaml'
+        self.inputs = dict(self.client_config)
+        self.inputs.update(
+            {
+                'example_subnet_cidr': '10.10.0.0/24',
+                'example_fixed_ip': '10.10.0.11',
+                'name_prefix': 'network_'
+            }
+        )
+        self.initialize_local_blueprint()
+        self.install_blueprint()
+        self.uninstall_blueprint()
+
+    def test_blueprint_example(self, *_):
+        self.test_name = 'test_blueprint_example'
+        self.blueprint_path = './examples/local/blueprint.yaml'
+        self.inputs = dict(self.client_config)
+        self.inputs.update(
+            {
+                'external_network_id': 'dda079ce-12cf-4309-879a-8e67aec94de4',
+                'example_subnet_cidr': '10.10.0.0/24',
+                'name_prefix': 'blueprint_',
+                'image_id': 'e41430f7-9131-495b-927f-e7dc4b8994c8',
+                'flavor_id': '3',
+                'agent_user': 'ubuntu'
+            }
+        )
+        self.initialize_local_blueprint()
+        self.install_blueprint()
+        time.sleep(10)
+        private_key = StringIO.StringIO()
+        try:
+            server_floating_ip = \
+                self.cfy_local.storage.get_node_instances(
+                    'example-floating_ip_address')[0]
+            server_key_instance = \
+                self.cfy_local.storage.get_node_instances(
+                    'example-keypair')[0]
+            ip_address = \
+                server_floating_ip.runtime_properties[
+                    'floating_ip_address']
+            private_key.write(
+                server_key_instance.runtime_properties['private_key'])
+            private_key.pos = 0
+        except (KeyError, IndexError) as e:
+            raise Exception('Missing Runtime Property: {0}'.format(str(e)))
+
+        with fabric_settings(
+                host_string=ip_address,
+                key=private_key.read(),
+                user=self.inputs.get('agent_user'),
+                abort_on_prompts=True):
+            fabric_run_output = fabric_run('last')
+            self.assertEqual(0, fabric_run_output.return_code)
+
+        # execute uninstall workflow
+        self.uninstall_blueprint()
+
+    def test_hello_world_example(self, *_):
+        self.addCleanup(self.cleanup_uninstall)
+        self.test_name = 'test_hello_world_example'
+        self.blueprint_path = \
+            './examples/cloudify-hello-world-example/openstack.yaml'
+        self.inputs = dict(self.client_config)
+        self.inputs.update(
+            {
+                'external_network_id': 'dda079ce-12cf-4309-879a-8e67aec94de4',
+                'name_prefix': 'hello_world',
+                'image': 'e41430f7-9131-495b-927f-e7dc4b8994c8',
+                'flavor': '2',
+            }
+        )
+        self.initialize_local_blueprint()
+        self.install_blueprint()
+        time.sleep(10)
+
+        try:
+            server_floating_ip = \
+                self.cfy_local.storage.get_node_instances('ip')[0]
+            ip_address = \
+                server_floating_ip.runtime_properties[
+                    'floating_ip_address']
+
+            # Before checking the response returned from the apache server
+            # installed on the server of this blueprint, it could take up to
+            # 30 seconds or less to be up and running, so that we need to
+            # have to wait this time
+            timeout = 30
+            current_time = time.time()
+            is_up = False
+
+            while not is_up and time.time() <= timeout + current_time:
+                try:
+                    response = requests.get('http://{0}'.format(ip_address))
+                    self.assertEqual(response.status_code, 200)
+                    is_up = True
+                except requests.exceptions.ConnectionError:
+                    pass
+
+            if not is_up:
+                raise Exception(
+                    'Server is not responding,'
+                    ' please check your blueprint configuration')
+
+        except (KeyError, IndexError) as e:
+            raise Exception('Missing Runtime Property: {0}'.format(str(e)))
+        # execute uninstall workflow
+        self.uninstall_blueprint()

.cicd/test_manager.py

@@ -0,0 +1,70 @@
+# #######
+# Copyright (c) 2019 Cloudify Platform Ltd. All rights reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+
+from integration_tests.tests.test_cases import PluginsTest
+from integration_tests.tests import utils as test_utils
+
+PLUGIN_NAME = 'cloudify-openstacksdk-plugin'
+
+
+class OpenstackPluginTestCase(PluginsTest):
+
+    base_path = os.path.dirname(os.path.realpath(__file__))
+
+    @property
+    def plugin_root_directory(self):
+        return os.path.abspath(os.path.join(self.base_path, '..'))
+
+    @property
+    def client_config(self):
+        return {
+            'auth_url': os.getenv('openstack_auth_url'),
+            'username': os.getenv('openstack_username'),
+            'password': os.getenv('openstack_password'),
+            'project_name': os.getenv('openstack_project_name'),
+            'region_name': os.getenv('openstack_region_name'),
+        }
+
+    def check_main_blueprint(self):
+        blueprint_id = 'manager_blueprint'
+        self.inputs = dict(self.client_config)
+        self.inputs.update(
+            {
+                'external_network_id': os.getenv(
+                    'external_network_id',
+                    'dda079ce-12cf-4309-879a-8e67aec94de4'),
+                'example_subnet_cidr': '10.10.0.0/24',
+                'name_prefix': 'blueprint_',
+                'image_id': 'e41430f7-9131-495b-927f-e7dc4b8994c8',
+                'flavor_id': '3',
+                'agent_user': 'ubuntu'
+            }
+        )
+        dep, ex_id = self.deploy_application(
+            test_utils.get_resource(
+                os.path.join(
+                    self.plugin_root_directory,
+                    'examples/manager/blueprint.yaml')),
+            timeout_seconds=200,
+            blueprint_id=blueprint_id,
+            deployment_id=blueprint_id,
+            inputs=self.inputs)
+        self.undeploy_application(dep.id)
+
+    def test_blueprints(self):
+        self.upload_mock_plugin(PLUGIN_NAME, self.plugin_root_directory)
+        self.check_main_blueprint()

.circleci/config.yml

@@ -1,5 +1,4 @@
 version: 2
-
 checkout:
   post:
     - >
@@ -8,78 +7,35 @@ checkout:
         git fetch origin +refs/pull/$PR_ID/merge:
         git checkout -qf FETCH_HEAD
       fi
-
 jobs:
-
   unittests:
     docker:
       - image: circleci/python:2.7.15-stretch
     steps:
       - checkout
       - run:
-          name: Download pip
-          command: curl "https://bootstrap.pypa.io/get-pip.py" -o "get-pip.py"
-      - run:
-          name: Install pip
-          command: sudo python get-pip.py
-      - run:
-          name: Install virtualenv
-          command: pip install --user virtualenv
-      - run:
-          name: Init virtualenv
-          command: virtualenv env
-      - run:
-          name: install tox
-          command: pip install --user tox
-      - run: /home/circleci/.local/bin/tox -e flake8
-      - run: /home/circleci/.local/bin/tox -e py27
-
-  wagon:
-    docker:
-      - image: amd64/centos:centos7.3.1611
-    steps:
-      - checkout
-      - run:
-          name: Install dependencies
-          command: yum -y install python-devel gcc openssl git libxslt-devel libxml2-devel openldap-devel libffi-devel openssl-devel libvirt-devel
-      - run:
-          name: Download pip
-          command: curl "https://bootstrap.pypa.io/get-pip.py" -o "get-pip.py"
-      - run:
-          name: Install pip
-          command: python get-pip.py
-      - run:
-          name: Upgrade pip
-          command: pip install --upgrade pip==9.0.1
-      - run:
-          name: Install virtualenv
-          command: pip install virtualenv
-      - run:
-          name: Init virtualenv
-          command: virtualenv env
-      - run:
-          name: Install wagon
-          command: pip install wagon==0.3.2
-      - run:
-          name: many_linux
-          command: echo "manylinux1_compatible = False" > "env/bin/_manylinux.py"
-      - run:
-          name: make workspace
-          command: mkdir -p workspace/build
-      - run:
-          name: Create wagon
-          command: source env/bin/activate && wagon create -s . -v -o workspace/build -f -a '--no-cache-dir -c constraints.txt'
-      - persist_to_workspace:
-          root: workspace
-          paths:
-            - build/*
-
+          name: Running Unit Tests
+          command: |
+            curl "https://bootstrap.pypa.io/get-pip.py" -o "get-pip.py"
+            sudo python get-pip.py
+            pip install --user virtualenv
+            virtualenv env
+            source env/bin/activate
+            pip install tox
+            tox -e flake8
+            tox -e py27
 workflows:
   version: 2
   tests:
     jobs:
       - unittests
-      - wagon:
-          filters:
-            branches:
-              only: /([0-9\.]*\-build|master|dev)/
+  nightly:
+    triggers:
+    - schedule:
+        cron: "0 0 * * *"
+        filters:
+          branches:
+            only:
+              - dev
+    jobs:
+      - unittests

.gitignore

@@ -1,26 +1,35 @@
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
+*$py.class
 
 # C extensions
 *.so
 
 # Distribution / packaging
 .Python
-env/
-bin/
 build/
 develop-eggs/
 dist/
+downloads/
 eggs/
+.eggs/
 lib/
 lib64/
 parts/
 sdist/
 var/
+wheels/
 *.egg-info/
 .installed.cfg
 *.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
 
 # Installer logs
 pip-log.txt
@@ -30,36 +39,71 @@ pip-delete-this-directory.txt
 htmlcov/
 .tox/
 .coverage
-cover/
+.coverage.*
 .cache
 nosetests.xml
 coverage.xml
+cover/
+*.cover
+.hypothesis/
+.pytest_cache/
 
 # Translations
 *.mo
-
-# Mr Developer
-.mr.developer.cfg
-.project
-.pydevproject
-
-# Rope
-.ropeproject
+*.pot
 
 # Django stuff:
 *.log
-*.pot
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
 
 # Sphinx documentation
 docs/_build/
 
-*.iml
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
 
-*COMMIT_MSG
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
 
-# QuickBuild
-.qbcache/
+# mkdocs documentation
+/site
 
-.idea/
+# mypy
+.mypy_cache/
+.idea/*
 
-.DS_Store
+.DS_Store
+*.wgn

CHANGELOG.txt

@@ -1,3 +1,5 @@
+3.0.0:
+    - Openstack Plugin v3: new plugin based on openstacksdk official library (instead of CLI package), new types, new examples, new tests.
 2.14.7:
     - Revert upgrade to OpenStack python clients to the old version used before 2.13.0
 2.14.6:

README.md

@@ -1,30 +1,3 @@
-cloudify-openstack-plugin
-=========================
-
-[![Circle CI](https://circleci.com/gh/cloudify-cosmo/cloudify-openstack-plugin/tree/master.svg?style=shield)](https://circleci.com/gh/cloudify-cosmo/cloudify-openstack-plugin/tree/master)
-[![Build Status](https://travis-ci.org/cloudify-cosmo/cloudify-openstack-plugin.svg?branch=master)](https://travis-ci.org/cloudify-cosmo/cloudify-openstack-plugin)
-
-Cloudify OpenStack Plugin
-
-## Usage
-
-See [Openstack Plugin](https://docs.cloudify.co/latest/developer/official_plugins/openstack/)
-
-
-## Known Issues
-
-You may experience such an error when using a local profile:
-
-```shell
-ERROR:cloudify.cli.main:(PyYAML 3.10 (/.../python2.7/site-packages), Requirement.parse('PyYAML>=3.12'), set(['oslo.config']))
-```
-
-Cloudify CLI requires PyYAML 3.10, whereas Openstack Python SDK Libraries require PyYAML 3.12. For this reason, if you wish to use Cloudify Openstack Plugin in a local profile, you will need to upgrade the PyYAML 3.12 in your virtualenv.
-
-Fix:
-
-```shell
-pip install -U pyyaml==3.12
-```
-
-At this stage, you should no longer use the flag `--install-plugins` with the `cfy` CLI.
+[![Build Status](https://circleci.com/gh/cloudify-cosmo/cloudify-openstack-plugin.svg?style=shield&circle-token=:circle-token)](https://circleci.com/gh/cloudify-cosmo/cloudify-openstack-plugin)
+
+# cloudify-openstack-plugin