Merge pull request #359 from cloudify-cosmo/CY-2555-Port-OSv2-To-Python3

CY-2555-Port-OS-v2-to-python3

.circleci/config.yml

@@ -29,7 +29,7 @@ executors:
 
 
 commands:
-  run_unittest:
+  run_unittests_py27:
     steps:
       - checkout
       - run:
@@ -44,12 +44,14 @@ commands:
 
   check_py3_compat:
     steps:
+      - checkout
       - run:
           name: install futurize
           command: pip install future --user
       - run:
-          name: remove compat module
-          command: rm openstack_sdk/_compat.py
+          name: remove python3 incompatible modules
+          command: |
+            rm openstack_plugin_common/_compat.py
       - run:
           name: find python3-incompatible code
           command: |
@@ -69,6 +71,27 @@ commands:
               exit 1
             fi
 
+  run_unittests_py36:
+    steps:
+      - checkout
+      - run:
+          name: Download pip
+          command: curl "https://bootstrap.pypa.io/get-pip.py" -o "get-pip.py"
+      - run:
+          name: Install pip
+          command: sudo python get-pip.py
+      - run:
+          name: Install virtualenv
+          command: pip install --user virtualenv
+      - run:
+          name: Init virtualenv
+          command: virtualenv env
+      - run:
+          name: install tox
+          command: pip install --user tox
+      - run: /home/circleci/.local/bin/tox -e flake8
+      - run: /home/circleci/.local/bin/tox -e nosetests
+
   generate_py27_wagon:
     steps:
       - run:
@@ -179,17 +202,16 @@ jobs:
   unittests_py27:
     executor: py27
     steps:
-      - run_unittest
+      - run_unittests_py27
 
   unittests_py36:
     executor: py36
     steps:
-      - run_unittest
+      - run_unittests_py36
 
   py3_compat:
     executor: py27
     steps:
-      - checkout
       - check_py3_compat
 
   wagon:
@@ -225,6 +247,8 @@ workflows:
   tests:
     jobs: &all_jobs
       - unittests_py27
+      - unittests_py36
+      - py3_compat
       - wagon:
           filters:
             branches:
@@ -265,18 +289,20 @@ workflows:
               - 2.14.18-build
     jobs:
       - unittests_py27
+      - unittests_py36
+      - py3_compat
       - wagon:
           filters:
             branches:
-              only: /(2.14.18-build|master)/
+              only: /(2.14.19-build|master)/
       - rhel_wagon:
           filters:
             branches:
-              only: /(2.14.18-build|master)/
+              only: /(2.14.19-build|master)/
       - integration_tests:
           requires:
             - wagon
             - rhel_wagon
           filters:
             branches:
-              only: /(2.14.18-build|master)/
+              only: /(2.14.19-build|master)/

.circleci/py3fixers

@@ -0,0 +1,23 @@
+--stage1
+-f lib2to3.fixes.fix_getcwdu
+-f lib2to3.fixes.fix_long
+-f lib2to3.fixes.fix_nonzero
+-f lib2to3.fixes.fix_input
+-f lib2to3.fixes.fix_raw_input
+-f lib2to3.fixes.fix_itertools
+-f lib2to3.fixes.fix_itertools_imports
+-f lib2to3.fixes.fix_exec
+-f lib2to3.fixes.fix_operator
+-f libfuturize.fixes.fix_execfile
+-f libpasteurize.fixes.fix_newstyle
+-f lib2to3.fixes.fix_filter
+# fix_dict is not idempotent
+# -f lib2to3.fixes.fix_dict
+-f lib2to3.fixes.fix_map
+-f lib2to3.fixes.fix_zip
+-f lib2to3.fixes.fix_xrange
+-f lib2to3.fixes.fix_basestring
+-f libfuturize.fixes.fix_cmp
+-f libfuturize.fixes.fix_division_safe
+-f lib2to3.fixes.fix_metaclass
+-f libfuturize.fixes.fix_unicode_keep_u

CHANGELOG.txt

@@ -1,3 +1,5 @@
+2.14.19:
+    - Make plugin python 3 complaint
 2.14.18:
     - Support Import Keypair Public Material
 2.14.17:

dev-requirements.txt

@@ -1,3 +1 @@
-https://github.com/cloudify-cosmo/cloudify-rest-client/archive/3.4.2-build.zip
-https://github.com/cloudify-cosmo/cloudify-dsl-parser/archive/3.4.2-build.zip
-https://github.com/cloudify-cosmo/cloudify-plugins-common/archive/3.4.2-build.zip
+future

glance_plugin/image.py

@@ -12,8 +12,7 @@
 #  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #  * See the License for the specific language governing permissions and
 #  * limitations under the License.
-import httplib
-from urlparse import urlparse
+from openstack_plugin_common._compat import (httplib, urlparse)
 
 from cloudify import ctx
 from cloudify.decorators import operation

glance_plugin/tests/test.py

@@ -54,7 +54,8 @@ def test_check_image_url(self):
         # test if it passes no file & url
         http_connection_mock = mock.MagicMock()
         http_connection_mock.return_value.getresponse.return_value.status = 200
-        with mock.patch('httplib.HTTPConnection', http_connection_mock):
+        with mock.patch('openstack_plugin_common._compat.'
+                        'httplib.HTTPConnection', http_connection_mock):
             glance_plugin.image._validate_image()
 
     def test_check_image_file(self):
@@ -80,7 +81,8 @@ def test_check_image_bad_file(self):
     def test_check_image_bad_url(self):
         http_connection_mock = mock.MagicMock()
         http_connection_mock.return_value.getresponse.return_value.status = 400
-        with mock.patch('httplib.HTTPConnection', http_connection_mock):
+        with mock.patch('openstack_plugin_common._compat.'
+                        'httplib.HTTPConnection', http_connection_mock):
             self.assertRaises(NonRecoverableError,
                               glance_plugin.image._validate_image)
 

keystone_plugin/tests/test_project.py

@@ -1,3 +1,4 @@
+from builtins import object
 import mock
 import unittest
 
@@ -33,7 +34,7 @@ class TestProject(unittest.TestCase):
     test_user = 'test-user'
     test_role = 'test-role'
 
-    class MockProjectOS:
+    class MockProjectOS(object):
         def __init__(self, id, name):
             self._id = id
             self._name = name

keystone_plugin/tests/test_user.py

@@ -1,3 +1,4 @@
+from builtins import object
 import mock
 import unittest
 
@@ -27,7 +28,7 @@ class TestUser(unittest.TestCase):
     updated_name = 'updated-name'
     test_deployment_id = 'test-deployment-id'
 
-    class MockUserOS:
+    class MockUserOS(object):
         def __init__(self, id, name):
             self._id = id
             self._name = name

neutron_plugin/floatingip.py

@@ -32,7 +32,7 @@
     delete_floatingip,
     floatingip_creation_validation
 )
-from network import NETWORK_OPENSTACK_TYPE
+from .network import NETWORK_OPENSTACK_TYPE
 
 FLOATINGIP_OPENSTACK_TYPE = 'floatingip'
 FLOATING_NETWORK_ERROR_PREFIX = \

neutron_plugin/rbac_policy.py

@@ -90,7 +90,7 @@ def find_resource_to_apply_rbac_policy(ctx):
 
 def validate_found_resource(input_dict, found_resource):
     if found_resource:
-        for key in found_resource.keys():
+        for key in found_resource:
             if key in input_dict and input_dict.get(key):
                 raise NonRecoverableError(
                     'Multiple definitions of resource for which '

neutron_plugin/router.py

@@ -72,7 +72,7 @@ def dict_merge(a, b):
         if not isinstance(b, dict):
             return b
         result = deepcopy(a)
-        for k, v in b.iteritems():
+        for k, v in b.items():
             if k in result:
                 ctx.logger.info('Match {0}'.format(k))
                 result[k] = dict_merge(result[k], v)
@@ -406,8 +406,8 @@ def _get_router_from_relationship(neutron_client):
     except NeutronClientException as e:
         raise NonRecoverableError('Error: {0}'.format(str(e)))
     if not isinstance(router, dict) or \
-            ROUTER_OPENSTACK_TYPE not in router.keys() or \
-            'id' not in router['router'].keys():
+            ROUTER_OPENSTACK_TYPE not in router or \
+            'id' not in router['router']:
         raise NonRecoverableError(
             'API returned unexpected structure.: {0}'.format(router))
 

neutron_plugin/security_group.py

@@ -36,6 +36,7 @@
     sg_creation_validation,
     RUNTIME_PROPERTIES_KEYS
 )
+from openstack_plugin_common._compat import text_type
 
 DEFAULT_RULE_VALUES = {
     'direction': 'ingress',
@@ -82,13 +83,14 @@ def create(
         try:
             neutron_client.show_security_group(sg['id'])
         except RequestException as e:
-            ctx.logger.debug("Waiting for SG to be visible. Attempt {}".format(
-                attempt))
+            ctx.logger.debug("Waiting for SG to be visible. Attempt {0} "
+                             " and exception is {1}".format(attempt,
+                                                            text_type(e)))
         else:
             break
     else:
         raise NonRecoverableError(
-            "Timed out waiting for security_group to exist", e)
+            "Timed out waiting for security_group to exist")
 
     set_sg_runtime_properties(sg, neutron_client)
 

neutron_plugin/tests/test_rbac_policy.py

@@ -1,3 +1,4 @@
+from builtins import object
 import mock
 import unittest
 
@@ -30,7 +31,7 @@ class TestRBACPolicy(unittest.TestCase):
     test_os_network_id = '333333333333333'
     test_deployment_id = 'test-deployment-id'
 
-    class MockRBACPolicyOS:
+    class MockRBACPolicyOS(object):
         def __init__(self,
                      id,
                      action,
@@ -65,7 +66,7 @@ def target_tenant(self):
 
         def to_dict(self):
             return dict(
-                [(k.strip('_'), v) for k, v in vars(self).iteritems()]
+                [(k.strip('_'), v) for k, v in vars(self).items()]
             )
 
     def mock_neutron_client(self, mock_rbac_policy):

nova_plugin/keypair.py

@@ -96,7 +96,7 @@ def create(nova_client, args, **kwargs):
             _mkdir_p(os.path.dirname(private_key_path))
             with open(private_key_path, 'w') as f:
                 f.write(keypair.private_key)
-            os.chmod(private_key_path, 0600)
+            os.chmod(private_key_path, 0o600)
         except Exception:
             _delete_private_key_file()
             delete_resource_and_runtime_properties(ctx, nova_client,

nova_plugin/server.py

@@ -69,6 +69,7 @@
 from openstack_plugin_common.security_group import \
     SECURITY_GROUP_OPENSTACK_TYPE
 from glance_plugin.image import handle_image_from_relationship
+from functools import reduce
 
 SERVER_OPENSTACK_TYPE = 'server'
 
@@ -111,7 +112,7 @@ def _get_management_network_id_and_name(neutron_client, ctx):
 
     if management_network_name:
         management_network_name = transform_resource_name(
-            ctx, management_network_name)
+            ctx, u'{0}'.format(management_network_name))
         management_network_id = neutron_client.cosmo_get_named(
             'network', management_network_name)
         management_network_id = management_network_id['id']
@@ -293,7 +294,7 @@ def create(nova_client, neutron_client, args, **_):
     provider_context = provider(ctx)
 
     def rename(name):
-        return transform_resource_name(ctx, name)
+        return transform_resource_name(ctx, u'{0}'.format(name))
 
     server = {
         'name': get_resource_id(ctx, SERVER_OPENSTACK_TYPE),
@@ -307,7 +308,7 @@ def rename(name):
     if 'meta' not in server:
         server['meta'] = dict()
 
-    transform_resource_name(ctx, server)
+    transform_resource_name(ctx, u'{0}'.format(server))
 
     ctx.logger.debug(
         "server.create() server before transformations: {0}".format(server))
@@ -386,7 +387,7 @@ def rename(name):
     assign_payload_as_runtime_properties(ctx, SERVER_OPENSTACK_TYPE, server)
     ctx.logger.debug(
         "Asking Nova to create server. All possible parameters are: [{0}]"
-        .format(','.join(server.keys())))
+        .format(','.join(list(server.keys()))))
 
     try:
         s = nova_client.servers.create(**server)
@@ -411,7 +412,7 @@ def get_network(port_id):
             'port-id': port['port']['id']
         }
 
-    return map(get_network, port_ids)
+    return list(map(get_network, port_ids))
 
 
 @operation(resumable=True)
@@ -1166,7 +1167,7 @@ def _validate_external_server_keypair(nova_client):
 
     keypair_instance_id = \
         [node_instance_id for node_instance_id, runtime_props in
-         ctx.capabilities.get_all().iteritems() if
+         ctx.capabilities.get_all().items() if
          runtime_props.get(OPENSTACK_ID_PROPERTY) == keypair_id][0]
     keypair_node_properties = _get_properties_by_node_instance_id(
         keypair_instance_id)

nova_plugin/tests/test_flavor.py

@@ -1,3 +1,4 @@
+from builtins import object
 import mock
 import unittest
 
@@ -30,7 +31,7 @@ class TestFlavor(unittest.TestCase):
     updated_name = 'updated-name'
     test_deployment_id = 'test-deployment-id'
 
-    class MockFlavorOS:
+    class MockFlavorOS(object):
         def __init__(self, id, name):
             self._id = id
             self._name = name

nova_plugin/tests/test_host_aggregate.py

@@ -1,3 +1,4 @@
+from builtins import object
 import mock
 import unittest
 
@@ -29,7 +30,7 @@ class TestHostAggregate(unittest.TestCase):
     updated_name = 'updated-name'
     test_deployment_id = 'test-deployment-id'
 
-    class MockHostAggregateOS:
+    class MockHostAggregateOS(object):
         def __init__(self, id, name, hosts=None):
             self._id = id
             self._name = name