Merge pull request #87 from openvstorage/develop

Promote develop

docs/guide.md

@@ -3,7 +3,6 @@
 ## Description
 
 This repository contains the automation library for Open vStorage.
-This library delegates component creation/removal to the REST API of Open vStorage through Python code.
 
 ## System requirements
 
@@ -26,17 +25,36 @@ This library delegates component creation/removal to the REST API of Open vStora
 - Automation library HELPERS logging file `/var/log/ovs/helpers.log`
 
 ## Sections
+### Api Library
+This library delegates component creation/removal to the REST API of Open vStorage through Python code.
+
+#### Helpers section
+Contains functions to assist in removal, setup and validation of components such as backends, disks, storagerouters and -drivers, as well as gathering of metadata etc.
+
+#### Remove section
+Contains functions for removal of arakoon clusters, backends, roles, vDisks and vPools from Open vStorage.
+
+#### Setup section
+Contains functions to set up new arakoon clusters, backends, domains, proxies, roles, vDisks and vPools in Open vStorage.
 
-### Helpers section
-Contains helping function that provide required meta information during setup, removal or validation
+#### Validation section
+Contains function to validate functionality of Open vStorage components. 
+This includes decorators for checking prerequisites of functions throughout the package.
 
-### Remove section
-Contains removal functions that makes it possible to remove components from Open vStorage
+###Scenario helpers section
+Classes in this section are used to execute the actual tests (referred to as[scenarios](#header_scenarios))
 
-### Setup section
-Contains setup functions that makes it possible to add components to Open vStorage
+###<a name="header_scenarios"></a>Scenarios section
+This section contains code for testing a variety of integration scenarios.\
+Currently present tests:
+- api checkup post-reboot
+- several arakoon related checks
+- addition and removal of
+    - backends
+    - storagerouters
+    - vDisks, vMachines and vPools
+- health checks
+- installation tests
+- hypervisor tests
 
-### Validation section
-Provides validation for setup or removal of Open vStorage components. 
-E.g. when a vPool is added, the required components are checked if they are present
 

helpers/albanode.py

@@ -13,32 +13,32 @@
 #
 # Open vStorage is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY of any kind.
+
 from ovs.dal.hybrids.albanode import AlbaNode
 from ovs.dal.lists.albanodelist import AlbaNodeList
 from ovs.extensions.generic.logger import Logger
+from ..helpers.ci_constants import CIConstants
 
 
-class AlbaNodeHelper(object):
+class AlbaNodeHelper(CIConstants):
     """
     Alba node helper class
     """
 
     LOGGER = Logger('helpers-ci_albanode')
     IGNORE_KEYS = ('_error', '_duration', '_version', '_success')
 
-    @staticmethod
-    def _map_alba_nodes(api):
+    @classmethod
+    def _map_alba_nodes(cls, *args, **kwargs):
         """
         Will map the alba_node_id with its guid counterpart and return the map dict
-        :param api: specify a valid api connection to the setup
-        :type api: helpers.api.OVSClient
         """
-        mapping = {}
 
+        mapping = {}
         options = {
             'contents': 'node_id,_relations',
         }
-        response = api.get(
+        response = cls.api.get(
             api='alba/nodes',
             params=options
         )

helpers/api.py

@@ -193,17 +193,17 @@ def _call(self, api, params, func, **kwargs):
         if self._volatile_client is not None:
             self._token = self._volatile_client.get(self._key)
         first_connect = self._token is None
-        headers, url = self._prepare(params=params)
+        headers, _url = self._prepare(params=params)
         try:
-            return self._process(func(url=url.format(api), headers=headers, verify=self._verify, **kwargs))
+            return self._process(func(url=_url.format(api), headers=headers, verify=self._verify, **kwargs))
         except ForbiddenException:
             if self._volatile_client is not None:
                 self._volatile_client.delete(self._key)
             if first_connect is True:  # First connect, so no token was present yet, so no need to try twice without token
                 raise
             self._token = None
-            headers, url = self._prepare(params=params)
-            return self._process(func(url=url.format(api), headers=headers, verify=self._verify, **kwargs))
+            headers, _url = self._prepare(params=params)
+            return self._process(func(url=_url.format(api), headers=headers, verify=self._verify, **kwargs))
         except Exception:
             if self._volatile_client is not None:
                 self._volatile_client.delete(self._key)
@@ -264,7 +264,12 @@ def wait_for_task(self, task_id, timeout=None):
             if timeout is not None and timeout < (time.time() - start):
                 raise TimeOutError('Waiting for task {0} has timed out.'.format(task_id))
             task_metadata = self.get('/tasks/{0}/'.format(task_id))
-            print task_metadata
+            output = 'Task with ID: {0: >40}, current status: {1: >8}, ready: {2: >2}. Result data: {3}'.format(task_metadata['id'],
+                                                                                                                 task_metadata['status'],
+                                                                                                                 task_metadata['successful'],
+                                                                                                                 task_metadata['result'])
+            print output
+            OVSClient._logger.debug(output)
             finished = task_metadata['status'] in ('FAILURE', 'SUCCESS')
             if finished is False:
                 if task_metadata != previous_metadata:
@@ -286,8 +291,6 @@ def _to_json(dict_or_json):
         :return: json data
         :rtype: string
         """
-        try:
-            json_object = json.loads(str(dict_or_json))
-        except ValueError:
+        if isinstance(dict_or_json, dict):
             return json.dumps(dict_or_json)
-        return json_object
+        return dict_or_json

helpers/backend.py

@@ -13,15 +13,17 @@
 #
 # Open vStorage is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY of any kind.
+
+from ovs.dal.hybrids.albabackend import AlbaBackend
 from ovs.dal.lists.albabackendlist import AlbaBackendList
 from ovs.dal.lists.backendlist import BackendList
 from ovs.dal.lists.backendtypelist import BackendTypeList
-from ovs.dal.hybrids.albabackend import AlbaBackend
 from ovs.extensions.generic.logger import Logger
+from ..helpers.ci_constants import CIConstants
 from ..helpers.exceptions import PresetNotFoundError, AlbaBackendNotFoundError
 
 
-class BackendHelper(object):
+class BackendHelper(CIConstants):
     """
     BackendHelper class
     """
@@ -127,45 +129,40 @@ def get_albabackend_by_name(albabackend_name):
             BackendHelper.LOGGER.error(error_msg)
             raise NameError(error_msg)
 
-    @staticmethod
-    def get_asd_safety(albabackend_guid, asd_id, api):
+    @classmethod
+    def get_asd_safety(cls, albabackend_guid, asd_id, *args, **kwargs):
         """
         Request the calculation of the disk safety
         :param albabackend_guid: guid of the alba backend
         :type albabackend_guid: str
         :param asd_id: id of the asd
         :type asd_id: str
-        :param api: specify a valid api connection to the setup
-        :type api: helpers.api.OVSClient
         :return: asd safety
         :rtype: dict
         """
         params = {'asd_id': asd_id}
-        task_guid = api.get('alba/backends/{0}/calculate_safety'.format(albabackend_guid), params=params)
-        result = api.wait_for_task(task_id=task_guid, timeout=30)
+        task_guid = cls.api.get('alba/backends/{0}/calculate_safety'.format(albabackend_guid), params=params)
+        result = cls.api.wait_for_task(task_id=task_guid, timeout=30)
 
         if result[0] is False:
             errormsg = "Calculate safety for '{0}' failed with '{1}'".format(asd_id, result[1])
             BackendHelper.LOGGER.error(errormsg)
             raise RuntimeError(errormsg)
         return result[1]
 
-    @staticmethod
-    def get_backend_local_stack(albabackend_name, api):
+    @classmethod
+    def get_backend_local_stack(cls, albabackend_name, *args, **kwargs):
         """
         Fetches the local stack property of a backend
 
         :param albabackend_name: backend name
         :type albabackend_name: str
-        :param api: specify a valid api connection to the setup
-        :type api: helpers.api.OVSClient
         """
         options = {
             'contents': 'local_stack',
         }
-        return api.get(api='/alba/backends/{0}/'.format(BackendHelper.get_alba_backend_guid_by_name(albabackend_name)),
-                       params={'queryparams': options}
-                       )
+        return cls.api.get(api='/alba/backends/{0}/'.format(BackendHelper.get_alba_backend_guid_by_name(albabackend_name)),
+                           params={'queryparams': options})
 
     @staticmethod
     def get_alba_backends():

helpers/ci_constants.py

@@ -0,0 +1,55 @@
+# Copyright (C) 2016 iNuron NV
+#
+# This file is part of Open vStorage Open Source Edition (OSE),
+# as available from
+#
+#      http://www.openvstorage.org and
+#      http://www.openvstorage.com.
+#
+# This file is free software; you can redistribute it and/or modify it
+# under the terms of the GNU Affero General Public License v3 (GNU AGPLv3)
+# as published by the Free Software Foundation, in version 3 as it comes
+# in the LICENSE.txt file of the Open vStorage OSE distribution.
+#
+# Open vStorage is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY of any kind.
+import json
+from ci.api_lib.helpers.api import OVSClient
+
+
+class CIConstants(object):
+    """
+    Collection of multiple constants and constant related instances
+    """
+
+    CONFIG_LOC = "/opt/OpenvStorage/ci/config/setup.json"
+    TEST_SCENARIO_LOC = "/opt/OpenvStorage/ci/scenarios/"
+    TESTRAIL_LOC = "/opt/OpenvStorage/ci/config/testrail.json"
+
+    with open(CONFIG_LOC, 'r') as JSON_CONFIG:
+        SETUP_CFG = json.load(JSON_CONFIG)
+
+    HYPERVISOR_INFO = SETUP_CFG['ci'].get('hypervisor')
+    DOMAIN_INFO = SETUP_CFG['setup']['domains']
+    BACKEND_INFO = SETUP_CFG['setup']['backends']
+    STORAGEROUTER_INFO = SETUP_CFG['setup']['storagerouters']
+
+    class classproperty(property):
+        def __get__(self, cls, owner):
+            return classmethod(self.fget).__get__(None, owner)()
+
+    @classproperty
+    def api(cls):
+        return OVSClient(cls.SETUP_CFG['ci']['grid_ip'],
+                         cls.SETUP_CFG['ci']['user']['api']['username'],
+                         cls.SETUP_CFG['ci']['user']['api']['password'])
+
+    @classmethod
+    def get_vpool_names(cls):
+        names = []
+        for sr_ip, items in cls.STORAGEROUTER_INFO.iteritems():
+            vpools = items.get('vpools')
+            for vp_name, vp_info in vpools.iteritems():
+                if vp_name not in names:
+                    names.append(vp_name)
+        return names

helpers/disk.py

@@ -24,14 +24,10 @@ class DiskHelper(object):
     DiskHelper class
     """
 
-    def __init__(self):
-        pass
-
     @staticmethod
     def get_diskpartitions_by_guid(diskguid):
         """
         Fetch disk partitions by disk guid
-
         :param diskguid: ip address of a storagerouter
         :type diskguid: str
         :return: list of DiskPartition Objects
@@ -41,59 +37,53 @@ def get_diskpartitions_by_guid(diskguid):
         return [dp for dp in DiskPartitionList.get_partitions() if dp.disk_guid == diskguid]
 
     @staticmethod
-    def get_roles_from_disks(storagerouter_ip=None):
+    def get_roles_from_disks(storagerouter_guid=None):
         """
         Fetch disk roles from all disks with optional storagerouter_ip
-
-        :param storagerouter_ip: ip address of a storage router
-        :type storagerouter_ip: str
+        :param storagerouter_guid: guid of a storage router
+        :type storagerouter_guid: str
         :return: list of lists with roles
         :rtype: list > list
         """
-        if not storagerouter_ip:
+        if not storagerouter_guid:
             return [partition.roles for disk in DiskList.get_disks() for partition in disk.partitions]
         else:
-            storagerouter_guid = StoragerouterHelper.get_storagerouter_guid_by_ip(storagerouter_ip)
             return [partition.roles for disk in DiskList.get_disks()
                     if disk.storagerouter_guid == storagerouter_guid for partition in disk.partitions]
 
     @staticmethod
-    def get_disk_by_diskname(storagerouter_ip, disk_name):
+    def get_disk_by_diskname(storagerouter_guid, disk_name):
         """
-        Get a disk object by storagerouter ip and disk name
-
-        :param storagerouter_ip: ip address of a storage router
-        :type storagerouter_ip: str
+        Get a disk object by storagerouter guid and disk name
+        :param storagerouter_guid: guid address of a storage router
+        :type storagerouter_guid: str
         :param disk_name: name of a disk (e.g. sda)
         :type disk_name: str
         :return: disk object
         :rtype: ovs.dal.hybrids.Disk
         """
-
-        storagerouter = StoragerouterHelper.get_storagerouter_by_ip(storagerouter_ip=storagerouter_ip)
+        storagerouter = StoragerouterHelper.get_storagerouter_by_guid(storagerouter_guid=storagerouter_guid)
         for disk in storagerouter.disks:
             if disk.name == disk_name:
                 return disk
 
     @staticmethod
-    def get_roles_from_disk(storagerouter_ip, disk_name):
+    def get_roles_from_disk(storagerouter_guid, disk_name):
         """
         Get the roles from a certain disk
-
-        :param storagerouter_ip: ip address of a storage router
-        :type storagerouter_ip: str
+        :param storagerouter_guid: guid address of a storage router
+        :type storagerouter_guid: str
         :param disk_name: name of a disk (e.g. sda)
         :type disk_name: str
         :return: list of roles of all partitions on a certain disk
         :rtype: list
         """
-
-        disk = DiskHelper.get_disk_by_diskname(storagerouter_ip, disk_name)
+        disk = DiskHelper.get_disk_by_diskname(storagerouter_guid, disk_name)
         roles_on_disk = []
         if disk:
             for diskpartition in disk.partitions:
                 for role in diskpartition.roles:
                     roles_on_disk.append(role)
             return roles_on_disk
         else:
-            raise RuntimeError("Disk with name `{0}` not found on storagerouter `{1}`".format(disk_name, storagerouter_ip))
+            raise RuntimeError("Disk with name `{0}` not found on storagerouter `{1}`".format(disk_name, storagerouter_guid))