From 06725002f53c566fec810302c50cd1d0dfe0a150 Mon Sep 17 00:00:00 2001 From: jeroenmaelbrancke Date: Tue, 6 Feb 2018 13:20:53 +0100 Subject: [PATCH 1/5] Merge pull request #73 from openvstorage/extra_features_setup_json Extra features setup json (cherry picked from commit 0f6561f1ff80e3bb8c3dc2c570b880b8cfb84192) # Conflicts: # helpers/storagedriver.py # setup/vpool.py --- helpers/storagedriver.py | 47 +++++++++++++++++++++++++++++++++++++++- setup/vpool.py | 31 +++++++++++++++++++++++--- 2 files changed, 74 insertions(+), 4 deletions(-) diff --git a/helpers/storagedriver.py b/helpers/storagedriver.py index 7e67495..073f983 100644 --- a/helpers/storagedriver.py +++ b/helpers/storagedriver.py @@ -13,9 +13,12 @@ # # Open vStorage is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY of any kind. +import json from ovs.dal.hybrids.storagedriver import StorageDriver from ovs.dal.lists.storagedriverlist import StorageDriverList -from ovs.log.log_handler import LogHandler +from ovs.extensions.generic.configuration import Configuration +from ovs.extensions.generic.sshclient import SSHClient +from ovs.extensions.services.servicefactory import ServiceFactory class StoragedriverHelper(object): @@ -73,3 +76,45 @@ def get_storagedrivers(): :rtype: (ovs.dal.hybrids.storagedriver.STORAGEDRIVER) """ return StorageDriverList.get_storagedrivers() + + @staticmethod + def change_config(storagedriver, config): + """ + Change the config of the volumedriver and reload the config. + Restart will be triggered if no vDisk are running on the volumedriver. + :param storagedriver: StorageDriver object + :type storagedriver: StorageDriver + :param config: Volumedriver config + :type config: dict + :return: + """ + service_manager = ServiceFactory.get_manager() + config_key = '/ovs/vpools/{0}/hosts/{1}/config'.format(storagedriver.vpool.guid, storagedriver.name) + current_config = Configuration.get(config_key) + + if 'volume_manager' in config: + volume_manager = current_config['volume_manager'] + for key, value in config['volume_manager'].iteritems(): + volume_manager[key] = value + + if 'backend_connection_manager' in config: + backend_connection_manager = current_config['backend_connection_manager'] + for key, value in config['backend_connection_manager'].iteritems(): + if key == 'proxy': + for current_config_key, current_config_value in backend_connection_manager.iteritems(): + if current_config_key.isdigit(): + for proxy_key, proxy_config in config['backend_connection_manager']['proxy'].iteritems(): + current_config_value[proxy_key] = proxy_config + + else: + backend_connection_manager[key] = value + StoragedriverHelper.LOGGER.info("New config: {0}".format(json.dumps(current_config, indent=4))) + Configuration.set(config_key, json.dumps(current_config, indent=4), raw=True) + client = SSHClient(storagedriver.storagerouter, 'root') + service_name = 'ovs-volumedriver_{0}'.format(storagedriver.vpool.name) + + if len(storagedriver.vdisks_guids) == 0: + StoragedriverHelper.LOGGER.info("Restarting service: {0}".format(service_name)) + service_manager.restart_service(service_name, client) + else: + StoragedriverHelper.LOGGER.info("Not restarting service: {0}, amount of vdisks: {1}".format(service_name, len(storagedriver.vdisks_guids))) diff --git a/setup/vpool.py b/setup/vpool.py index 212167d..4e2fa1d 100644 --- a/setup/vpool.py +++ b/setup/vpool.py @@ -13,11 +13,14 @@ # # Open vStorage is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY of any kind. - from ovs.lib.generic import GenericController +from ovs.extensions.generic.logger import Logger +from ovs.lib.helpers.toolbox import Toolbox from ovs.log.log_handler import LogHandler from ..helpers.backend import BackendHelper +from ..helpers.storagedriver import StoragedriverHelper from ..helpers.storagerouter import StoragerouterHelper +from ..helpers.vpool import VPoolHelper from ..validate.decorators import required_roles, check_vpool @@ -27,6 +30,10 @@ class VPoolSetup(object): ADD_VPOOL_TIMEOUT = 500 REQUIRED_VPOOL_ROLES = ['DB', 'WRITE', 'DTL'] + # These will be all possible settings for the StorageDriver. Messing them up is their own responsibility (they should not bypass the API by default!!) + STORAGEDRIVER_PARAMS = {"volume_manager": (dict, None, False), + "backend_connection_manager": (dict, None, False)} + def __init__(self): pass @@ -72,7 +79,11 @@ def add_vpool(vpool_name, vpool_details, api, storagerouter_ip, proxy_amount=2, 'parallelism': {'proxies': proxy_amount} } api_data = {'call_parameters': call_parameters} - + + # Setting for mds_safety + if vpool_details.get('mds_safety') is not None: + call_parameters['mds_config_params'] = {'mds_safety': vpool_details['mds_safety']} + # Setting possible alba accelerated alba if vpool_details['fragment_cache']['location'] == 'backend': call_parameters['backend_info_aa'] = {'alba_backend_guid': BackendHelper.get_albabackend_by_name(vpool_details['fragment_cache']['backend']['name']).guid, @@ -113,7 +124,21 @@ def add_vpool(vpool_name, vpool_details, api, storagerouter_ip, proxy_amount=2, raise RuntimeError(error_msg) else: VPoolSetup.LOGGER.info('Creation of vPool `{0}` should have succeeded on storagerouter `{1}`'.format(vpool_name, storagerouter_ip)) - return storagerouter_ip, '/mnt/{0}'.format(vpool_name) + + # Settings volumedriver + storagedriver_config = vpool_details.get('storagedriver') + if storagedriver_config is not None: + Toolbox.verify_required_params(VPoolSetup.STORAGEDRIVER_PARAMS, storagedriver_config) + VPoolSetup.LOGGER.info('Updating volumedriver configuration of vPool `{0}` on storagerouter `{1}`.'.format(vpool_name, storagerouter_ip)) + vpool = VPoolHelper.get_vpool_by_name(vpool_name) + storagedriver = [sd for sd in vpool.storagedrivers if sd.storagerouter.ip == storagerouter_ip][0] + if not storagedriver: + error_msg = 'Unable to find the storagedriver of vPool {0} on storagerouter {1}'.format(vpool_name, storagerouter_ip) + raise RuntimeError(error_msg) + StoragedriverHelper.change_config(storagedriver, storagedriver_config) + VPoolSetup.LOGGER.info('Updating volumedriver config of vPool `{0}` should have succeeded on storagerouter `{1}`'.format(vpool_name, storagerouter_ip)) + + return storagerouter_ip, '/mnt/{0}'.format(vpool_name) @staticmethod def execute_scrubbing(): From 543982695888031ba2729bd5b4a9d29409e4a8b0 Mon Sep 17 00:00:00 2001 From: Jeffrey Devloo Date: Wed, 7 Feb 2018 10:25:44 +0100 Subject: [PATCH 2/5] Remove the newer logger and replace it with the old one --- helpers/storagedriver.py | 1 + setup/vpool.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/helpers/storagedriver.py b/helpers/storagedriver.py index 073f983..5cf66af 100644 --- a/helpers/storagedriver.py +++ b/helpers/storagedriver.py @@ -19,6 +19,7 @@ from ovs.extensions.generic.configuration import Configuration from ovs.extensions.generic.sshclient import SSHClient from ovs.extensions.services.servicefactory import ServiceFactory +from ovs.log.log_handler import LogHandler class StoragedriverHelper(object): diff --git a/setup/vpool.py b/setup/vpool.py index 4e2fa1d..2c481b1 100644 --- a/setup/vpool.py +++ b/setup/vpool.py @@ -14,7 +14,6 @@ # Open vStorage is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY of any kind. from ovs.lib.generic import GenericController -from ovs.extensions.generic.logger import Logger from ovs.lib.helpers.toolbox import Toolbox from ovs.log.log_handler import LogHandler from ..helpers.backend import BackendHelper From 2c22485f732154a2cc71e0522b6fd830381824f2 Mon Sep 17 00:00:00 2001 From: jeroenmaelbrancke Date: Wed, 7 Feb 2018 16:07:45 +0100 Subject: [PATCH 3/5] Change the volumedriver config after the complete setup of the vpools (extending) --- setup/storagedriver.py | 49 ++++++++++++++++++++++++++++++++++++++++++ setup/vpool.py | 20 +---------------- 2 files changed, 50 insertions(+), 19 deletions(-) create mode 100644 setup/storagedriver.py diff --git a/setup/storagedriver.py b/setup/storagedriver.py new file mode 100644 index 0000000..f08221e --- /dev/null +++ b/setup/storagedriver.py @@ -0,0 +1,49 @@ +# Copyright (C) 2016 iNuron NV +# +# This file is part of Open vStorage Open Source Edition (OSE), +# as available from +# +# http://www.openvstorage.org and +# http://www.openvstorage.com. +# +# This file is free software; you can redistribute it and/or modify it +# under the terms of the GNU Affero General Public License v3 (GNU AGPLv3) +# as published by the Free Software Foundation, in version 3 as it comes +# in the LICENSE.txt file of the Open vStorage OSE distribution. +# +# Open vStorage is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY of any kind. +from ovs.lib.helpers.toolbox import Toolbox +from ovs.log.log_handler import LogHandler +from ..helpers.storagedriver import StoragedriverHelper +from ..helpers.vpool import VPoolHelper + + +class StoragedriverSetup(object): + LOGGER = LogHandler.get(source='setup', name='ci_vpool_setup') + + # These will be all possible settings for the StorageDriver. Messing them up is their own responsibility (they should not bypass the API by default!!) + STORAGEDRIVER_PARAMS = {"volume_manager": (dict, None, False), + "backend_connection_manager": (dict, None, False)} + + @staticmethod + def change_config(vpool_name, vpool_details, storagerouter_ip, *args, **kwargs): + + # Settings volumedriver + storagedriver_config = vpool_details.get('storagedriver') + if storagedriver_config is not None: + Toolbox.verify_required_params(StoragedriverSetup.STORAGEDRIVER_PARAMS, storagedriver_config) + StoragedriverSetup.LOGGER.info( + 'Updating volumedriver configuration of vPool `{0}` on storagerouter `{1}`.'.format(vpool_name, + storagerouter_ip)) + vpool = VPoolHelper.get_vpool_by_name(vpool_name) + storagedriver = [sd for sd in vpool.storagedrivers if sd.storagerouter.ip == storagerouter_ip][0] + if not storagedriver: + error_msg = 'Unable to find the storagedriver of vPool {0} on storagerouter {1}'.format(vpool_name, + storagerouter_ip) + raise RuntimeError(error_msg) + StoragedriverHelper.change_config(storagedriver, storagedriver_config) + vpool.invalidate_dynamics('configuration') + StoragedriverSetup.LOGGER.info( + 'Updating volumedriver config of vPool `{0}` should have succeeded on storagerouter `{1}`'.format( + vpool_name, storagerouter_ip)) diff --git a/setup/vpool.py b/setup/vpool.py index 2c481b1..a43cc0b 100644 --- a/setup/vpool.py +++ b/setup/vpool.py @@ -29,10 +29,6 @@ class VPoolSetup(object): ADD_VPOOL_TIMEOUT = 500 REQUIRED_VPOOL_ROLES = ['DB', 'WRITE', 'DTL'] - # These will be all possible settings for the StorageDriver. Messing them up is their own responsibility (they should not bypass the API by default!!) - STORAGEDRIVER_PARAMS = {"volume_manager": (dict, None, False), - "backend_connection_manager": (dict, None, False)} - def __init__(self): pass @@ -123,21 +119,7 @@ def add_vpool(vpool_name, vpool_details, api, storagerouter_ip, proxy_amount=2, raise RuntimeError(error_msg) else: VPoolSetup.LOGGER.info('Creation of vPool `{0}` should have succeeded on storagerouter `{1}`'.format(vpool_name, storagerouter_ip)) - - # Settings volumedriver - storagedriver_config = vpool_details.get('storagedriver') - if storagedriver_config is not None: - Toolbox.verify_required_params(VPoolSetup.STORAGEDRIVER_PARAMS, storagedriver_config) - VPoolSetup.LOGGER.info('Updating volumedriver configuration of vPool `{0}` on storagerouter `{1}`.'.format(vpool_name, storagerouter_ip)) - vpool = VPoolHelper.get_vpool_by_name(vpool_name) - storagedriver = [sd for sd in vpool.storagedrivers if sd.storagerouter.ip == storagerouter_ip][0] - if not storagedriver: - error_msg = 'Unable to find the storagedriver of vPool {0} on storagerouter {1}'.format(vpool_name, storagerouter_ip) - raise RuntimeError(error_msg) - StoragedriverHelper.change_config(storagedriver, storagedriver_config) - VPoolSetup.LOGGER.info('Updating volumedriver config of vPool `{0}` should have succeeded on storagerouter `{1}`'.format(vpool_name, storagerouter_ip)) - - return storagerouter_ip, '/mnt/{0}'.format(vpool_name) + return storagerouter_ip, '/mnt/{0}'.format(vpool_name) @staticmethod def execute_scrubbing(): From 55b276cea855f99ea29ff5b2e50b05f66871c4e4 Mon Sep 17 00:00:00 2001 From: jeroenmaelbrancke Date: Wed, 7 Feb 2018 16:30:33 +0100 Subject: [PATCH 4/5] Unused imports and formatting --- setup/storagedriver.py | 13 ++++--------- setup/vpool.py | 3 --- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/setup/storagedriver.py b/setup/storagedriver.py index f08221e..34ae470 100644 --- a/setup/storagedriver.py +++ b/setup/storagedriver.py @@ -20,7 +20,7 @@ class StoragedriverSetup(object): - LOGGER = LogHandler.get(source='setup', name='ci_vpool_setup') + LOGGER = LogHandler.get(source='setup', name='ci_storagedriver_setup') # These will be all possible settings for the StorageDriver. Messing them up is their own responsibility (they should not bypass the API by default!!) STORAGEDRIVER_PARAMS = {"volume_manager": (dict, None, False), @@ -33,17 +33,12 @@ def change_config(vpool_name, vpool_details, storagerouter_ip, *args, **kwargs): storagedriver_config = vpool_details.get('storagedriver') if storagedriver_config is not None: Toolbox.verify_required_params(StoragedriverSetup.STORAGEDRIVER_PARAMS, storagedriver_config) - StoragedriverSetup.LOGGER.info( - 'Updating volumedriver configuration of vPool `{0}` on storagerouter `{1}`.'.format(vpool_name, - storagerouter_ip)) + StoragedriverSetup.LOGGER.info('Updating volumedriver configuration of vPool `{0}` on storagerouter `{1}`.'.format(vpool_name, storagerouter_ip)) vpool = VPoolHelper.get_vpool_by_name(vpool_name) storagedriver = [sd for sd in vpool.storagedrivers if sd.storagerouter.ip == storagerouter_ip][0] if not storagedriver: - error_msg = 'Unable to find the storagedriver of vPool {0} on storagerouter {1}'.format(vpool_name, - storagerouter_ip) + error_msg = 'Unable to find the storagedriver of vPool {0} on storagerouter {1}'.format(vpool_name, storagerouter_ip) raise RuntimeError(error_msg) StoragedriverHelper.change_config(storagedriver, storagedriver_config) vpool.invalidate_dynamics('configuration') - StoragedriverSetup.LOGGER.info( - 'Updating volumedriver config of vPool `{0}` should have succeeded on storagerouter `{1}`'.format( - vpool_name, storagerouter_ip)) + StoragedriverSetup.LOGGER.info('Updating volumedriver config of vPool `{0}` should have succeeded on storagerouter `{1}`'.format(vpool_name, storagerouter_ip)) diff --git a/setup/vpool.py b/setup/vpool.py index a43cc0b..0b17c54 100644 --- a/setup/vpool.py +++ b/setup/vpool.py @@ -14,12 +14,9 @@ # Open vStorage is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY of any kind. from ovs.lib.generic import GenericController -from ovs.lib.helpers.toolbox import Toolbox from ovs.log.log_handler import LogHandler from ..helpers.backend import BackendHelper -from ..helpers.storagedriver import StoragedriverHelper from ..helpers.storagerouter import StoragerouterHelper -from ..helpers.vpool import VPoolHelper from ..validate.decorators import required_roles, check_vpool From 9683f10badd5bd2008db5e265410f14b5d5bedaf Mon Sep 17 00:00:00 2001 From: Jeffrey Devloo Date: Thu, 8 Feb 2018 12:14:39 +0100 Subject: [PATCH 5/5] Bring 0.3 into master --- setup/storagedriver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup/storagedriver.py b/setup/storagedriver.py index 34ae470..44fae1c 100644 --- a/setup/storagedriver.py +++ b/setup/storagedriver.py @@ -14,13 +14,13 @@ # Open vStorage is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY of any kind. from ovs.lib.helpers.toolbox import Toolbox -from ovs.log.log_handler import LogHandler +from ovs.extensions.generic.logger import Logger from ..helpers.storagedriver import StoragedriverHelper from ..helpers.vpool import VPoolHelper class StoragedriverSetup(object): - LOGGER = LogHandler.get(source='setup', name='ci_storagedriver_setup') + LOGGER = Logger('setup-ci_storagedriver_setup') # These will be all possible settings for the StorageDriver. Messing them up is their own responsibility (they should not bypass the API by default!!) STORAGEDRIVER_PARAMS = {"volume_manager": (dict, None, False),