diff --git a/.gitignore b/.gitignore index 1010a238..2b19cc4a 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,10 @@ dist/ # VS Code config workspace.code-workspace + +# cache specific +*.pyc +**__pycache__** + +# removing pycharm specific +.idea diff --git a/CHANGELOG.md b/CHANGELOG.md index 824cedee..97e0946c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [2.0.0] - 2024-02-08 + +### Changed + +- CASM-4350: To address the 1MiB size limit of Kubernetes ConfigMaps, the + `cray-product-catalog` Kubernetes ConfigMap is split into multiple smaller + ConfigMaps with each product's `component_versions` data in its own ConfigMap. + Modified `create`, `modify`, `delete` and `query` feature to support + the split of single ConfigMap into multiple ConfigMaps. +- Added new argument `max_attempts` to `modify_config_map` function in + [`catalog_delete.py`](cray_product_catalog/catalog_delete.py), because we need not retry 100 + times when read ConfigMap fails for a product ConfigMap. +- CASM-4504: Added label "type=cray-product-catalog" to all cray-product-catalog related ConfigMaps +- Implemented migration of cray-product-catalog ConfigMap to multiple ConfigMaps as part of pre-upgrade steps +- Added migration job to the configmap-hook + ### Dependencies - Bump `tj-actions/changed-files` from 40 to 42 ([#307](https://github.com/Cray-HPE/cray-product-catalog/pull/307), [#309](https://github.com/Cray-HPE/cray-product-catalog/pull/309)) @@ -443,7 +459,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Change default reviewers to CMS-core-product-support -[Unreleased]: https://github.com/Cray-HPE/cray-product-catalog/compare/v1.10.0...HEAD +[Unreleased]: https://github.com/Cray-HPE/cray-product-catalog/compare/v2.0.0...HEAD + +[2.0.0]: https://github.com/Cray-HPE/cray-product-catalog/compare/v1.10.0...v2.0.0 [1.10.0]: https://github.com/Cray-HPE/cray-product-catalog/compare/v1.9.0...v1.10.0 diff --git a/charts/cray-product-catalog/templates/configmap-hook.yaml b/charts/cray-product-catalog/templates/configmap-hook.yaml index 6dd44313..8c32f855 100644 --- a/charts/cray-product-catalog/templates/configmap-hook.yaml +++ b/charts/cray-product-catalog/templates/configmap-hook.yaml @@ -1,7 +1,7 @@ {{/* MIT License -(C) Copyright 2021-2022 Hewlett Packard Enterprise Development LP +(C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), @@ -43,10 +43,11 @@ spec: command: ["/bin/sh"] args: - "-c" - - "kubectl get cm -n services cpc-backup -o yaml | + - "kubectl delete cm -n services cray-product-catalog && + kubectl get cm -n services cpc-backup -o yaml | yq eval '.metadata.name = \"cray-product-catalog\"' - | yq eval 'del(.metadata.resourceVersion, .metadata.uid, .metadata.annotations, .metadata.creationTimestamp, .metadata.selfLink, .metadata.managedFields)' - | - kubectl apply -f - && kubectl delete cm -n services cpc-backup" + kubectl create -f - && kubectl delete cm -n services cpc-backup" --- apiVersion: batch/v1 @@ -56,6 +57,7 @@ metadata: namespace: services annotations: "helm.sh/hook": pre-upgrade,pre-rollback + "helm.sh/hook-weight": "2" spec: template: metadata: @@ -73,4 +75,34 @@ spec: - "kubectl get cm -n services cray-product-catalog -o yaml | yq eval '.metadata.name = \"cpc-backup\"' - | yq eval 'del(.metadata.resourceVersion, .metadata.uid, .metadata.annotations, .metadata.creationTimestamp, .metadata.selfLink, .metadata.managedFields)' - | - kubectl apply -f -" \ No newline at end of file + kubectl create -f -" + +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ .Release.Name }}-migrate + annotations: + "helm.sh/hook": pre-upgrade + "helm.sh/hook-weight": "1" +spec: + backoffLimit: 0 + template: + spec: + containers: + - args: + - -c + - /usr/bin/catalog_migrate + command: + - /bin/sh + env: + - name: CONFIG_MAP_NAME + value: {{ .Values.migration.configMap }} + - name: CONFIG_MAP_NAMESPACE + value: {{ .Values.migration.configMapNamespace }} + image: "{{ .Values.migration.image.repository }}:{{ .Values.global.appVersion }}" + imagePullPolicy: IfNotPresent + name: migrate-catalog + restartPolicy: Never + serviceAccountName: {{ .Values.migration.serviceAccountName }} + diff --git a/charts/cray-product-catalog/templates/configmap.yaml b/charts/cray-product-catalog/templates/configmap.yaml index 697bee06..bc5f7f3c 100644 --- a/charts/cray-product-catalog/templates/configmap.yaml +++ b/charts/cray-product-catalog/templates/configmap.yaml @@ -1,7 +1,7 @@ {{/* MIT License -(C) Copyright 2021-2022 Hewlett Packard Enterprise Development LP +(C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), @@ -25,4 +25,6 @@ apiVersion: v1 kind: ConfigMap metadata: name: cray-product-catalog + labels: + type: cray-product-catalog data: null diff --git a/charts/cray-product-catalog/values.yaml b/charts/cray-product-catalog/values.yaml index 8b423477..bd636e27 100644 --- a/charts/cray-product-catalog/values.yaml +++ b/charts/cray-product-catalog/values.yaml @@ -1,7 +1,7 @@ # # MIT License # -# (C) Copyright 2021-2022 Hewlett Packard Enterprise Development LP +# (C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the "Software"), @@ -25,3 +25,13 @@ kubectl: image: repository: artifactory.algol60.net/csm-docker/stable/docker-kubectl tag: 1.19.15 +migration: + image: + repository: artifactory.algol60.net/csm-docker/S-T-A-B-L-E/cray-product-catalog-update + tag: 0.0.0-docker + configMap: cray-product-catalog + configMapNamespace: services + serviceAccountName: cray-product-catalog + +global: + appVersion: 0.0.0-docker diff --git a/cray_product_catalog/catalog_delete.py b/cray_product_catalog/catalog_delete.py index 43dd3457..5fb13fc7 100755 --- a/cray_product_catalog/catalog_delete.py +++ b/cray_product_catalog/catalog_delete.py @@ -2,7 +2,7 @@ # # MIT License # -# (C) Copyright 2021-2023 Hewlett Packard Enterprise Development LP +# (C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the "Software"), @@ -55,13 +55,174 @@ from cray_product_catalog.logging import configure_logging from cray_product_catalog.util import load_k8s +from cray_product_catalog.util.catalog_data_helper import format_product_cm_name +from cray_product_catalog.constants import ( + CONFIG_MAP_FIELDS, + PRODUCT_CM_FIELDS, +) urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) LOGGER = logging.getLogger(__name__) +# kubernetes API response code +ERR_NOT_FOUND = 404 +ERR_CONFLICT = 409 -def modify_config_map(name, namespace, product, product_version, key=None): +# retries +MAX_RETRIES = 100 +MAX_RETRIES_FOR_PROD_CM = 10 + + +class ModifyConfigMapUtil: + """Utility class to manage the ConfigMap modification + """ + + def __init__(self): + self.__main_cm = None + self.__product_cm = None + self.__cm_namespace = None + self.__product_name = None + self.__product_version = None + self.__max_retries_for_main_cm = None + self.__max_retries_for_prod_cm = None + self.__key = None + self.__main_cm_fields = None + self.__product_cm_fields = None + + # property definitions + @property + def main_cm(self): + return self.__main_cm + + @main_cm.setter + def main_cm(self, __main_cm): + self.__main_cm = __main_cm + + @property + def product_cm(self): + return self.__product_cm + + @product_cm.setter + def product_cm(self, __product_cm): + self.__product_cm = __product_cm + + @property + def cm_namespace(self): + return self.__cm_namespace + + @cm_namespace.setter + def cm_namespace(self, __cm_namespace): + self.__cm_namespace = __cm_namespace + + @property + def product_name(self): + return self.__product_name + + @product_name.setter + def product_name(self, __product_name): + self.__product_name = __product_name + + @property + def product_version(self): + return self.__product_version + + @product_version.setter + def product_version(self, __product_version): + self.__product_version = __product_version + + @property + def max_retries_for_main_cm(self): + return self.__max_retries_for_main_cm + + @max_retries_for_main_cm.setter + def max_retries_for_main_cm(self, __max_retries_for_main_cm): + self.__max_retries_for_main_cm = __max_retries_for_main_cm + + @property + def max_retries_for_prod_cm(self): + return self.__max_retries_for_prod_cm + + @max_retries_for_prod_cm.setter + def max_retries_for_prod_cm(self, __max_retries_for_prod_cm): + self.__max_retries_for_prod_cm = __max_retries_for_prod_cm + + @property + def key(self): + return self.__key + + @key.setter + def key(self, __key): + self.__key = __key + + @property + def main_cm_fields(self): + return self.__main_cm_fields + + @main_cm_fields.setter + def main_cm_fields(self, __main_cm_fields): + self.__main_cm_fields = __main_cm_fields + + @property + def product_cm_fields(self): + return self.__product_cm_fields + + @product_cm_fields.setter + def product_cm_fields(self, __product_cm_fields): + self.__product_cm_fields = __product_cm_fields + + # private methods + def __key_belongs_to_main_cm_fields(self): + return self.__key in self.__main_cm_fields + + def __key_belongs_to_prod_cm_fields(self): + return self.__key in self.__product_cm_fields + + def __modify_main_cm(self): + LOGGER.info("Removing from config_map=%s in namespace=%s for %s/%s (key=%s)", + self.main_cm, self.cm_namespace, self.product_name, self.product_version, self.key) + modify_config_map(self.__main_cm, self.__cm_namespace, self.__product_name, self.__product_version, + self.__key, self.__max_retries_for_main_cm, ) + + def __modify_product_cm(self): + LOGGER.info("Removing from config_map=%s in namespace=%s for %s/%s (key=%s)", + self.product_cm, self.cm_namespace, self.product_name, self.product_version, self.key) + modify_config_map(self.__product_cm, self.__cm_namespace, self.__product_name, self.__product_version, + self.__key, self.__max_retries_for_prod_cm, ) + + # public method + def modify(self): + """ + Method to initiate modification of ConfigMaps. + Before executing this method make sure to set these properties of the class: + * main_cm # Name of main ConfigMap + * product_cm # name of product-specific ConfigMap + * cm_namespace # Namespace containing all ConfigMaps + * product_name # Product name + * product_version # Product version + * max_retries_for_main_cm # Max failure retries for main ConfigMap + * max_retries_for_prod_cm # Max failure retries for product ConfigMap + * key # Key to delete; if you want to execute complete product or a particular version, ignore it + * main_cm_fields # Fields present in main ConfigMap + * product_cm_fields # Fields present in product-specific ConfigMap + """ + if self.__key: + if self.__key_belongs_to_main_cm_fields(): + self.__modify_main_cm() + + elif self.__key_belongs_to_prod_cm_fields(): + self.__modify_product_cm() + + else: + LOGGER.warning("key=%s NOT present in Main/Product ConfigMap, exiting", self.key) + + return + + self.__modify_main_cm() + self.__modify_product_cm() + + +def modify_config_map(name, namespace, product, product_version, key=None, max_attempts=MAX_RETRIES): """Remove a product version from the catalog ConfigMap. If a key is specified, delete the `key` content from a specific section @@ -74,7 +235,7 @@ def modify_config_map(name, namespace, product, product_version, key=None): 4. Repeat steps 2-3 if ConfigMap does not reflect the changes requested """ k8sclient = ApiClient() - retries = 100 + retries = max_attempts retry = Retry( total=retries, read=retries, connect=retries, backoff_factor=0.3, status_forcelist=(500, 502, 503, 504) @@ -82,7 +243,6 @@ def modify_config_map(name, namespace, product, product_version, key=None): k8sclient.rest_client.pool_manager.connection_pool_kw['retries'] = retry api_instance = client.CoreV1Api(k8sclient) attempt = 0 - max_attempts = 100 while True: @@ -101,7 +261,7 @@ def modify_config_map(name, namespace, product, product_version, key=None): LOGGER.exception("Error calling read_namespaced_config_map") # ConfigMap doesn't exist yet - if err.status == 404 and attempt < max_attempts: + if err.status == ERR_NOT_FOUND and attempt < max_attempts: LOGGER.warning("ConfigMap %s/%s doesn't exist, attempting again.", namespace, name) continue raise # unrecoverable @@ -155,27 +315,48 @@ def modify_config_map(name, namespace, product, product_version, key=None): name, namespace, client.V1ConfigMap(data=config_map_data) ) LOGGER.info("ConfigMap update attempt %s successful", attempt) - except ApiException: - LOGGER.exception("Error calling patch_namespaced_config_map") + except ApiException as exc: + if exc.status == ERR_CONFLICT: + # A conflict is raised if the resourceVersion field was unexpectedly + # incremented, e.g. if another process updated the ConfigMap. This + # provides concurrency protection. + LOGGER.warning("Conflict updating ConfigMap") + else: + LOGGER.exception("Error calling patch_namespaced_config_map") def main(): """ Main function """ + + # logging configuration configure_logging() + # Parameters to identify ConfigMap and product/version to remove PRODUCT = os.environ.get("PRODUCT").strip() # required PRODUCT_VERSION = os.environ.get("PRODUCT_VERSION").strip() # required - CONFIG_MAP = os.environ.get("CONFIG_MAP", "cray-product-catalog").strip() CONFIG_MAP_NS = os.environ.get("CONFIG_MAP_NAMESPACE", "services").strip() + CONFIG_MAP = os.environ.get("CONFIG_MAP", "cray-product-catalog").strip() + PRODUCT_CONFIG_MAP = format_product_cm_name(CONFIG_MAP, PRODUCT) KEY = os.environ.get("KEY", "").strip() or None - args = (CONFIG_MAP, CONFIG_MAP_NS, PRODUCT, PRODUCT_VERSION, KEY) - LOGGER.info( - "Removing from ConfigMap=%s in namespace=%s for %s/%s (key=%s)", - *args - ) + # k8 related configurations load_k8s() - modify_config_map(*args) + + # building the utility class + modify_config_map_util = ModifyConfigMapUtil() + modify_config_map_util.main_cm = CONFIG_MAP + modify_config_map_util.product_cm = PRODUCT_CONFIG_MAP + modify_config_map_util.cm_namespace = CONFIG_MAP_NS + modify_config_map_util.product_name = PRODUCT + modify_config_map_util.product_version = PRODUCT_VERSION + modify_config_map_util.max_retries_for_main_cm = MAX_RETRIES + modify_config_map_util.max_retries_for_prod_cm = MAX_RETRIES_FOR_PROD_CM + modify_config_map_util.key = KEY + modify_config_map_util.main_cm_fields = CONFIG_MAP_FIELDS + modify_config_map_util.product_cm_fields = PRODUCT_CM_FIELDS + + # Modifying ConfigMap + modify_config_map_util.modify() if __name__ == "__main__": diff --git a/cray_product_catalog/catalog_update.py b/cray_product_catalog/catalog_update.py index 96a95d6d..2907f5ea 100755 --- a/cray_product_catalog/catalog_update.py +++ b/cray_product_catalog/catalog_update.py @@ -2,7 +2,7 @@ # # MIT License # -# (C) Copyright 2020-2023 Hewlett Packard Enterprise Development LP +# (C) Copyright 2020-2024 Hewlett Packard Enterprise Development LP # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the "Software"), @@ -54,13 +54,16 @@ from cray_product_catalog.schema.validate import validate from cray_product_catalog.util.k8s import load_k8s from cray_product_catalog.util.merge_dict import merge_dict +from cray_product_catalog.util.catalog_data_helper import split_catalog_data, format_product_cm_name +from cray_product_catalog.constants import PRODUCT_CATALOG_CONFIG_MAP_LABEL + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) # Parameters to identify ConfigMap and content in it to update PRODUCT = os.environ.get("PRODUCT").strip() # required PRODUCT_VERSION = os.environ.get("PRODUCT_VERSION").strip() # required -CONFIG_MAP = os.environ.get("CONFIG_MAP", "cray-product-catalog").strip() +MAIN_CONFIG_MAP = os.environ.get("CONFIG_MAP", "cray-product-catalog").strip() CONFIG_MAP_NAMESPACE = os.environ.get("CONFIG_MAP_NAMESPACE", "services").strip() # One of (YAML_CONTENT_FILE, YAML_CONTENT_STRING) required. For backwards compatibility, YAML_CONTENT # may also be given in place of YAML_CONTENT_FILE. @@ -134,6 +137,21 @@ def active_field_exists(product_data): return any("active" in product_data[version] for version in product_data) +def create_config_map(api_instance, name, namespace): + """Create new product ConfigMap.""" + try: + new_cm = V1ConfigMap() + new_cm.metadata = V1ObjectMeta(name=name, labels=PRODUCT_CATALOG_CONFIG_MAP_LABEL) + api_instance.create_namespaced_config_map( + namespace=namespace, body=new_cm + ) + LOGGER.info("Created product ConfigMap %s/%s", namespace, name) + return True + except ApiException: + LOGGER.exception("Error calling create_namespaced_config_map") + return False + + def update_config_map(data: dict, name, namespace): """ Get the ConfigMap `data` to be added. @@ -155,7 +173,7 @@ def update_config_map(data: dict, name, namespace): api_instance = client.CoreV1Api(k8sclient) attempt = 0 - while True: + while attempt < retries: # Wait a while to check the ConfigMap in case multiple products are # attempting to update the same ConfigMap, or the ConfigMap doesn't @@ -172,10 +190,17 @@ def update_config_map(data: dict, name, namespace): LOGGER.exception("Error calling read_namespaced_config_map") # ConfigMap doesn't exist yet - if err.status == ERR_NOT_FOUND: + if err.status != ERR_NOT_FOUND: + raise # unrecoverable + if name == MAIN_CONFIG_MAP: + # If main ConfigMap is not found wait until it is available LOGGER.warning("ConfigMap %s/%s doesn't exist, attempting again", namespace, name) - continue - raise # unrecoverable + else: + # If product ConfigMap is not available then create + LOGGER.info("Product ConfigMap %s/%s doesn't exist, attempting to create", namespace, name) + if not create_config_map(api_instance, name, namespace): + raise # unrecoverable + continue # Determine if ConfigMap needs to be updated config_map_data = response.data or {} # if no ConfigMap data exists @@ -243,13 +268,17 @@ def update_config_map(data: dict, name, namespace): else: LOGGER.exception("Error calling replace_namespaced_config_map") + if attempt == retries: + LOGGER.error("Exceeded number of attempts; Not updating ConfigMap %s/%s.", namespace, name) + raise SystemExit(1) + def main(): """ Main function """ configure_logging() LOGGER.info( "Updating ConfigMap=%s in namespace=%s for product/version=%s/%s", - CONFIG_MAP, CONFIG_MAP_NAMESPACE, PRODUCT, PRODUCT_VERSION + MAIN_CONFIG_MAP, CONFIG_MAP_NAMESPACE, PRODUCT, PRODUCT_VERSION ) if SET_ACTIVE_VERSION and REMOVE_ACTIVE_FIELD: @@ -283,7 +312,20 @@ def main(): if VALIDATE_SCHEMA: validate_schema(data) - update_config_map(data, CONFIG_MAP, CONFIG_MAP_NAMESPACE) + product_config_map = format_product_cm_name(MAIN_CONFIG_MAP, PRODUCT) + + LOGGER.debug("Splitting cray-product-catalog data") + main_cm_data, prod_cm_data = split_catalog_data(data) + + if prod_cm_data and product_config_map == '': + LOGGER.error("Not updating ConfigMaps because the provided product name is invalid: '%s'", PRODUCT) + raise SystemExit(1) + + update_config_map(main_cm_data, MAIN_CONFIG_MAP, CONFIG_MAP_NAMESPACE) + + # If product_config_map is not an empty string and prod_cm_data is not an empty dict + if prod_cm_data: + update_config_map(prod_cm_data, product_config_map, CONFIG_MAP_NAMESPACE) if __name__ == "__main__": diff --git a/cray_product_catalog/constants.py b/cray_product_catalog/constants.py index 41db09cd..e152b3a9 100644 --- a/cray_product_catalog/constants.py +++ b/cray_product_catalog/constants.py @@ -1,6 +1,6 @@ # MIT License # -# (C) Copyright 2021-2023 Hewlett Packard Enterprise Development LP +# (C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the "Software"), @@ -33,3 +33,9 @@ COMPONENT_HELM = 'helm' COMPONENT_S3 = 's3' COMPONENT_MANIFESTS = 'manifests' +CONFIG_MAP_FIELDS = {'configuration', 'images', 'recipes'} +PRODUCT_CM_FIELDS = {'component_versions'} +PRODUCT_CATALOG_CONFIG_MAP_LABEL_KEY = 'type' +PRODUCT_CATALOG_CONFIG_MAP_LABEL = {PRODUCT_CATALOG_CONFIG_MAP_LABEL_KEY: PRODUCT_CATALOG_CONFIG_MAP_NAME} +PRODUCT_CATALOG_CONFIG_MAP_LABEL_STR = f"{PRODUCT_CATALOG_CONFIG_MAP_LABEL_KEY}={PRODUCT_CATALOG_CONFIG_MAP_NAME}" +PRODUCT_CATALOG_CONFIG_MAP_REPLICA = 'cray-product-catalog-temp' diff --git a/cray_product_catalog/migration/__init__.py b/cray_product_catalog/migration/__init__.py new file mode 100644 index 00000000..2d32c7da --- /dev/null +++ b/cray_product_catalog/migration/__init__.py @@ -0,0 +1,44 @@ +# MIT License +# +# (C) Copyright 2023-2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +""" +File defines few constants +""" + +import os +import re +from cray_product_catalog.constants import PRODUCT_CATALOG_CONFIG_MAP_LABEL_STR, PRODUCT_CATALOG_CONFIG_MAP_NAME + +# ConfigMap name for temporary main ConfigMap +CONFIG_MAP_TEMP = f"{PRODUCT_CATALOG_CONFIG_MAP_NAME}-temp" + +# namespace for ConfigMaps +PRODUCT_CATALOG_CONFIG_MAP_NAME = os.environ.get("CONFIG_MAP_NAME", "cray-product-catalog").strip() +PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE = os.environ.get("CONFIG_MAP_NAMESPACE", "services").strip() + +# ConfigMap names +CRAY_DATA_CATALOG_LABEL = PRODUCT_CATALOG_CONFIG_MAP_LABEL_STR + +# product ConfigMap pattern +PRODUCT_CONFIG_MAP_PATTERN = re.compile('^(cray-product-catalog)-([a-z0-9.-]+)$') +RESOURCE_VERSION = 'resource_version' + +RETRY_COUNT = 10 diff --git a/cray_product_catalog/migration/config_map_data_handler.py b/cray_product_catalog/migration/config_map_data_handler.py new file mode 100644 index 00000000..1ad5a4ae --- /dev/null +++ b/cray_product_catalog/migration/config_map_data_handler.py @@ -0,0 +1,181 @@ +# +# MIT License +# +# (C) Copyright 2023-2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + +""" +File contains functions to +Split data in `cray-product-catalog` ConfigMap +Create temporary and product ConfigMaps +Rename ConfigMap +""" + +import logging + +import yaml + +from cray_product_catalog.util.catalog_data_helper import split_catalog_data, format_product_cm_name +from cray_product_catalog.migration.kube_apis import KubernetesApi +from cray_product_catalog.constants import PRODUCT_CATALOG_CONFIG_MAP_LABEL +from cray_product_catalog.migration import ( + CONFIG_MAP_TEMP, PRODUCT_CATALOG_CONFIG_MAP_NAME, PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE +) + +LOGGER = logging.getLogger(__name__) + + +class ConfigMapDataHandler: + """ Class to migrate ConfigMap data to multiple ConfigMaps """ + + def __init__(self) -> None: + self.k8s_obj = KubernetesApi() + + def create_product_config_maps(self, product_config_map_data_list): + """Create new product ConfigMap for each product in product_config_map_data_list + + Args: + product_config_map_data_list (list): list of data to be stored in each product ConfigMap + """ + for product_data in product_config_map_data_list: + product_name = list(product_data.keys())[0] + LOGGER.debug("Creating ConfigMap for product %s", product_name) + prod_cm_name = format_product_cm_name(PRODUCT_CATALOG_CONFIG_MAP_NAME, product_name) + if prod_cm_name == '': + LOGGER.error("Failed to create ConfigMap %s/%s because the provided product name is invalid: '%s'", + PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE, prod_cm_name, product_name) + return False + + if not self.k8s_obj.create_config_map(prod_cm_name, PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE, product_data, + PRODUCT_CATALOG_CONFIG_MAP_LABEL): + LOGGER.info("Failed to create product ConfigMap %s/%s", PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE, + prod_cm_name) + return False + LOGGER.info("Created product ConfigMap %s/%s", PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE, prod_cm_name) + return True + + def create_temp_config_map(self, config_map_data): + """Create temporary main ConfigMap `cray-product-catalog-temp` + + Args: + config_map_data (dict): Data to be stored in the ConfigMap `cray-product-catalog-temp` + """ + + if self.k8s_obj.create_config_map(CONFIG_MAP_TEMP, PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE, + config_map_data, PRODUCT_CATALOG_CONFIG_MAP_LABEL): + LOGGER.info("Created temp ConfigMap %s/%s", + PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE, CONFIG_MAP_TEMP) + return True + LOGGER.error("Creating ConfigMap %s/%s failed", PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE, CONFIG_MAP_TEMP) + return False + + def migrate_config_map_data(self, config_map_data): + """Migrate cray-product-catalog ConfigMap data to multiple product ConfigMaps with + `component_versions` data for each product + + Returns: + {Dictionary, List}: Main ConfigMap Data, list of product ConfigMap data + """ + + LOGGER.info( + "Migrating data in ConfigMap=%s in namespace=%s to multiple ConfigMaps", + PRODUCT_CATALOG_CONFIG_MAP_NAME, PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE + ) + # Get list of products + products_list = list(config_map_data.keys()) + product_config_map_data_list = [] + for product in products_list: + product_data = yaml.safe_load(config_map_data[product]) + # Get list of versions associated with product + product_versions_list = list(product_data.keys()) + product_versions_data = {} + main_versions_data = {} + for version_data in product_versions_list: + LOGGER.debug("Splitting cray-product-catalog data for product %s", product) + main_cm_data, prod_cm_data = split_catalog_data(product_data[version_data]) + # prod_cm_data is not an empty dictionary + if prod_cm_data: + product_config_map_data = {} + product_versions_data[version_data] = prod_cm_data + # main_cm_data is not an empty dictionary + if main_cm_data: + main_versions_data[version_data] = main_cm_data + # If `component_versions` data exists for a product, create new product ConfigMap + if product_versions_data: + product_config_map_data = { + product: yaml.safe_dump(product_versions_data, default_flow_style=False) + } + product_config_map_data_list.append(product_config_map_data) + # Data with key other than `component_versions` should be updated to config_map_data, + # so that new main ConfigMap will not have data with key `component_versions` + if main_versions_data: + config_map_data[product] = yaml.safe_dump(main_versions_data, default_flow_style=False) + else: + config_map_data[product] = '' + return config_map_data, product_config_map_data_list + + def rename_config_map(self, rename_from, rename_to, namespace, label): + """ Renaming is actually deleting one ConfigMap and then updating the name of other ConfigMap and patching it. + :param str rename_from: Name of ConfigMap to rename + :param str rename_to: Name of ConfigMap after rename + :param str namespace: Namespace in which ConfigMap has to be updated + :param dict label: Label of ConfigMap to be renamed + :return: bool, If Success True else False + """ + + if not self.k8s_obj.delete_config_map(rename_to, namespace): + LOGGER.error("Failed to delete ConfigMap %s", rename_to) + return False + attempt = 0 + del_failed = False + + while attempt < 10: + attempt += 1 + response = self.k8s_obj.read_config_map(rename_from, namespace) + if not response: + LOGGER.error("Failed to read ConfigMap %s, retrying..", rename_from) + continue + cm_data = response.data + + if self.k8s_obj.create_config_map(rename_to, namespace, cm_data, label): + if self.k8s_obj.delete_config_map(rename_from, namespace): + LOGGER.info("Renaming ConfigMap successful") + return True + LOGGER.error("Failed to delete ConfigMap %s, retrying..", rename_from) + del_failed = True + break + LOGGER.error("Failed to create ConfigMap %s, retrying..", rename_to) + # Since only delete of backed up ConfigMap failed, retrying only delete operation + attempt = 0 + if del_failed: + while attempt < 10: + attempt += 1 + if self.k8s_obj.delete_config_map(rename_from, namespace): + del_failed = False + break + LOGGER.error("Failed to delete ConfigMap %s, retrying..", rename_from) + # Returning success as migration is successful only backed up ConfigMap is not deleted. + if del_failed: + LOGGER.info("Failed to delete ConfigMap %s, but migration is successful", rename_from) + else: + LOGGER.info("Renaming ConfigMap successful") + return True + return False diff --git a/cray_product_catalog/migration/exit_handler.py b/cray_product_catalog/migration/exit_handler.py new file mode 100644 index 00000000..a218c463 --- /dev/null +++ b/cray_product_catalog/migration/exit_handler.py @@ -0,0 +1,88 @@ +# MIT License +# +# (C) Copyright 2023-2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +""" +File contains logic to handle exit scenarios: +a. Graceful +b. Non-graceful +c. Rollback case +""" + +import logging +from typing import List +from re import fullmatch + + +from cray_product_catalog.migration import CRAY_DATA_CATALOG_LABEL, \ + PRODUCT_CONFIG_MAP_PATTERN +from cray_product_catalog.migration import PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE +from cray_product_catalog.migration.kube_apis import KubernetesApi + +LOGGER = logging.getLogger(__name__) + + +def _is_product_config_map(config_map_name: str) -> bool: + """Function to check product ConfigMap pattern. + Returns True if pattern match found + """ + if config_map_name is None or config_map_name == "": + return False + if fullmatch(PRODUCT_CONFIG_MAP_PATTERN, config_map_name): + return True + return False + + +class ExitHandler: + """Class to handle exit and rollback classes""" + + def __init__(self): + self.k8api = KubernetesApi() # Kubernetes API object + + def __get_all_created_product_config_maps(self) -> List: + """Get all created product ConfigMaps""" + cm_name = filter(_is_product_config_map, + self.k8api.list_config_map_names( + label=CRAY_DATA_CATALOG_LABEL, + namespace=PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE) + ) + return list(cm_name) + + def rollback(self): + """Method to handle roll back + Deleting temporary ConfigMap and all created product ConfigMaps + whose names are determined using the pattern PRODUCT_CONFIG_MAP_PATTERN + """ + LOGGER.warning("Initiating rollback") + product_config_maps = self.__get_all_created_product_config_maps() # collecting product ConfigMaps + + LOGGER.info("Deleting product ConfigMaps") # attempting to delete product ConfigMaps + non_deleted_product_config_maps = [] + for config_map in product_config_maps: + LOGGER.debug("Deleting product ConfigMap %s", config_map) + if not self.k8api.delete_config_map(name=config_map, namespace=PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE): + non_deleted_product_config_maps.append(config_map) + + if non_deleted_product_config_maps: # checking if any product ConfigMap is not deleted + LOGGER.error("Error deleting ConfigMaps: %s. Delete these manually", + non_deleted_product_config_maps) + return + + LOGGER.info("Rollback successful") diff --git a/cray_product_catalog/migration/kube_apis.py b/cray_product_catalog/migration/kube_apis.py new file mode 100644 index 00000000..6b9ed001 --- /dev/null +++ b/cray_product_catalog/migration/kube_apis.py @@ -0,0 +1,163 @@ +# +# MIT License +# +# (C) Copyright 2023-2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + +""" +Kubernets API +""" + +import logging +from kubernetes import client +from kubernetes.client.rest import ApiException +from kubernetes.client.api_client import ApiClient +from kubernetes.client.models.v1_config_map import V1ConfigMap +from kubernetes.client.models.v1_object_meta import V1ObjectMeta +from urllib3.util.retry import Retry +from urllib3.exceptions import MaxRetryError +from cray_product_catalog.logging import configure_logging +from cray_product_catalog.util.k8s import load_k8s +from . import RETRY_COUNT + + +class KubernetesApi: + """Class for wrapping Kubernetes API""" + def __init__(self): + configure_logging() + self.logger = logging.getLogger(__name__) + load_k8s() + + retry = Retry( + total=RETRY_COUNT, read=RETRY_COUNT, connect=RETRY_COUNT, backoff_factor=0.3, + status_forcelist=(500, 502, 503, 504) + ) + self.kclient = ApiClient() + self.kclient.rest_client.pool_manager.connection_pool_kw['retries'] = retry + self.api_instance = client.CoreV1Api(self.kclient) + + def create_config_map(self, name, namespace, data, label): + """Creates ConfigMap + :param dict data: Content of ConfigMap + :param str name: ConfigMap name to be created + :param str namespace: Namespace in which ConfigMap has to be created + :param dict label: Label with which ConfigMap has to be created + :return: bool + """ + try: + cm_body = V1ConfigMap( + metadata=V1ObjectMeta( + name=name, + labels=label + ), + data=data + ) + self.api_instance.create_namespaced_config_map( + namespace=namespace, body=cm_body + ) + return True + except MaxRetryError as err: + self.logger.exception('MaxRetryError: %s', err) + return False + except ApiException as err: + # The full string representation of ApiException is very long, so just log err.reason. + self.logger.exception('ApiException: %s', err.reason) + return False + + def list_config_map(self, namespace, label): + """ Reads all the ConfigMaps with certain label in particular namespace + :param str namespace: Value of namespace from where ConfigMap has to be listed + :param str label: String format of label "type=xyz" + :return: V1ConfigMapList + If there is any error, returns None + """ + if not all((label, namespace)): + self.logger.info("Either label or namespace is empty, not reading ConfigMap.") + return None + try: + return self.api_instance.list_namespaced_config_map(namespace, label_selector=label).items + except MaxRetryError as err: + self.logger.exception('MaxRetryError: %s', err) + return None + except ApiException as err: + # The full string representation of ApiException is very long, so just log err.reason. + self.logger.exception('ApiException: %s', err.reason) + return None + + def list_config_map_names(self, namespace, label): + """ Reads all the ConfigMaps with certain label in particular namespace + :param str namespace: Value of namespace from where ConfigMap has to be listed + :param str label: String format of label "type=xyz" + :return: [str] + """ + cm_output = self.list_config_map(namespace, label) + + list_cm_names = [] + + if not cm_output: + return list_cm_names + + # parse the output to get only names + for cm in cm_output: + try: + list_cm_names.append(cm.metadata.name) + except Exception: + continue + + return list_cm_names + + def read_config_map(self, name, namespace): + """Reads ConfigMap based on provided name and namespace + :param Str name: Name of ConfigMap to read + :param Str namespace: Namespace from which ConfigMap has to be read + :return: V1ConfigMap + Returns None in case of any error + """ + # Check if both values are not empty + if not all((name, namespace)): + self.logger.exception("Either name or namespace is empty, not reading ConfigMap.") + return None + try: + return self.api_instance.read_namespaced_config_map(name, namespace) + except MaxRetryError as err: + self.logger.exception('MaxRetryError: %s', err) + return None + except ApiException as err: + # The full string representation of ApiException is very long, so just log err.reason. + self.logger.exception('ApiException: %s', err.reason) + return None + + def delete_config_map(self, name, namespace): + """Delete the ConfigMap + :param Str name: Name of ConfigMap to be deleted + :param Str namespace: Namespace from which ConfigMap has to be deleted + :return: bool; If success True else False + """ + try: + self.api_instance.delete_namespaced_config_map(name, namespace) + return True + except MaxRetryError as err: + self.logger.exception('MaxRetryError: %s', err) + return False + except ApiException as err: + # The full string representation of ApiException is very long, so just log err.reason. + self.logger.exception('ApiException: %s', err.reason) + return False diff --git a/cray_product_catalog/migration/main.py b/cray_product_catalog/migration/main.py new file mode 100644 index 00000000..0a564ddd --- /dev/null +++ b/cray_product_catalog/migration/main.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +# +# MIT License +# +# (C) Copyright 2023-2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + +""" +This script splits the data in ConfigMap `cray-product-catalog` into multiple smaller +ConfigMaps with each product's `component_versions` data in its own product ConfigMap. +If the split is not succesful then it rollbacks to its initial state where ConfigMap +`cray-product-catalog` will contain complete data which includes `component-versions` +""" + +import logging + +from cray_product_catalog.constants import PRODUCT_CATALOG_CONFIG_MAP_LABEL +from cray_product_catalog.migration.config_map_data_handler import ConfigMapDataHandler +from cray_product_catalog.migration import ( + PRODUCT_CATALOG_CONFIG_MAP_NAME, + PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE, + CONFIG_MAP_TEMP +) +from cray_product_catalog.migration.exit_handler import ExitHandler + +LOGGER = logging.getLogger(__name__) + + +def main(): + """Main function""" + LOGGER.info("Migrating %s ConfigMap data to multiple product ConfigMaps", PRODUCT_CATALOG_CONFIG_MAP_NAME) + config_map_obj = ConfigMapDataHandler() + exit_handler = ExitHandler() + attempt = 0 + max_attempts = 2 + migration_failed = False + + while attempt < max_attempts: + attempt += 1 + migration_failed = False + curr_resource_version = '' + response = config_map_obj.k8s_obj.read_config_map( + PRODUCT_CATALOG_CONFIG_MAP_NAME, PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE + ) + if response: + if not response.metadata.resource_version: + LOGGER.error("Error reading resourceVersion, exiting migration process...") + raise SystemExit(1) + init_resource_version = response.metadata.resource_version + if not response.data: + LOGGER.error("Error reading ConfigMap data, exiting migration process...") + raise SystemExit(1) + config_map_data = response.data + else: + LOGGER.error("Error reading ConfigMap, exiting migration process...") + raise SystemExit(1) + + try: + main_config_map_data, product_config_map_data_list = config_map_obj.migrate_config_map_data(config_map_data) + except Exception: + LOGGER.error("Failed to split ConfigMap Data, exiting migration process...") + raise SystemExit(1) + + # Create ConfigMaps for each product with `component_versions` data + if not config_map_obj.create_product_config_maps(product_config_map_data_list): + LOGGER.info("Calling rollback handler...") + exit_handler.rollback() + raise SystemExit(1) + # Create temporary main ConfigMap with all data except `component_versions` for all products + if not config_map_obj.create_temp_config_map(main_config_map_data): + LOGGER.info("Calling rollback handler...") + exit_handler.rollback() + raise SystemExit(1) + + LOGGER.info("Verifying resource_version value is same to confirm there is no change in %s", + PRODUCT_CATALOG_CONFIG_MAP_NAME) + + response = config_map_obj.k8s_obj.read_config_map( + PRODUCT_CATALOG_CONFIG_MAP_NAME, PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE + ) + if response: + if not response.metadata.resource_version: + LOGGER.error("Error reading resourceVersion, exiting migration process...") + exit_handler.rollback() + raise SystemExit(1) + curr_resource_version = response.metadata.resource_version + + if curr_resource_version != init_resource_version: + migration_failed = True + LOGGER.info("resource_version has changed, so cannot rename %s ConfigMap to %s ConfigMap", + CONFIG_MAP_TEMP, PRODUCT_CATALOG_CONFIG_MAP_NAME) + LOGGER.info("Re-trying migration process...") + exit_handler.rollback() + continue + break + + if migration_failed: + LOGGER.info("ConfigMap %s is modified by other process, exiting migration process...", + PRODUCT_CATALOG_CONFIG_MAP_NAME) + raise SystemExit(1) + + LOGGER.info("Renaming %s ConfigMap name to %s ConfigMap", + CONFIG_MAP_TEMP, PRODUCT_CATALOG_CONFIG_MAP_NAME) + + # Creating main ConfigMap `cray-product-catalog` using the data in `cray-product-catalog-temp` + if config_map_obj.rename_config_map( + CONFIG_MAP_TEMP, PRODUCT_CATALOG_CONFIG_MAP_NAME, + PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE, PRODUCT_CATALOG_CONFIG_MAP_LABEL + ): + LOGGER.info("Migration successful") + else: + LOGGER.info("Renaming %s to %s ConfigMap failed, calling rollback handler...", + CONFIG_MAP_TEMP, PRODUCT_CATALOG_CONFIG_MAP_NAME) + exit_handler.rollback() + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/cray_product_catalog/query.py b/cray_product_catalog/query.py index 4988f1d3..3271e826 100644 --- a/cray_product_catalog/query.py +++ b/cray_product_catalog/query.py @@ -1,6 +1,6 @@ # MIT License # -# (C) Copyright 2021-2023 Hewlett Packard Enterprise Development LP +# (C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the "Software"), @@ -43,9 +43,11 @@ COMPONENT_VERSIONS_PRODUCT_MAP_KEY, PRODUCT_CATALOG_CONFIG_MAP_NAME, PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE, + PRODUCT_CATALOG_CONFIG_MAP_LABEL_STR ) from cray_product_catalog.schema.validate import validate from cray_product_catalog.util import load_k8s +from cray_product_catalog.util.merge_dict import merge_dict LOGGER = logging.getLogger(__name__) @@ -94,38 +96,59 @@ def __init__(self, name=PRODUCT_CATALOG_CONFIG_MAP_NAME, namespace=PRODUCT_CATAL self.namespace = namespace self.k8s_client = self._get_k8s_api() try: - config_map = self.k8s_client.read_namespaced_config_map(name, namespace) + configmaps = self.k8s_client.list_namespaced_config_map( + namespace, label_selector=PRODUCT_CATALOG_CONFIG_MAP_LABEL_STR + ).items except MaxRetryError as err: raise ProductCatalogError( - f'Unable to connect to Kubernetes to read {namespace}/{name} ConfigMap: {err}' + f'Unable to connect to Kubernetes to read {namespace} namespace: {err}' ) from err except ApiException as err: # The full string representation of ApiException is very long, so just log err.reason. raise ProductCatalogError( - f'Error reading {namespace}/{name} ConfigMap: {err.reason}' + f'Error listing ConfigMaps in {namespace} namespace: {err.reason}' ) from err - if config_map.data is None: - raise ProductCatalogError( - f'No data found in {namespace}/{name} ConfigMap.' + if len(configmaps) == 0: + LOGGER.info( + 'No ConfigMaps found in namespace %s with label %s."', + namespace, PRODUCT_CATALOG_CONFIG_MAP_LABEL_STR ) - - try: - self.products = [ - InstalledProductVersion(product_name, product_version, product_version_data) - for product_name, product_versions in config_map.data.items() - for product_version, product_version_data in safe_load(product_versions).items() - ] - except YAMLError as err: - raise ProductCatalogError( - f'Failed to load ConfigMap data: {err}' - ) from err + LOGGER.info('Getting data from ConfigMap %s/%s without label %s', + namespace, name, PRODUCT_CATALOG_CONFIG_MAP_LABEL_STR) + try: + config_map = self.k8s_client.read_namespaced_config_map(name, namespace) + except MaxRetryError as err: + raise ProductCatalogError( + f'Unable to connect to Kubernetes to read {namespace}/{name} ConfigMap: {err}' + ) from err + except ApiException as err: + raise ProductCatalogError( + f'Error reading {namespace}/{name} ConfigMap: {err.reason}' + ) from err + if config_map.data is None: + raise ProductCatalogError( + f'No data found in {namespace}/{name} ConfigMap.' + ) + try: + self.products = load_cm_data(config_map) + except YAMLError as err: + raise ProductCatalogError( + f'Failed to load ConfigMap data: {err}' + ) from err + else: + try: + self.products = load_config_map_data(self.name, configmaps) + except YAMLError as err: + raise ProductCatalogError( + f'Failed to load ConfigMap data: {err}' + ) from err invalid_products = [ str(p) for p in self.products if not p.is_valid ] if invalid_products: - LOGGER.debug( + LOGGER.warning( 'The following products have product catalog data that is not valid against the expected schema: %s', ", ".join(invalid_products) ) @@ -174,6 +197,49 @@ def get_product(self, name, version=None): return matching_products[0] +def load_cm_data(config_map): + """Parse read_namespaced_config_map output and get array of InstalledProductVersion objects. + + Args: + config_map (V1ConfigMap): ConfigMap object + + Returns: + An array of InstalledProductVersion objects. + """ + return [ + InstalledProductVersion(product_name, product_version, product_version_data) + for product_name, product_versions in config_map.data.items() + for product_version, product_version_data in safe_load(product_versions).items() + ] + + +def load_config_map_data(name, configmaps): + """Parse list_namespaced_config_map output and get array of InstalledProductVersion objects. + + Args: + name: Main ConfigMap name with which all product ConfigMaps name starts + configmaps (V1ConfigMapList): list of ConfigMap objects. + + Returns: + An array of InstalledProductVersion objects. + """ + config_map_data = {} + for cm in configmaps: + if not cm.metadata.name.startswith(name): + continue + for product_name, product_versions in cm.data.items(): + for product_version, product_version_data in safe_load(product_versions).items(): + cm_key = product_name + ':' + product_version + if cm_key in config_map_data: + config_map_data[cm_key] = merge_dict(config_map_data[cm_key], product_version_data) + else: + config_map_data[cm_key] = product_version_data + return [ + InstalledProductVersion(key.split(':',)[0], key.split(':')[1], product_version_data) + for key, product_version_data in config_map_data.items() + ] + + class InstalledProductVersion: """A representation of a version of a product that is currently installed. @@ -220,8 +286,9 @@ def docker_images(self): @property def helm_charts(self): """Get Helm charts associated with this InstalledProductVersion. + Returns: - A list of tuples of (chart name, chart version) + A list of tuples of (chart_name, chart_version) """ return [(component['name'], component['version']) for component in self.component_data.get(COMPONENT_HELM) or []] @@ -229,6 +296,7 @@ def helm_charts(self): @property def s3_artifacts(self): """Get S3 artifacts associated with this InstalledProductVersion. + Returns: A list of tuples of (artifact bucket, artifact key) """ @@ -238,6 +306,7 @@ def s3_artifacts(self): @property def loftsman_manifests(self): """Get Loftsman manifests associated with this InstalledProductVersion. + Returns: A list of manifests """ diff --git a/cray_product_catalog/schema/schema.yaml b/cray_product_catalog/schema/schema.yaml index a073ac13..b985a003 100644 --- a/cray_product_catalog/schema/schema.yaml +++ b/cray_product_catalog/schema/schema.yaml @@ -1,7 +1,7 @@ # # MIT License # -# (C) Copyright 2021-2023 Hewlett Packard Enterprise Development LP +# (C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the "Software"), @@ -176,4 +176,4 @@ definitions: items: description: Loftsman manifest artifact. type: string - minLength: 1 \ No newline at end of file + minLength: 1 diff --git a/cray_product_catalog/util/catalog_data_helper.py b/cray_product_catalog/util/catalog_data_helper.py new file mode 100755 index 00000000..51642b3d --- /dev/null +++ b/cray_product_catalog/util/catalog_data_helper.py @@ -0,0 +1,68 @@ +# MIT License +# +# (C) Copyright 2023-2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +""" +Contains a utility function for splitting `cray-product-catalog` data. +""" + +import re + +from cray_product_catalog.constants import ( + PRODUCT_CM_FIELDS +) + + +def split_catalog_data(data): + """Split the passed data into data needed by main and product ConfigMaps.""" + all_unique_keys = set(data.keys()) + comm_keys_bw_cms = all_unique_keys.intersection(PRODUCT_CM_FIELDS) + + # If none of the PRODUCT_CM_FIELDS are available in all_unique_keys, then + # return empty dict as second return value + if not comm_keys_bw_cms: + return {key: data[key] for key in all_unique_keys - PRODUCT_CM_FIELDS}, {} + return {key: data[key] for key in all_unique_keys - PRODUCT_CM_FIELDS}, \ + {key: data[key] for key in comm_keys_bw_cms} + + +def format_product_cm_name(config_map, product): + """Formatting PRODUCT_CONFIG_NAME based on the product name passed and the same is used as key + under data in the ConfigMap. + The name of a ConfigMap must be a valid DNS subdomain name. In addition, it must obey the following rules: + - contain no more than 253 characters + - contain only lowercase alphanumeric characters, hypens('-'), or periods('.') + - start with an alphanumeric character + - end with an alphanumeric character + The product name, which is a key under the data, must obey the following rule: + - contain only alphanumeric characters, hyphens ('-'), underscores ('_'), and periods ('.') + Because the product name can have uppercase characters and underscores ('_'), which are + prohibited in the ConfigMap name, we convert underscores ('_') to hyphens ('-') and uppercase + to lowercase. + """ + pat = re.compile('^([a-z0-9])*[a-z0-9.-]*([a-z0-9])$') + prod_config_map = config_map + '-' + product.replace('_', '-').lower() + + if len(prod_config_map) > 253: + return '' + if not re.fullmatch(pat, prod_config_map): + return '' + return prod_config_map diff --git a/setup.py b/setup.py index 43638ba6..baf0dec5 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ # # MIT License # -# (C) Copyright 2021-2022 Hewlett Packard Enterprise Development LP +# (C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the "Software"), @@ -60,7 +60,8 @@ entry_points={ 'console_scripts': [ 'catalog_delete=cray_product_catalog.catalog_delete:main', - 'catalog_update=cray_product_catalog.catalog_update:main' + 'catalog_update=cray_product_catalog.catalog_update:main', + 'catalog_migrate=cray_product_catalog.migration.main:main', ] } ) diff --git a/tests/migration/__init__.py b/tests/migration/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/migration/migration_mock.py b/tests/migration/migration_mock.py new file mode 100755 index 00000000..3fe74fc6 --- /dev/null +++ b/tests/migration/migration_mock.py @@ -0,0 +1,130 @@ +# +# MIT License +# +# (C) Copyright 2023-2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + +""" +Mock data for ConfigMapDataHandler +""" + +INITIAL_MAIN_CM_DATA = { + 'HFP-firmware': """ + 22.10.2: + component_versions: + docker: + - name: cray-product-catalog-update + version: 0.1.3 + 23.01.1: + component_versions: + docker: + - name: cray-product-catalog-update + version: 0.1.3""", + 'analytics': """ + 1.4.18: + component_versions: + s3: + - bucket: boot-images + key: Analytics/Cray-Analytics.x86_64-1.4.18.squashfs + configuration: + clone_url: https://vcs.cmn.lemondrop.hpc.amslabs.hpecorp.net/vcs/cray/analytics-config-management.git + commit: 4f1aee2086b58b319d4a9ee167086004fca09e47 + import_branch: cray/analytics/1.4.18 + import_date: 2023-02-28 04:37:34.914586 + ssh_url: git@vcs.cmn.lemondrop.hpc.amslabs.hpecorp.net:cray/analytics-config-management.git + 1.4.20: + component_versions: + s3: + - bucket: boot-images + key: Analytics/Cray-Analytics.x86_64-1.4.20.squashfs + configuration: + clone_url: https://vcs.cmn.lemondrop.hpc.amslabs.hpecorp.net/vcs/cray/analytics-config-management.git + commit: 8424f5f97f12a3403afc57ac55deca0dadc8f3dd + import_branch: cray/analytics/1.4.20 + import_date: 2023-03-23 16:55:22.295666 + ssh_url: git@vcs.cmn.lemondrop.hpc.amslabs.hpecorp.net:cray/analytics-config-management.git""" +} + +MAIN_CM_DATA_EXPECTED = { + 'HFP-firmware': '', + 'analytics': """1.4.18: + configuration: + clone_url: https://vcs.cmn.lemondrop.hpc.amslabs.hpecorp.net/vcs/cray/analytics-config-management.git + commit: 4f1aee2086b58b319d4a9ee167086004fca09e47 + import_branch: cray/analytics/1.4.18 + import_date: 2023-02-28 04:37:34.914586 + ssh_url: git@vcs.cmn.lemondrop.hpc.amslabs.hpecorp.net:cray/analytics-config-management.git +1.4.20: + configuration: + clone_url: https://vcs.cmn.lemondrop.hpc.amslabs.hpecorp.net/vcs/cray/analytics-config-management.git + commit: 8424f5f97f12a3403afc57ac55deca0dadc8f3dd + import_branch: cray/analytics/1.4.20 + import_date: 2023-03-23 16:55:22.295666 + ssh_url: git@vcs.cmn.lemondrop.hpc.amslabs.hpecorp.net:cray/analytics-config-management.git\n""" +} + +PROD_CM_DATA_LIST_EXPECTED = [ + { + 'HFP-firmware': """22.10.2: + component_versions: + docker: + - name: cray-product-catalog-update + version: 0.1.3 +23.01.1: + component_versions: + docker: + - name: cray-product-catalog-update + version: 0.1.3\n""" + }, + { + 'analytics': """1.4.18: + component_versions: + s3: + - bucket: boot-images + key: Analytics/Cray-Analytics.x86_64-1.4.18.squashfs +1.4.20: + component_versions: + s3: + - bucket: boot-images + key: Analytics/Cray-Analytics.x86_64-1.4.20.squashfs\n""" + } +] + + +class MockYaml: + """Mock class created to test test_create_product_catalog_invalid_product_data.""" + + def __init__(self, resource_version): + """Initialize metadata and data object of ConfigMap data.""" + self.metadata = MetaData(resource_version) + self.data = { + 'sat': '\t', + } + + +class MetaData: + """ + Class to provide dummy metadata object with name and resource_version + """ + def __init__(self, resource_version): + """Initialize ConfigMap name and resoource_version""" + self.name = 'cray-product-catalog' + self.resource_version = resource_version diff --git a/tests/migration/test_config_map_data_handler.py b/tests/migration/test_config_map_data_handler.py new file mode 100755 index 00000000..487d9b40 --- /dev/null +++ b/tests/migration/test_config_map_data_handler.py @@ -0,0 +1,672 @@ +# +# MIT License +# +# (C) Copyright 2023-2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + +""" +Tests for validating ConfigMapDataHandler +""" + +import unittest +from unittest.mock import patch, call, Mock +from typing import Dict, List + +from cray_product_catalog.migration.main import main +from cray_product_catalog.migration.config_map_data_handler import ConfigMapDataHandler +from cray_product_catalog.constants import ( + PRODUCT_CATALOG_CONFIG_MAP_NAME, PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE, + PRODUCT_CATALOG_CONFIG_MAP_LABEL +) +from cray_product_catalog.migration import CONFIG_MAP_TEMP +from tests.migration.migration_mock import ( + MAIN_CM_DATA_EXPECTED, PROD_CM_DATA_LIST_EXPECTED, INITIAL_MAIN_CM_DATA, MockYaml +) + + +def mock_split_catalog_data(): + """Mocking function to return custom data""" + return MAIN_CM_DATA_EXPECTED, PROD_CM_DATA_LIST_EXPECTED + + +class TestConfigMapDataHandler(unittest.TestCase): + """ Tests for validating ConfigMapDataHandler """ + + def setUp(self) -> None: + """Set up mocks.""" + self.mock_load_k8s_mig = patch('cray_product_catalog.migration.kube_apis.load_k8s').start() + self.mock_corev1api_mig = patch('cray_product_catalog.migration.kube_apis.client.CoreV1Api').start() + self.mock_ApiClient_mig = patch('cray_product_catalog.migration.kube_apis.ApiClient').start() + self.mock_client_mig = patch('cray_product_catalog.migration.kube_apis.client').start() + + self.mock_k8api_read = patch( + 'cray_product_catalog.migration.config_map_data_handler.KubernetesApi.read_config_map').start() + self.mock_k8api_create = patch( + 'cray_product_catalog.migration.config_map_data_handler.KubernetesApi.create_config_map').start() + self.mock_k8api_delete = patch( + 'cray_product_catalog.migration.config_map_data_handler.KubernetesApi.delete_config_map').start() + + def tearDown(self) -> None: + patch.stopall() + + def test_migrate_config_map_data(self): + """ Validating the migration of data into multiple product ConfigMaps data """ + + main_cm_data: Dict + prod_cm_data_list: List + cmdh = ConfigMapDataHandler() + main_cm_data, prod_cm_data_list = cmdh.migrate_config_map_data(INITIAL_MAIN_CM_DATA) + + self.assertEqual(main_cm_data, MAIN_CM_DATA_EXPECTED) + self.assertEqual(prod_cm_data_list, PROD_CM_DATA_LIST_EXPECTED) + + def test_create_product_config_maps(self): + """ Validating product ConfigMaps are created """ + + # mock some additional functions + self.mock_v1_object_Meta_mig = patch('cray_product_catalog.migration.kube_apis.V1ObjectMeta').start() + + with self.assertLogs() as captured: + # call method under test + cmdh = ConfigMapDataHandler() + cmdh.create_product_config_maps(PROD_CM_DATA_LIST_EXPECTED) + + dummy_prod_cm_names = ['cray-product-catalog-hfp-firmware', 'cray-product-catalog-analytics'] + + self.mock_k8api_create.assert_has_calls(calls=[ # Create ConfigMap called twice + call( + dummy_prod_cm_names[0], PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE, + PROD_CM_DATA_LIST_EXPECTED[0], PRODUCT_CATALOG_CONFIG_MAP_LABEL), call().__bool__(), + call( + dummy_prod_cm_names[1], PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE, + PROD_CM_DATA_LIST_EXPECTED[1], PRODUCT_CATALOG_CONFIG_MAP_LABEL), call().__bool__(), + ] + ) + + # Verify the exact log message + self.assertEqual( + captured.records[0].getMessage(), + f"Created product ConfigMap {PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE}/{dummy_prod_cm_names[0]}") + + self.assertEqual( + captured.records[1].getMessage(), + f"Created product ConfigMap {PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE}/{dummy_prod_cm_names[1]}") + + def test_create_second_product_config_map_failed(self): + """ Validating scenario where creation of second product ConfigMap failed """ + + # mock some additional functions + self.mock_v1_object_Meta_mig = patch('cray_product_catalog.migration.kube_apis.V1ObjectMeta').start() + + with self.assertLogs() as captured: + self.mock_k8api_create.side_effect = [True, False] + + # call method under test + cmdh = ConfigMapDataHandler() + cmdh.create_product_config_maps(PROD_CM_DATA_LIST_EXPECTED) + + dummy_prod_cm_names = ['cray-product-catalog-hfp-firmware', 'cray-product-catalog-analytics'] + + self.mock_k8api_create.assert_has_calls(calls=[ # Create ConfigMap called twice + call( + dummy_prod_cm_names[0], PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE, + PROD_CM_DATA_LIST_EXPECTED[0], PRODUCT_CATALOG_CONFIG_MAP_LABEL), + call( + dummy_prod_cm_names[1], PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE, + PROD_CM_DATA_LIST_EXPECTED[1], PRODUCT_CATALOG_CONFIG_MAP_LABEL), + ] + ) + + # Verify the exact log message + self.assertEqual( + captured.records[0].getMessage(), + f"Created product ConfigMap {PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE}/{dummy_prod_cm_names[0]}") + + self.assertEqual( + captured.records[1].getMessage(), + f"Failed to create product ConfigMap {PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE}/{dummy_prod_cm_names[1]}") + + def test_create_first_product_config_map_failed(self): + """ Validating scenario where creation of first product ConfigMap failed. """ + + # mock some additional functions + self.mock_v1_object_Meta_mig = patch('cray_product_catalog.migration.kube_apis.V1ObjectMeta').start() + + with self.assertLogs() as captured: + self.mock_k8api_create.side_effect = [False, False] + + # call method under test + cmdh = ConfigMapDataHandler() + cmdh.create_product_config_maps(PROD_CM_DATA_LIST_EXPECTED) + + dummy_prod_cm_names = ['cray-product-catalog-hfp-firmware', 'cray-product-catalog-analytics'] + + self.mock_k8api_create.assert_called_once_with( + dummy_prod_cm_names[0], PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE, + PROD_CM_DATA_LIST_EXPECTED[0], PRODUCT_CATALOG_CONFIG_MAP_LABEL + ) + + # Verify the exact log message + self.assertEqual(len(captured.records), 1) + + self.assertEqual( + captured.records[0].getMessage(), + f"Failed to create product ConfigMap {PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE}/{dummy_prod_cm_names[0]}") + + def test_create_temp_config_map(self): + """ Validating temp main ConfigMap is created """ + + # mock some additional functions + self.mock_v1_object_Meta_mig = patch('cray_product_catalog.migration.kube_apis.V1ObjectMeta').start() + + with self.assertLogs(level='DEBUG') as captured: + # call method under test + cmdh = ConfigMapDataHandler() + cmdh.create_temp_config_map(MAIN_CM_DATA_EXPECTED) + + self.mock_k8api_create.assert_called_once_with( + CONFIG_MAP_TEMP, PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE, + MAIN_CM_DATA_EXPECTED, PRODUCT_CATALOG_CONFIG_MAP_LABEL + ) + + # Verify the exact log message + self.assertEqual( + captured.records[0].getMessage(), + f"Created temp ConfigMap {PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE}/{CONFIG_MAP_TEMP}") + + def test_create_temp_config_map_failed(self): + """ Validating temp main ConfigMap creation failed """ + + # mock some additional functions + self.mock_v1_object_Meta_mig = patch('cray_product_catalog.migration.kube_apis.V1ObjectMeta').start() + self.mock_k8api_create.return_value = False + + with self.assertLogs() as captured: + # call method under test + cmdh = ConfigMapDataHandler() + cmdh.create_temp_config_map(MAIN_CM_DATA_EXPECTED) + + self.mock_k8api_create.assert_called_once_with( + CONFIG_MAP_TEMP, PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE, + MAIN_CM_DATA_EXPECTED, PRODUCT_CATALOG_CONFIG_MAP_LABEL + ) + + # Verify the exact log message + self.assertEqual( + captured.records[0].getMessage(), + f"Creating ConfigMap {PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE}/{CONFIG_MAP_TEMP} failed") + + def test_rename_config_map(self): + """ Validating product ConfigMaps are created """ + + with self.assertLogs(level="DEBUG") as captured: + # call method under test + self.mock_k8api_delete.return_value = True + self.mock_k8api_read.return_value = Mock(data=MAIN_CM_DATA_EXPECTED) + + cmdh = ConfigMapDataHandler() + cmdh.rename_config_map(rename_from=CONFIG_MAP_TEMP, + rename_to=PRODUCT_CATALOG_CONFIG_MAP_NAME, + namespace=PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE, + label=PRODUCT_CATALOG_CONFIG_MAP_LABEL) + + self.mock_k8api_delete.assert_has_calls(calls=[ # Delete ConfigMap called twice + call(PRODUCT_CATALOG_CONFIG_MAP_NAME, + PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE), + call(CONFIG_MAP_TEMP, + PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE), + ] + ) + + self.mock_k8api_read.assert_called_once_with(CONFIG_MAP_TEMP, + PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE) + + self.mock_k8api_create.assert_called_once_with(PRODUCT_CATALOG_CONFIG_MAP_NAME, + PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE, + MAIN_CM_DATA_EXPECTED, + PRODUCT_CATALOG_CONFIG_MAP_LABEL) + + # Verify the exact log message + self.assertEqual( + captured.records[0].getMessage(), + "Renaming ConfigMap successful") + + def test_rename_config_map_failed_1(self): + """ Validating rename ConfigMap failure scenario where: + deleting cray-product-catalog ConfigMap failed. """ + + with self.assertLogs() as captured: + self.mock_k8api_delete.side_effect = [False, False] + # call method under test + cmdh = ConfigMapDataHandler() + cmdh.rename_config_map(CONFIG_MAP_TEMP, PRODUCT_CATALOG_CONFIG_MAP_NAME, + PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE, + PRODUCT_CATALOG_CONFIG_MAP_LABEL) + + self.mock_k8api_delete.assert_called_once_with(PRODUCT_CATALOG_CONFIG_MAP_NAME, + PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE) + + # Verify the exact log message + self.assertEqual( + captured.records[0].getMessage(), + f"Failed to delete ConfigMap {PRODUCT_CATALOG_CONFIG_MAP_NAME}") + + def test_rename_config_map_failed_2(self): + """ Validating rename ConfigMap failure scenario where: + creating cray-product-catalog ConfigMap failed. """ + + with self.assertLogs(level="DEBUG") as captured: + self.mock_k8api_create.return_value = False + self.mock_k8api_read.return_value = Mock(data=MAIN_CM_DATA_EXPECTED) + + # call method under test + cmdh = ConfigMapDataHandler() + cmdh.rename_config_map(CONFIG_MAP_TEMP, + PRODUCT_CATALOG_CONFIG_MAP_NAME, + PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE, + PRODUCT_CATALOG_CONFIG_MAP_LABEL) + + self.mock_k8api_delete.assert_called_once_with(PRODUCT_CATALOG_CONFIG_MAP_NAME, + PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE) + calls = [ # Delete ConfigMap called twice + call( + PRODUCT_CATALOG_CONFIG_MAP_NAME, + PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE), + call( + CONFIG_MAP_TEMP, + PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE), + ] + self.mock_k8api_create.assert_has_calls(calls=[ + call(PRODUCT_CATALOG_CONFIG_MAP_NAME, PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE, + MAIN_CM_DATA_EXPECTED, PRODUCT_CATALOG_CONFIG_MAP_LABEL), + call(PRODUCT_CATALOG_CONFIG_MAP_NAME, PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE, + MAIN_CM_DATA_EXPECTED, PRODUCT_CATALOG_CONFIG_MAP_LABEL), + call(PRODUCT_CATALOG_CONFIG_MAP_NAME, PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE, + MAIN_CM_DATA_EXPECTED, PRODUCT_CATALOG_CONFIG_MAP_LABEL), + call(PRODUCT_CATALOG_CONFIG_MAP_NAME, PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE, + MAIN_CM_DATA_EXPECTED, PRODUCT_CATALOG_CONFIG_MAP_LABEL), + call(PRODUCT_CATALOG_CONFIG_MAP_NAME, PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE, + MAIN_CM_DATA_EXPECTED, PRODUCT_CATALOG_CONFIG_MAP_LABEL), + call(PRODUCT_CATALOG_CONFIG_MAP_NAME, PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE, + MAIN_CM_DATA_EXPECTED, PRODUCT_CATALOG_CONFIG_MAP_LABEL), + call(PRODUCT_CATALOG_CONFIG_MAP_NAME, PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE, + MAIN_CM_DATA_EXPECTED, PRODUCT_CATALOG_CONFIG_MAP_LABEL), + call(PRODUCT_CATALOG_CONFIG_MAP_NAME, PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE, + MAIN_CM_DATA_EXPECTED, PRODUCT_CATALOG_CONFIG_MAP_LABEL), + call(PRODUCT_CATALOG_CONFIG_MAP_NAME, PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE, + MAIN_CM_DATA_EXPECTED, PRODUCT_CATALOG_CONFIG_MAP_LABEL), + call(PRODUCT_CATALOG_CONFIG_MAP_NAME, PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE, + MAIN_CM_DATA_EXPECTED, PRODUCT_CATALOG_CONFIG_MAP_LABEL), + + ]) + + # Verify the exact log message + self.assertEqual( + captured.records[0].getMessage(), + f"Failed to create ConfigMap {PRODUCT_CATALOG_CONFIG_MAP_NAME}, retrying..") + + def test_rename_config_map_failed_3(self): + """ Validating rename ConfigMap failure scenario where: + first operation of deleting cray-product-catalog-temp ConfigMap failed but later passed. """ + + with self.assertLogs(level="DEBUG") as captured: + self.mock_k8api_read.return_value = Mock(data=MAIN_CM_DATA_EXPECTED) + self.mock_k8api_delete.side_effect = [True, False, True] + + # call method under test + cmdh = ConfigMapDataHandler() + cmdh.rename_config_map(CONFIG_MAP_TEMP, + PRODUCT_CATALOG_CONFIG_MAP_NAME, + PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE, + PRODUCT_CATALOG_CONFIG_MAP_LABEL) + + self.mock_k8api_delete.assert_has_calls(calls=[ # Delete ConfigMap called twice + call( + PRODUCT_CATALOG_CONFIG_MAP_NAME, + PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE), + call( + CONFIG_MAP_TEMP, + PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE), + ]) + self.mock_k8api_create.assert_called_once_with(PRODUCT_CATALOG_CONFIG_MAP_NAME, + PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE, + MAIN_CM_DATA_EXPECTED, PRODUCT_CATALOG_CONFIG_MAP_LABEL) + + # Verify the exact log message + self.assertEqual(2, len(captured.records)) + self.assertEqual( + captured.records[0].getMessage(), + f"Failed to delete ConfigMap {CONFIG_MAP_TEMP}, retrying..") + self.assertEqual( + captured.records[1].getMessage(), + "Renaming ConfigMap successful") + + def test_rename_config_map_failed_4(self): + """ Validating rename ConfigMap failure scenario where: + everytime deleting cray-product-catalog-temp ConfigMap failed. """ + + with self.assertLogs(level="DEBUG") as captured: + self.mock_k8api_read.return_value = Mock(data=MAIN_CM_DATA_EXPECTED) + self.mock_k8api_delete.side_effect = [True, False, False, False, False, False, False, False, + False, False, False, False] + + # call method under test + cmdh = ConfigMapDataHandler() + cmdh.rename_config_map(CONFIG_MAP_TEMP, + PRODUCT_CATALOG_CONFIG_MAP_NAME, + PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE, + PRODUCT_CATALOG_CONFIG_MAP_LABEL) + + self.mock_k8api_delete.assert_has_calls(calls=[ + call( + PRODUCT_CATALOG_CONFIG_MAP_NAME, + PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE), + call( + CONFIG_MAP_TEMP, + PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE), + ]) + self.mock_k8api_create.assert_called_once_with(PRODUCT_CATALOG_CONFIG_MAP_NAME, + PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE, + MAIN_CM_DATA_EXPECTED, PRODUCT_CATALOG_CONFIG_MAP_LABEL) + + # Verify the exact log message + self.assertEqual(12, len(captured.records)) + self.assertEqual( + captured.records[-2].getMessage(), + f"Failed to delete ConfigMap {CONFIG_MAP_TEMP}, retrying..") + self.assertEqual( + captured.records[-1].getMessage(), + f"Failed to delete ConfigMap {CONFIG_MAP_TEMP}, but migration is successful") + + def test_main_for_successful_migration(self): + """Validating that migration is successful""" + self.mock_migrate_config_map = patch( + 'cray_product_catalog.migration.config_map_data_handler.ConfigMapDataHandler.migrate_config_map_data' + ).start() + self.mock_create_prod_cms = patch( + 'cray_product_catalog.migration.config_map_data_handler.ConfigMapDataHandler.create_product_config_maps' + ).start() + self.mock_create_temp_cm = patch( + 'cray_product_catalog.migration.config_map_data_handler.ConfigMapDataHandler.create_temp_config_map' + ).start() + self.mock_rename_cm = patch( + 'cray_product_catalog.migration.config_map_data_handler.ConfigMapDataHandler.rename_config_map' + ).start() + + with self.assertLogs(level="DEBUG") as captured: + self.mock_k8api_read.return_value = Mock(data=MAIN_CM_DATA_EXPECTED) + self.mock_migrate_config_map.return_value = mock_split_catalog_data() + self.mock_create_prod_cms.return_value = True + self.mock_create_temp_cm.return_value = True + self.mock_rename_cm.return_value = True + + # Call method under test + main() + + self.assertEqual( + captured.records[-1].getMessage(), + "Migration successful") + + def test_main_failed_1(self): + """Validating that migration failed as renaming failed""" + + self.mock_migrate_config_map = patch( + 'cray_product_catalog.migration.config_map_data_handler.ConfigMapDataHandler.migrate_config_map_data' + ).start() + self.mock_create_prod_cms = patch( + 'cray_product_catalog.migration.config_map_data_handler.ConfigMapDataHandler.create_product_config_maps' + ).start() + self.mock_create_temp_cm = patch( + 'cray_product_catalog.migration.config_map_data_handler.ConfigMapDataHandler.create_temp_config_map' + ).start() + self.mock_rename_cm = patch( + 'cray_product_catalog.migration.config_map_data_handler.ConfigMapDataHandler.rename_config_map' + ).start() + + with self.assertRaises(SystemExit) as captured: + self.mock_k8api_read.return_value = Mock(data=MAIN_CM_DATA_EXPECTED) + self.mock_migrate_config_map.return_value = mock_split_catalog_data() + self.mock_create_prod_cms.return_value = True + self.mock_create_temp_cm.return_value = True + self.mock_rename_cm.return_value = False + + # Call method under test + main() + + self.assertTrue( + "Renaming cray-product-catalog-temp to cray-product-catalog ConfigMap failed, " + "calling rollback handler..." in captured.exception + ) + + self.assertTrue( + "Rollback successful" in captured.exception + ) + + def test_main_failed_2(self): + """Validating that migration failed as create temp cm failed""" + + self.mock_migrate_config_map = patch( + 'cray_product_catalog.migration.config_map_data_handler.ConfigMapDataHandler.migrate_config_map_data' + ).start() + self.mock_create_prod_cms = patch( + 'cray_product_catalog.migration.config_map_data_handler.ConfigMapDataHandler.create_product_config_maps' + ).start() + self.mock_create_temp_cm = patch( + 'cray_product_catalog.migration.config_map_data_handler.ConfigMapDataHandler.create_temp_config_map' + ).start() + self.mock_rename_cm = patch( + 'cray_product_catalog.migration.config_map_data_handler.ConfigMapDataHandler.rename_config_map' + ).start() + + with self.assertRaises(SystemExit) as captured: + self.mock_k8api_read.return_value = Mock(data=MAIN_CM_DATA_EXPECTED) + self.mock_migrate_config_map.return_value = mock_split_catalog_data() + self.mock_create_prod_cms.return_value = True + self.mock_create_temp_cm.return_value = False + + # Call method under test + main() + + self.assertTrue( + "Rollback successful" in captured.exception + ) + + self.mock_rename_cm.assert_not_called() + + def test_main_failed_3(self): + """Validating that migration failed as create product ConfigMaps failed""" + + self.mock_migrate_config_map = patch( + 'cray_product_catalog.migration.config_map_data_handler.ConfigMapDataHandler.migrate_config_map_data' + ).start() + self.mock_create_prod_cms = patch( + 'cray_product_catalog.migration.config_map_data_handler.ConfigMapDataHandler.create_product_config_maps' + ).start() + self.mock_create_temp_cm = patch( + 'cray_product_catalog.migration.config_map_data_handler.ConfigMapDataHandler.create_temp_config_map' + ).start() + self.mock_rename_cm = patch( + 'cray_product_catalog.migration.config_map_data_handler.ConfigMapDataHandler.rename_config_map' + ).start() + + with self.assertRaises(SystemExit) as captured: + self.mock_k8api_read.return_value = Mock(data=MAIN_CM_DATA_EXPECTED) + self.mock_migrate_config_map.return_value = mock_split_catalog_data() + self.mock_create_prod_cms.return_value = False + + # Call method under test + main() + + self.assertTrue( + "Rollback successful" in captured.exception + ) + + self.mock_create_temp_cm.assert_not_called() + self.mock_rename_cm.assert_not_called() + + def test_main_failed_4(self): + """Validating that migration failed as migrate_config_map failed with exception""" + + self.mock_migrate_config_map = patch( + 'cray_product_catalog.migration.config_map_data_handler.ConfigMapDataHandler.migrate_config_map_data' + ).start() + self.mock_create_prod_cms = patch( + 'cray_product_catalog.migration.config_map_data_handler.ConfigMapDataHandler.create_product_config_maps' + ).start() + self.mock_create_temp_cm = patch( + 'cray_product_catalog.migration.config_map_data_handler.ConfigMapDataHandler.create_temp_config_map' + ).start() + self.mock_rename_cm = patch( + 'cray_product_catalog.migration.config_map_data_handler.ConfigMapDataHandler.rename_config_map' + ).start() + + with self.assertRaises(SystemExit) as captured: + self.mock_k8api_read.return_value = Mock(data=MAIN_CM_DATA_EXPECTED) + self.mock_migrate_config_map.return_value = Exception() + + # Call method under test + main() + + self.assertTrue( + "Failed to split ConfigMap Data, exiting migration process..." in captured.exception + ) + + self.mock_create_prod_cms.assert_not_called() + self.mock_create_temp_cm.assert_not_called() + self.mock_rename_cm.assert_not_called() + + def test_main_failed_6(self): + """Validating that migration failed as read_config_map returned empty response / data.""" + + self.mock_migrate_config_map = patch( + 'cray_product_catalog.migration.config_map_data_handler.ConfigMapDataHandler.migrate_config_map_data' + ).start() + self.mock_create_prod_cms = patch( + 'cray_product_catalog.migration.config_map_data_handler.ConfigMapDataHandler.create_product_config_maps' + ).start() + self.mock_create_temp_cm = patch( + 'cray_product_catalog.migration.config_map_data_handler.ConfigMapDataHandler.create_temp_config_map' + ).start() + self.mock_rename_cm = patch( + 'cray_product_catalog.migration.config_map_data_handler.ConfigMapDataHandler.rename_config_map' + ).start() + + with self.assertRaises(SystemExit) as captured: + self.mock_k8api_read.return_value = Mock(data="") + + # Call method under test + main() + + self.assertTrue( + "Error reading ConfigMap, exiting migration process..." in captured.exception + ) + + self.mock_migrate_config_map.assert_not_called() + self.mock_create_prod_cms.assert_not_called() + self.mock_create_temp_cm.assert_not_called() + self.mock_rename_cm.assert_not_called() + + def test_main_failed_7(self): + """Validating that migration failed as initial and final resource version is different""" + + self.mock_migrate_config_map = patch( + 'cray_product_catalog.migration.config_map_data_handler.ConfigMapDataHandler.migrate_config_map_data' + ).start() + self.mock_create_prod_cms = patch( + 'cray_product_catalog.migration.config_map_data_handler.ConfigMapDataHandler.create_product_config_maps' + ).start() + self.mock_create_temp_cm = patch( + 'cray_product_catalog.migration.config_map_data_handler.ConfigMapDataHandler.create_temp_config_map' + ).start() + self.mock_rename_cm = patch( + 'cray_product_catalog.migration.config_map_data_handler.ConfigMapDataHandler.rename_config_map' + ).start() + + with self.assertRaises(SystemExit) as captured: + self.mock_k8api_read.side_effect = [MockYaml(1), + MockYaml(2), + MockYaml(1), + MockYaml(2)] + self.mock_migrate_config_map.return_value = mock_split_catalog_data() + self.mock_create_prod_cms.return_value = True + self.mock_create_temp_cm.return_value = True + + # Call method under test + main() + + self.assertTrue( + "Re-trying migration process..." in captured.exception + ) + + self.assertTrue( + "Rollback successful" in captured.exception + ) + + self.assertTrue( + f"ConfigMap {PRODUCT_CATALOG_CONFIG_MAP_NAME} is modified, exiting migration process..." + ) + self.mock_rename_cm.assert_not_called() + + def test_main_failed_8(self): + """Validating that migration is successful in second attempt as initial and final resource + version is different in first attempt""" + + self.mock_migrate_config_map = patch( + 'cray_product_catalog.migration.config_map_data_handler.ConfigMapDataHandler.migrate_config_map_data' + ).start() + self.mock_create_prod_cms = patch( + 'cray_product_catalog.migration.config_map_data_handler.ConfigMapDataHandler.create_product_config_maps' + ).start() + self.mock_create_temp_cm = patch( + 'cray_product_catalog.migration.config_map_data_handler.ConfigMapDataHandler.create_temp_config_map' + ).start() + self.mock_rename_cm = patch( + 'cray_product_catalog.migration.config_map_data_handler.ConfigMapDataHandler.rename_config_map' + ).start() + + with self.assertLogs(level="DEBUG") as captured: + # with self.assertRaises(SystemExit) as captured: + self.mock_k8api_read.side_effect = [MockYaml(1), + MockYaml(2), + MockYaml(1), + MockYaml(1)] + self.mock_migrate_config_map.return_value = mock_split_catalog_data() + self.mock_create_prod_cms.return_value = True + self.mock_create_temp_cm.return_value = True + self.mock_rename_cm.return_value = True + + # Call method under test + main() + # Verify the exact log message + self.assertEqual(10, len(captured.records)) + self.assertEqual( + captured.records[3].getMessage(), + "Re-trying migration process..." + ) + self.assertEqual( + captured.records[6].getMessage(), + "Rollback successful" + ) + + self.assertEqual( + captured.records[-1].getMessage(), + "Migration successful" + ) diff --git a/tests/migration/test_exit_handler.py b/tests/migration/test_exit_handler.py new file mode 100644 index 00000000..032f2fdc --- /dev/null +++ b/tests/migration/test_exit_handler.py @@ -0,0 +1,109 @@ +# MIT License +# +# (C) Copyright 2023-2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +""" +File contains unit test classes for validating exit handler cases. +""" + +import unittest +from unittest.mock import patch, call + +from cray_product_catalog.migration.exit_handler import _is_product_config_map, ExitHandler +from cray_product_catalog.constants import PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE + + +class TestExitHandler(unittest.TestCase): + """unittest class for Data catalog ConfigMap deletion logic""" + + def setUp(self) -> None: + self.mock_load_k8s_mig = patch('cray_product_catalog.migration.kube_apis.load_k8s').start() + self.mock_corev1api_mig = patch('cray_product_catalog.migration.kube_apis.client.CoreV1Api').start() + self.mock_ApiClient_mig = patch('cray_product_catalog.migration.kube_apis.ApiClient').start() + + self.mock_k8api_del = patch( + 'cray_product_catalog.migration.exit_handler.KubernetesApi.delete_config_map').start() + self.mock_k8api_list = patch( + 'cray_product_catalog.migration.exit_handler.KubernetesApi.list_config_map_names').start() + + def tearDown(self) -> None: + patch.stopall() + + def test_product_config_map_pattern(self): + """Test cases for checking all valid patterns of product ConfigMap""" + base_str = "cray-product-catalog" + valid_patterns = ( + f"{base_str}-cos", + f"{base_str}-90-lojp", + f"{base_str}-cos.89-1234jk", + ) + + for valid_pattern in valid_patterns: + self.assertTrue(_is_product_config_map(valid_pattern)) + + def test_product_config_map_invalid_pattern(self): + """Test cases for checking all invalid patterns of product ConfigMap""" + base_str = "cray-product-catalog" + invalid_patterns = ( + f"{base_str}", + "90-lojp", + "cos2.3.45.x86" + ) + + for invalid_pattern in invalid_patterns: + self.assertFalse(_is_product_config_map(invalid_pattern)) + + def test_rollback_failure_from_product_config_map_deletion(self): + """Validating the scenario where one of the product ConfigMap is not deleted""" + + with self.assertLogs() as captured: + self.mock_k8api_del.side_effect = [True, False] # delete is called two times + + dummy_products = ["cray-product-catalog-cos", "cray-product-catalog-sma"] + self.mock_k8api_list.return_value = dummy_products + eh = ExitHandler() + eh.rollback() + # Verify the exact log message from last return + self.assertEqual(captured.records[-1].getMessage(), f"Error deleting ConfigMaps: " + f"{[dummy_products[-1]]}. Delete these manually") + + def test_rollback_all_success(self): + """Validating the scenario of successful rollback""" + + with self.assertLogs() as captured: + self.mock_k8api_del.return_value = True # delete is called three times + + dummy_products = ["cray-product-catalog-cos", "cray-product-catalog-sma"] + self.mock_k8api_list.return_value = dummy_products + eh = ExitHandler() + eh.rollback() + # Verify the exact log message from last return + self.assertEqual(captured.records[-1].getMessage(), "Rollback successful") + + # three calls in sequence for complete flow + self.mock_k8api_del.assert_has_calls(calls=[ + call( + name=dummy_products[0], + namespace=PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE), + call( + name=dummy_products[1], + namespace=PRODUCT_CATALOG_CONFIG_MAP_NAMESPACE), + ] + ) diff --git a/tests/mock_update_catalog.py b/tests/mock_update_catalog.py new file mode 100755 index 00000000..780a52b4 --- /dev/null +++ b/tests/mock_update_catalog.py @@ -0,0 +1,107 @@ +# MIT License +# +# (C) Copyright 2023-2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + +""" +Mock data for catalog_update unit tests +""" + +import os +from unittest import mock +import kubernetes.client.rest +from tests.mocks import COS_VERSIONS, Name + +# Mocking environment variables before import so that: +# 1. import (create_config_map, update_config_map and main) is successful +# 2. Additionally they will be used in testcase to verify the tests. +mock.patch.dict( + os.environ, { + 'PRODUCT': 'sat', + 'PRODUCT_VERSION': '1.0.0', + 'YAML_CONTENT_STRING': 'Test data', + 'CONFIG_MAP_NAMESPACE': 'myNamespace' + } +).start() + +UPDATE_DATA = { + '2.0.0': { + 'component_versions': { + 'docker': [ + {'name': 'cray/cray-cos', 'version': '1.0.0'}, + {'name': 'cray/cos-cfs-install', 'version': '1.4.0'} + ] + } + } +} + +ERR_NOT_FOUND = 404 + + +class Response: + """ + Class to generate response for k8s api call api_instance.read_namespaced_config_map(name, namespace) + """ + def __init__(self): + self.data = COS_VERSIONS + self.metadata = Name() + + +class ApiException(kubernetes.client.rest.ApiException): + """ + Custom Exception to define status + """ + def __init__(self): + super().__init__() + self.status = ERR_NOT_FOUND + + +class ApiInstance(): + """ + Class to raise custom exception and ignore function calls + """ + def __init__(self, raise_exception=False): + self.raise_exception = raise_exception + self.count = 0 + + def create_namespaced_config_map(self, namespace='a', body='b'): + """ + Dummy function to raise exception, if needed + """ + if self.raise_exception: + raise ApiException() + + def read_namespaced_config_map(self, name, namespace): + """ + Dummy function to : + 1. Raise exception + 2. Generate and return proper response with data and metadata + """ + # if this is called for first time return exception, so that product cm is created. + if self.count == 0: + self.count += 1 + raise ApiException() + return Response() + + def patch_namespaced_config_map(self, name, namespace, body='xxx'): + """ + Dummy function to handle the call in code; does nothing + """ diff --git a/tests/mocks.py b/tests/mocks.py index 85566b26..afc6b987 100644 --- a/tests/mocks.py +++ b/tests/mocks.py @@ -1,6 +1,6 @@ # MIT License # -# (C) Copyright 2021-2023 Hewlett Packard Enterprise Development LP +# (C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the "Software"), @@ -25,9 +25,12 @@ Mock data for ProductCatalog and InstalledProductVersion unit tests """ +import datetime from yaml import safe_dump +from cray_product_catalog.query import InstalledProductVersion +MOCK_NAMESPACE = 'mock-namespace' # Two versions of a product named SAT where: # - The two versions have have no docker images in common with one another. # - Both have configurations, but neither have images or recipes @@ -209,3 +212,124 @@ 'cpe': safe_dump(CPE_VERSION), 'other_product': safe_dump(OTHER_PRODUCT_VERSION) } + +# A mock version of the data returned after loading the ConfigMap data +MOCK_PRODUCTS = \ + [InstalledProductVersion('sat', version, SAT_VERSIONS.get(version)) for version in SAT_VERSIONS] + \ + [InstalledProductVersion('cos', version, COS_VERSIONS.get(version)) for version in COS_VERSIONS] + \ + [InstalledProductVersion('cpe', version, CPE_VERSION.get(version)) for version in CPE_VERSION] + \ + [InstalledProductVersion('other_product', version, OTHER_PRODUCT_VERSION.get(version)) + for version in OTHER_PRODUCT_VERSION.keys()] + + +class MockInvalidYaml: + """Mock class created to test test_create_product_catalog_invalid_product_data.""" + + def __init__(self): + """Initialize metadata and data object of ConfigMap data.""" + self.metadata = Name() + self.data = { + 'sat': '\t', + } + + +class Name: + """ + Class to provide dummy metadata object with name and resource_version + """ + def __init__(self): + """Initialize ConfigMap name and resoource_version""" + self.name = 'cray-product-catalog' + self.resource_version = 1 + + +# Helper variables for catalog_data_helper: Start +YAML_DATA = """ + active: false + component_versions: + docker: + - name: artifactory.algol60.net/uan-docker/stable/cray-uan-config + version: 1.11.1 + - name: artifactory.algol60.net/csm-docker/stable/cray-product-catalog-update + version: 1.3.2 + helm: + - name: cray-uan-install + version: 1.11.1 + repositories: + - members: + - uan-2.6.0-sle-15sp4 + name: uan-2.6-sle-15sp4 + type: group + manifests: + - config-data/argo/loftsman/uan/2.6.0-rc.1/manifests/uan.yaml + configuration: + clone_url: https://vcs.cmn.lemondrop.hpc.amslabs.hpecorp.net/vcs/cray/uan-config-management.git + commit: 6a5f52dfbfe7ea1a5f8ea5079c50995112c17025 + import_branch: cray/uan/2.6.0-rc.1-3-gcc65df9 + import_date: 2023-04-12 14:31:40.364230 + ssh_url: git@vcs.cmn.lemondrop.hpc.amslabs.hpecorp.net:cray/uan-config-management.git + images: + cray-application-sles15sp4.x86_64-0.5.19: + id: 8159f93f-7e18-4875-a8a8-b0fb83c48f07""" + +YAML_DATA_MISSING_PROD_CM_DATA = """ + active: false + configuration: + clone_url: https://vcs.cmn.lemondrop.hpc.amslabs.hpecorp.net/vcs/cray/uan-config-management.git + commit: 6a5f52dfbfe7ea1a5f8ea5079c50995112c17025 + import_branch: cray/uan/2.6.0-rc.1-3-gcc65df9 + import_date: 2023-04-12 14:31:40.364230 + ssh_url: git@vcs.cmn.lemondrop.hpc.amslabs.hpecorp.net:cray/uan-config-management.git + images: + cray-application-sles15sp4.x86_64-0.5.19: + id: 8159f93f-7e18-4875-a8a8-b0fb83c48f07""" + +YAML_DATA_MISSING_MAIN_DATA = """ + component_versions: + docker: + - name: artifactory.algol60.net/uan-docker/stable/cray-uan-config + version: 1.11.1 + - name: artifactory.algol60.net/csm-docker/stable/cray-product-catalog-update + version: 1.3.2 + helm: + - name: cray-uan-install + version: 1.11.1 + repositories: + - members: + - uan-2.6.0-sle-15sp4 + name: uan-2.6-sle-15sp4 + type: group + manifests: + - config-data/argo/loftsman/uan/2.6.0-rc.1/manifests/uan.yaml""" + +MAIN_CM_DATA = { + 'active': False, + 'configuration': + { + 'clone_url': 'https://vcs.cmn.lemondrop.hpc.amslabs.hpecorp.net/vcs/cray/uan-config-management.git', + 'commit': '6a5f52dfbfe7ea1a5f8ea5079c50995112c17025', + 'import_branch': 'cray/uan/2.6.0-rc.1-3-gcc65df9', + 'import_date': datetime.datetime(2023, 4, 12, 14, 31, 40, 364230), + 'ssh_url': 'git@vcs.cmn.lemondrop.hpc.amslabs.hpecorp.net:cray/uan-config-management.git', + 'images': {'cray-application-sles15sp4.x86_64-0.5.19': {'id': '8159f93f-7e18-4875-a8a8-b0fb83c48f07'}} + } +} + +PROD_CM_DATA = { + 'component_versions': + { + 'docker': [ + {'name': 'artifactory.algol60.net/uan-docker/stable/cray-uan-config', 'version': '1.11.1'}, + {'name': 'artifactory.algol60.net/csm-docker/stable/cray-product-catalog-update', 'version': '1.3.2'} + ], + 'helm': [ + {'name': 'cray-uan-install', 'version': '1.11.1'} + ], + 'repositories': [ + {'members': ['uan-2.6.0-sle-15sp4'], 'name': 'uan-2.6-sle-15sp4', 'type': 'group'} + ], + 'manifests': ['config-data/argo/loftsman/uan/2.6.0-rc.1/manifests/uan.yaml'] + } +} + +# Helper variables for catalog_data_helper: End diff --git a/tests/test_catalog_delete.py b/tests/test_catalog_delete.py new file mode 100644 index 00000000..62805d51 --- /dev/null +++ b/tests/test_catalog_delete.py @@ -0,0 +1,141 @@ +# MIT License +# +# (C) Copyright 2023-2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +""" +File contains unit test classes for validating ConfigMap deletion logic. +Deleting keys/product or a specific version of a product from ConfigMap +""" +import unittest +from unittest.mock import patch, call + +from cray_product_catalog.catalog_delete import ModifyConfigMapUtil + + +class TestModifyConfigMapUtil(unittest.TestCase): + """unittest class for data catalog ConfigMap deletion logic""" + + def setUp(self) -> None: + self.mock_modify_config_map = patch('cray_product_catalog.catalog_delete.modify_config_map').start() + + self.modify_config_map_util = ModifyConfigMapUtil() + self.modify_config_map_util.main_cm = "main_cm" + self.modify_config_map_util.product_cm = "product_cm" + self.modify_config_map_util.cm_namespace = "cm_namespace" + self.modify_config_map_util.product_name = "product_name" + self.modify_config_map_util.product_version = "product_version" + self.modify_config_map_util.max_retries_for_main_cm = 100 + self.modify_config_map_util.max_retries_for_prod_cm = 10 + self.modify_config_map_util.key = "key" + self.modify_config_map_util.main_cm_fields = ["main_a", "main_b", "main_c"] + self.modify_config_map_util.product_cm_fields = ["prod_1", "prod_2", "prod_3"] + + def tearDown(self) -> None: + patch.stopall() + + def test_object_properties(self): + """Test cases for checking objects properties are not wrongly arranged""" + mcmu = ModifyConfigMapUtil() + mcmu.main_cm = "1" + mcmu.product_cm = "2" + mcmu.cm_namespace = "3" + mcmu.product_name = "4" + mcmu.product_version = "5" + mcmu.max_retries_for_main_cm = "6" + mcmu.max_retries_for_prod_cm = "6.1" + mcmu.key = "7" + mcmu.main_cm_fields = ["8", "9", "10"] + mcmu.product_cm_fields = ["11", "12", "13"] + + self.assertEqual(mcmu.main_cm, "1") + self.assertEqual(mcmu.product_cm, "2") + self.assertEqual(mcmu.cm_namespace, "3") + self.assertEqual(mcmu.product_name, "4") + self.assertEqual(mcmu.product_version, "5") + self.assertEqual(mcmu.max_retries_for_main_cm, "6") + self.assertEqual(mcmu.max_retries_for_prod_cm, "6.1") + self.assertEqual(mcmu.key, "7") + self.assertEqual(mcmu.main_cm_fields, ["8", "9", "10"]) + self.assertEqual(mcmu.product_cm_fields, ["11", "12", "13"]) + + del mcmu + + def test_delete_from_both_config_map(self): + """Test cases to assert delete calls into both main and product ConfigMap""" + self.modify_config_map_util.key = None + self.modify_config_map_util.modify() + + self.mock_modify_config_map.assert_has_calls( + calls=[ + # main ConfigMap call + call(self.modify_config_map_util.main_cm, + self.modify_config_map_util.cm_namespace, + self.modify_config_map_util.product_name, + self.modify_config_map_util.product_version, + self.modify_config_map_util.key, + self.modify_config_map_util.max_retries_for_main_cm, + ), + # product ConfigMap call + call(self.modify_config_map_util.product_cm, + self.modify_config_map_util.cm_namespace, + self.modify_config_map_util.product_name, + self.modify_config_map_util.product_version, + self.modify_config_map_util.key, + self.modify_config_map_util.max_retries_for_prod_cm, + )] + ) + + def test_delete_from_main_config_map(self): + """Test cases to assert delete calls into main ConfigMap""" + self.modify_config_map_util.key = "main_a" + self.modify_config_map_util.modify() + + self.mock_modify_config_map.assert_called_once_with( + self.modify_config_map_util.main_cm, + self.modify_config_map_util.cm_namespace, + self.modify_config_map_util.product_name, + self.modify_config_map_util.product_version, + self.modify_config_map_util.key, + self.modify_config_map_util.max_retries_for_main_cm, + ) + + def test_delete_from_product_config_map(self): + """Test cases to assert delete calls into product ConfigMap""" + self.modify_config_map_util.key = "prod_3" + self.modify_config_map_util.modify() + + self.mock_modify_config_map.assert_called_once_with( + self.modify_config_map_util.product_cm, + self.modify_config_map_util.cm_namespace, + self.modify_config_map_util.product_name, + self.modify_config_map_util.product_version, + self.modify_config_map_util.key, + self.modify_config_map_util.max_retries_for_prod_cm, + ) + + def test_invalid_key(self): + """Test cases to assert invalid key""" + self.modify_config_map_util.key = "invalid string key" + self.modify_config_map_util.modify() + self.mock_modify_config_map.assert_not_called() + + self.modify_config_map_util.key = 909 # non string is invalid as well + self.modify_config_map_util.modify() + self.mock_modify_config_map.assert_not_called() diff --git a/tests/test_query.py b/tests/test_query.py index 3b7b109f..77ea7f35 100644 --- a/tests/test_query.py +++ b/tests/test_query.py @@ -1,6 +1,6 @@ # MIT License # -# (C) Copyright 2021-2023 Hewlett Packard Enterprise Development LP +# (C) Copyright 2021-2024 Hewlett Packard Enterprise Development LP # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the "Software"), @@ -31,16 +31,20 @@ from unittest.mock import Mock, patch from kubernetes.config import ConfigException -from yaml import safe_dump from cray_product_catalog.query import ( ProductCatalog, InstalledProductVersion, ProductCatalogError ) + +from cray_product_catalog.constants import ( + PRODUCT_CATALOG_CONFIG_MAP_LABEL_STR, PRODUCT_CATALOG_CONFIG_MAP_NAME +) + from tests.mocks import ( - COS_VERSIONS, MOCK_PRODUCT_CATALOG_DATA, SAT_VERSIONS, - CPE_VERSION, MOCK_INVALID_PRODUCT_DATA + COS_VERSIONS, SAT_VERSIONS, CPE_VERSION, MOCK_PRODUCT_CATALOG_DATA, + MOCK_INVALID_PRODUCT_DATA, MOCK_PRODUCTS, MockInvalidYaml, MOCK_NAMESPACE ) @@ -79,6 +83,7 @@ def setUp(self): """Set up mocks.""" self.mock_k8s_api = patch.object(ProductCatalog, '_get_k8s_api').start().return_value self.mock_product_catalog_data = copy.deepcopy(MOCK_PRODUCT_CATALOG_DATA) + self.mock_k8s_api.list_namespaced_config_map.return_value = Mock(items=self.mock_product_catalog_data) self.mock_k8s_api.read_namespaced_config_map.return_value = Mock(data=self.mock_product_catalog_data) def tearDown(self): @@ -87,12 +92,16 @@ def tearDown(self): def create_and_assert_product_catalog(self): """Assert the product catalog was created as expected.""" - product_catalog = ProductCatalog('mock-name', 'mock-namespace') - self.mock_k8s_api.read_namespaced_config_map.assert_called_once_with('mock-name', 'mock-namespace') + product_catalog = ProductCatalog(PRODUCT_CATALOG_CONFIG_MAP_NAME, MOCK_NAMESPACE) + self.mock_k8s_api.list_namespaced_config_map.assert_called_once_with( + MOCK_NAMESPACE, label_selector=PRODUCT_CATALOG_CONFIG_MAP_LABEL_STR + ) return product_catalog def test_create_product_catalog(self): """Test creating a simple ProductCatalog.""" + self.mock_load_config_map_data = patch('cray_product_catalog.query.load_config_map_data').start() + self.mock_load_config_map_data.return_value = MOCK_PRODUCTS product_catalog = self.create_and_assert_product_catalog() expected_names_and_versions = [ (name, version) for name in ('sat', 'cos') for version in ('2.0.0', '2.0.1') @@ -104,26 +113,62 @@ def test_create_product_catalog(self): def test_create_product_catalog_invalid_product_data(self): """Test creating a ProductCatalog when the product catalog contains invalid YAML.""" - self.mock_product_catalog_data['sat'] = '\t' + self.mock_k8s_api.list_namespaced_config_map.return_value = Mock(items=[MockInvalidYaml()]) with self.assertRaisesRegex(ProductCatalogError, 'Failed to load ConfigMap data'): self.create_and_assert_product_catalog() def test_create_product_catalog_null_data(self): - """Test creating a ProductCatalog when the product catalog contains null data.""" + """Test creating a ProductCatalog when the product catalog ConfigMaps with label + contains null data, but the product catalog ConfigMap without label has data""" + self.mock_load_config_map_data = patch('cray_product_catalog.query.load_config_map_data').start() + self.mock_load_config_map_data.return_value = [] + self.mock_k8s_api.list_namespaced_config_map.return_value = Mock(items=[]) + self.mock_load_cm_data = patch('cray_product_catalog.query.load_cm_data').start() + self.mock_load_cm_data.return_value = MOCK_PRODUCTS + with self.assertLogs(level=logging.DEBUG) as logs: + product_catalog = self.create_and_assert_product_catalog() + self.mock_k8s_api.read_namespaced_config_map.assert_called_once_with( + PRODUCT_CATALOG_CONFIG_MAP_NAME, MOCK_NAMESPACE + ) + self.assertEqual(product_catalog.products, MOCK_PRODUCTS) + + def test_create_product_catalog_invalid_product_data1(self): + """Test creating a ProductCatalog when the product catalog ConfigMaps with label + contains null data, but the product catalog ConfigMap without label has invalid YAML.""" + self.mock_load_config_map_data = patch('cray_product_catalog.query.load_config_map_data').start() + self.mock_load_config_map_data.return_value = [] + self.mock_k8s_api.list_namespaced_config_map.return_value = Mock(items=[]) + self.mock_k8s_api.read_namespaced_config_map.return_value = Mock(data=MockInvalidYaml().data) + with self.assertRaisesRegex(ProductCatalogError, 'Failed to load ConfigMap data'): + self.create_and_assert_product_catalog() + + def test_create_product_catalog_null_data1(self): + """Test creating a ProductCatalog when the product catalog ConfigMaps with label + and without label contains null data.""" + self.mock_load_config_map_data = patch('cray_product_catalog.query.load_config_map_data').start() + self.mock_load_config_map_data.return_value = [] + self.mock_k8s_api.list_namespaced_config_map.return_value = Mock(items=[]) self.mock_k8s_api.read_namespaced_config_map.return_value = Mock(data=None) - with self.assertRaisesRegex(ProductCatalogError, - 'No data found in mock-namespace/mock-name ConfigMap.'): + message = 'No data found in ' + MOCK_NAMESPACE + '/' + PRODUCT_CATALOG_CONFIG_MAP_NAME + ' ConfigMap.' + with self.assertRaisesRegex(ProductCatalogError, message): self.create_and_assert_product_catalog() + self.mock_k8s_api.read_namespaced_config_map.assert_called_once_with( + PRODUCT_CATALOG_CONFIG_MAP_NAME, MOCK_NAMESPACE + ) def check_for_invalid_product_schema(self, prod, ver): """Check for an ProductCatalog entry containing valid YAML but does not match schema. + Args: prod (string): Product Name ver (string): Product version """ - mock_data = {prod: safe_dump(MOCK_INVALID_PRODUCT_DATA.get(prod))} - self.mock_k8s_api.read_namespaced_config_map.return_value = Mock(data=mock_data) - + self.mock_load_config_map_data = patch('cray_product_catalog.query.load_config_map_data').start() + self.mock_load_config_map_data.return_value = [ + InstalledProductVersion(prod, ver, MOCK_INVALID_PRODUCT_DATA.get(prod).get(ver)) + ] + self.mock_k8s_api.list_namespaced_config_map.return_value = Mock(items=[ + MOCK_INVALID_PRODUCT_DATA.get(prod).get(ver)]) with self.assertLogs(level=logging.DEBUG) as logs_cm: product_catalog = self.create_and_assert_product_catalog() @@ -156,6 +201,8 @@ def test_create_product_catalog_invalid_product_schema_for_manifests(self): def test_get_matching_product(self): """Test getting a particular product by name/version.""" + self.mock_load_config_map_data = patch('cray_product_catalog.query.load_config_map_data').start() + self.mock_load_config_map_data.return_value = MOCK_PRODUCTS product_catalog = self.create_and_assert_product_catalog() expected_matching_name_and_version = ('cos', '2.0.0') actual_matching_product = product_catalog.get_product('cos', '2.0.0') @@ -167,6 +214,8 @@ def test_get_matching_product(self): def test_get_latest_matching_product(self): """Test getting the latest version of a product""" + self.mock_load_config_map_data = patch('cray_product_catalog.query.load_config_map_data').start() + self.mock_load_config_map_data.return_value = MOCK_PRODUCTS product_catalog = self.create_and_assert_product_catalog() expected_matching_name_and_version = ('sat', '2.0.1') actual_matching_product = product_catalog.get_product('sat') diff --git a/tests/test_update_catalog.py b/tests/test_update_catalog.py new file mode 100755 index 00000000..3049616a --- /dev/null +++ b/tests/test_update_catalog.py @@ -0,0 +1,192 @@ +# MIT License +# +# (C) Copyright 2023-2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. +# + + +"""Unit tests for cray_product_catalog.catalog_update module""" + +import unittest +from unittest import mock +from tests.mock_update_catalog import ( + UPDATE_DATA, ApiInstance, ApiException +) +from cray_product_catalog.catalog_update import ( + create_config_map, + update_config_map, + main +) + + +class TestCatalogUpdate(unittest.TestCase): + """ + Tests for catalog update + """ + def setUp(self): + """Set up mocks.""" + self.mock_v1configmap = mock.patch('cray_product_catalog.catalog_update.V1ConfigMap').start() + self.mock_load_k8s = mock.patch('cray_product_catalog.catalog_update.load_k8s').start() + self.mock_ApiClient = mock.patch('cray_product_catalog.catalog_update.ApiClient').start() + self.mock_client = mock.patch('cray_product_catalog.catalog_update.client').start() + + def test_create_config_map_success_log(self): + """ + Test for validating create_config_map method logs + Verify expected log is generated based on the passed arguments + """ + name = "cos" + namespace = "product" + self.mock_v1configmap.metadata = None + with self.assertLogs() as captured: + # call method under test + create_config_map(ApiInstance(raise_exception=False), name, namespace) + self.assertEqual(len(captured.records), 1) # check that there is only one log message + + expected_log = "Created product ConfigMap " + namespace + "/" + name + self.assertEqual(captured.records[0].getMessage(), expected_log) # Verify the exact log message + + def test_create_config_map_failure_exception(self): + """ + Verify if expected logs are generated in exception + """ + name = "cos" + namespace = "product" + self.mock_v1configmap.metadata = None + + with self.assertLogs() as captured: + try: + # call method under test + create_config_map(ApiInstance(raise_exception=True), name, namespace) + except ApiException: + pass + self.assertEqual(len(captured.records), 1) # check that there is only one log message + self.assertEqual(captured.records[0].getMessage(), + "Error calling create_namespaced_config_map") # Verify the exact log message + + def test_update_config_map_max_retries(self): + """ + Verify update_config_map exits after max retries if Kubernetes API raise exception. + """ + name = "cos" + namespace = "product" + self.mock_v1configmap.metadata = None + + self.mock_read_config_map = mock.patch( + 'cray_product_catalog.catalog_update.client.CoreV1Api.read_namespaced_config_map' + ).start().side_effect = ApiException() + + with self.assertRaises(SystemExit) as captured: + with mock.patch( + 'cray_product_catalog.catalog_update.random.randint', return_value=0 + ): + # call method under test + update_config_map(UPDATE_DATA, name, namespace) + # Verify the exact log message + self.assertTrue( + f"Exceeded number of attempts; Not updating ConfigMap {namespace}/{name}." + in captured.exception) + + def test_update_config_map(self): + """ + Verify `create_config_map` is called if provided `name` is of product and not `CONFIG_MAP` (main cm) + """ + name = "cos" + namespace = "product" + data = UPDATE_DATA + + # mock some additional functions + self.mock_create_config_map = mock.patch('cray_product_catalog.catalog_update.create_config_map').start() + self.mock_v1_object_Meta = mock.patch('cray_product_catalog.catalog_update.V1ObjectMeta').start() + + with mock.patch( + 'cray_product_catalog.catalog_update.client.CoreV1Api', return_value=ApiInstance(raise_exception=True) + ): + with mock.patch( + 'cray_product_catalog.catalog_update.random.randint', return_value=0.5 + ): + # call method under test + update_config_map(data, name, namespace) + + # verify if create-config_map is called. Couldn't verify it with arguments as one of the arg is object. + self.mock_create_config_map.assert_called() + + def test_main_valid_product_configmap(self): + """ + Verify `update_config_map` is called with proper data if provided product information is available + """ + # Initialise some random data, need not to be exact format + prod_cm = {"Some random text for product"} + main_cm = {"Some random text for main"} + + self.mock_update_config_map = mock.patch('cray_product_catalog.catalog_update.update_config_map').start() + self.mock_create_config_map = mock.patch('cray_product_catalog.catalog_update.create_config_map').start() + self.mock_v1_object_Meta = mock.patch('cray_product_catalog.catalog_update.V1ObjectMeta').start() + + # mocking function to return custom data + def mock_split_catalog_data(): + return main_cm, prod_cm + + with mock.patch( + 'cray_product_catalog.catalog_update.split_catalog_data', return_value=mock_split_catalog_data() + ): + # Call method under test + main() + # sat is from PRODUCT environment variable. + expected_product_cm = 'cray-product-catalog-sat' + # myNamespace is from CONFIG_MAP_NAMESPACE environment variable. + expected_namespace = 'myNamespace' + self.mock_update_config_map.assert_called_with(prod_cm, expected_product_cm, expected_namespace) + + def test_main_for_empty_product_configmap(self): + """ + Verify `main` throws exception when PRODUCT_CONFIG_MAP='' + """ + prod_cm = {"Some random text for product"} + main_cm = {"Some random text for main"} + + self.mock_update_config_map = mock.patch('cray_product_catalog.catalog_update.update_config_map').start() + self.mock_create_config_map = mock.patch('cray_product_catalog.catalog_update.create_config_map').start() + self.mock_v1_object_Meta = mock.patch('cray_product_catalog.catalog_update.V1ObjectMeta').start() + + # mocking function to return custom data + def mock_split_catalog_data(): + return main_cm, prod_cm + + with mock.patch( + 'cray_product_catalog.catalog_update.split_catalog_data', return_value=mock_split_catalog_data() + ): + with mock.patch( + 'cray_product_catalog.catalog_update.format_product_cm_name', return_value='' + ): + with self.assertRaises(SystemExit) as context: + # call method under test + main() + # Verify the log message in exception + self.assertTrue( + "ERROR Not updating ConfigMaps because the provided product name is invalid: 'sat'" + in context.exception + ) + # Verify that update config map is not called in case of exception + self.mock_update_config_map.assert_not_called() + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/util/test_data_catalog_helper.py b/tests/util/test_data_catalog_helper.py new file mode 100755 index 00000000..c831627e --- /dev/null +++ b/tests/util/test_data_catalog_helper.py @@ -0,0 +1,124 @@ +# MIT License +# +# (C) Copyright 2023-2024 Hewlett Packard Enterprise Development LP +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +""" +Unit tests for the cray_product_catalog.util.catalog_data_helper module +""" + +import unittest +import yaml +from typing import Dict +from tests.mocks import ( + YAML_DATA, YAML_DATA_MISSING_MAIN_DATA, + YAML_DATA_MISSING_PROD_CM_DATA, + MAIN_CM_DATA, PROD_CM_DATA +) +from cray_product_catalog.util.catalog_data_helper import split_catalog_data, format_product_cm_name + + +class TestCatalogDataHelper(unittest.TestCase): + """Tests for catalog_data_helper.""" + + def test_split_data_sanity(self): + """Sanity check of split of YAML into main and product-specific data | +ve test case""" + + # expected data + main_cm_data_expected = MAIN_CM_DATA + prod_cm_data_expected = PROD_CM_DATA + + # YAML raw to Python object + yaml_object: Dict = yaml.safe_load(YAML_DATA) + + main_cm_data: Dict + prod_cm_data: Dict + main_cm_data, prod_cm_data = split_catalog_data(yaml_object) + + self.assertEqual(main_cm_data, main_cm_data_expected) + self.assertEqual(prod_cm_data, prod_cm_data_expected) + + def test_split_missing_prod_cm_data(self): + """Missing product ConfigMap data check""" + + # expected data + main_cm_data_expected = MAIN_CM_DATA + prod_cm_data_expected = {} + + # YAML raw to Python object + yaml_object: Dict = yaml.safe_load(YAML_DATA_MISSING_PROD_CM_DATA) + + main_cm_data: Dict + prod_cm_data: Dict + main_cm_data, prod_cm_data = split_catalog_data(yaml_object) + + self.assertEqual(main_cm_data, main_cm_data_expected) + self.assertEqual(prod_cm_data, prod_cm_data_expected) + + def test_split_missing_main_cm_data(self): + """Missing main ConfigMap data check""" + + # expected data + main_cm_data_expected = {} + prod_cm_data_expected = PROD_CM_DATA + + # YAML raw to Python object + yaml_object: Dict = yaml.safe_load(YAML_DATA_MISSING_MAIN_DATA) + + main_cm_data: Dict + prod_cm_data: Dict + main_cm_data, prod_cm_data = split_catalog_data(yaml_object) + + self.assertEqual(main_cm_data, main_cm_data_expected) + self.assertEqual(prod_cm_data, prod_cm_data_expected) + + def test_format_product_cm_name_sanity(self): + """Unit test case for product name formatting""" + product_name = "dummy-valid-1" + config_map = "cm" + self.assertEqual(format_product_cm_name(config_map, product_name), f"{config_map}-{product_name}") + + def test_format_product_name_transform(self): + """Unit test case for valid product name transformation""" + product_name = "23dummy_valid-1.x86" + config_map = "cm" + self.assertEqual(format_product_cm_name(config_map, product_name), f"{config_map}-23dummy-valid-1.x86") + + def test_format_product_name_invalid_cases(self): + """Unit test case for invalid product names""" + + # case with special characters + product_name = "po90-$_invalid" + config_map = "cm" + self.assertEqual(format_product_cm_name(config_map, product_name), "") + + # large name with non-blank ConfigMap case + product_name = "ola-9" * 60 + config_map = "cm" + self.assertEqual(format_product_cm_name(config_map, product_name), "") + + # large name with blank ConfigMap case + product_name = "ola-9" * 60 + config_map = "" + self.assertEqual(format_product_cm_name(config_map, product_name), "") + + +if __name__ == '__main__': + unittest.main() diff --git a/update_versions.conf b/update_versions.conf index 8f79c738..845c4c88 100644 --- a/update_versions.conf +++ b/update_versions.conf @@ -20,6 +20,7 @@ sourcefile: .docker_version tag: 0.0.0-docker targetfile: charts/cray-product-catalog/Chart.yaml +targetfile: charts/cray-product-catalog/values.yaml # This sourcefile does not exist as a static file in the repo. # It is created at build time. @@ -32,3 +33,4 @@ targetfile: charts/cray-product-catalog/Chart.yaml sourcefile-novalidate: .stable tag: S-T-A-B-L-E targetfile: charts/cray-product-catalog/Chart.yaml +targetfile: charts/cray-product-catalog/values.yaml