From a473fe92797b9b895d05a76de13eb43edee1e68c Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Tue, 7 Nov 2023 02:28:18 -0800 Subject: [PATCH 01/28] Initial work to demonstrate an actor config framework. * Modify the upgrade workflow to load the actor configuration to be available to the actors [_] Note: The same sort of code needs to be added to other workflows that need actor config. For instance, the pre-upgrade workflow. * Define potential config schemas for RHUI and for listing rpm packages to be treated specially during transations. * Add config schemas to the rpmtransactionconfigtaskscollector as a sample of using it. * The actor config is supposed to live in the actor: We decided that we weren't going to share config schema via python import. Instead, Actors would need to copy the schema into themselves. That way the framework can check whether the schema for two Actors is out of sync and force the user to correct the code before running. * However, the framework is currently only reading from the configs dir at the repository level, not inside the actor. Need to remove the copy from the repository level once that is fixed. * Get the rpm package liste from both legacy location and actor config. Depends-On: 870 --- commands/upgrade/__init__.py | 12 +++ .../actor.py | 7 +- .../configs/__init__.py | 0 .../configs/rpm.py | 64 ++++++++++++++ .../rpmtransactionconfigtaskscollector.py | 34 +++++-- ...asks_rpmtransactionconfigtaskscollector.py | 88 +++++++++++++++++-- .../system_upgrade/common/configs/__init__.py | 0 repos/system_upgrade/common/configs/rhui.py | 81 +++++++++++++++++ repos/system_upgrade/common/configs/rpm.py | 64 ++++++++++++++ 9 files changed, 330 insertions(+), 20 deletions(-) create mode 100644 repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/configs/__init__.py create mode 100644 repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/configs/rpm.py create mode 100644 repos/system_upgrade/common/configs/__init__.py create mode 100644 repos/system_upgrade/common/configs/rhui.py create mode 100644 repos/system_upgrade/common/configs/rpm.py diff --git a/commands/upgrade/__init__.py b/commands/upgrade/__init__.py index 1e15b59c45..0de7148ced 100644 --- a/commands/upgrade/__init__.py +++ b/commands/upgrade/__init__.py @@ -2,6 +2,7 @@ import sys import uuid +from leapp.actor import config as actor_config from leapp.cli.commands import command_utils from leapp.cli.commands.config import get_config from leapp.cli.commands.upgrade import breadcrumbs, util @@ -90,6 +91,17 @@ def upgrade(args, breadcrumbs): except LeappError as exc: raise CommandError(exc.message) workflow = repositories.lookup_workflow('IPUWorkflow')(auto_reboot=args.reboot) + + # Read the Actor Config and validate it against the schemas saved in the + # configuration. + actor_config_schemas = tuple(actor.config_schemas for actor in repositories.actors) + actor_config_schemas = actor_config.normalize_schemas(actor_config_schemas) + actor_config_path = cfg.get('actor_config', 'path') + # Note: actor_config.load() stores the loaded actor config into a global + # variable which can then be accessed by functions in that file. Is this + # the right way to store that information? + actor_config.load(actor_config_path, actor_config_schemas) + util.process_whitelist_experimental(repositories, workflow, configuration, logger) util.warn_if_unsupported(configuration) with beautify_actor_exception(): diff --git a/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/actor.py b/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/actor.py index a353158694..f0e48bd4f8 100644 --- a/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/actor.py +++ b/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/actor.py @@ -1,10 +1,9 @@ from leapp.actors import Actor +from leapp.configs.actor.rpm import Transaction_ToInstall, Transaction_ToKeep, Transaction_ToRemove from leapp.libraries.actor.rpmtransactionconfigtaskscollector import load_tasks from leapp.models import DistributionSignedRPM, RpmTransactionTasks from leapp.tags import FactsPhaseTag, IPUWorkflowTag -CONFIGURATION_BASE_PATH = '/etc/leapp/transaction' - class RpmTransactionConfigTasksCollector(Actor): """ @@ -13,11 +12,11 @@ class RpmTransactionConfigTasksCollector(Actor): After collecting task data from /etc/leapp/transaction directory, a message with relevant data will be produced. """ - + config_schemas = (Transaction_ToInstall, Transaction_ToKeep, Transaction_ToRemove) name = 'rpm_transaction_config_tasks_collector' consumes = (DistributionSignedRPM,) produces = (RpmTransactionTasks,) tags = (FactsPhaseTag, IPUWorkflowTag) def process(self): - self.produce(load_tasks(CONFIGURATION_BASE_PATH, self.log)) + self.produce(load_tasks(self.config, self.log)) diff --git a/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/configs/__init__.py b/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/configs/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/configs/rpm.py b/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/configs/rpm.py new file mode 100644 index 0000000000..c21e94ff03 --- /dev/null +++ b/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/configs/rpm.py @@ -0,0 +1,64 @@ +""" +Configuration keys for dnf transactions. +""" + +from leapp.actors.config import Config +from leapp.models import fields + + +# * Nested containers? +# * Duplication of default value in type_ and Config. If we eliminate that, we need to extract +# default from the type_ for the documentation. +# * We probably want to allow dicts in Config. But IIRC, dicts were +# specifically excluded for model fields. Do we need something that restricts +# where fields are valid? +# * Test that type validation is strict. For instance, giving an integer like 644 to +# a field.String() is an error. +class Transaction_ToInstall(Config): + section = "transaction" + name = "to_install" + type_ = fields.List(fields.String(), default=[]) + default = [] + description = """ + List of packages to be added to the upgrade transaction. + Signed packages which are already installed will be skipped. + """ + + +class Transaction_ToKeep(Config): + section = "transaction" + name = "to_keep" + type_ = fields.List(fields.String(), default=[ + "leapp", + "python2-leapp", + "python3-leapp", + "leapp-repository", + "snactor", + ]) + default = [ + "leapp", + "python2-leapp", + "python3-leapp", + "leapp-repository", + "snactor", + ] + description = """ + List of packages to be kept in the upgrade transaction. The default is + leapp, python2-leapp, python3-leapp, leapp-repository, snactor. If you + override this, remember to include the default values if applicable. + """ + + +class Transaction_ToRemove(Config): + section = "transaction" + name = "to_remove" + type_ = fields.List(fields.String(), default=[ + "initial-setup", + ]) + default = ["initial-setup"] + description = """ + List of packages to be removed from the upgrade transaction. The default + is initial-setup which should be removed to avoid it asking for EULA + acceptance during upgrade. If you override this, remember to include the + default values if applicable. + """ diff --git a/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/libraries/rpmtransactionconfigtaskscollector.py b/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/libraries/rpmtransactionconfigtaskscollector.py index 43ac1fc48b..713b42abcf 100644 --- a/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/libraries/rpmtransactionconfigtaskscollector.py +++ b/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/libraries/rpmtransactionconfigtaskscollector.py @@ -3,6 +3,10 @@ from leapp.libraries.stdlib import api from leapp.models import DistributionSignedRPM, RpmTransactionTasks +# Deprecated. This is the old, pre-actor config method of customizing which +# packages to keep, remove, and install. +_CONFIGURATION_BASE_PATH = '/etc/leapp/transaction' + def load_tasks_file(path, logger): # Loads the given file and converts it to a deduplicated list of strings that are stripped @@ -18,21 +22,35 @@ def load_tasks_file(path, logger): return [] -def load_tasks(base_dir, logger): +def load_tasks(config, logger, base_dir=_CONFIGURATION_BASE_PATH): # Loads configuration files to_install, to_keep, and to_remove from the given base directory rpms = next(api.consume(DistributionSignedRPM)) rpm_names = [rpm.name for rpm in rpms.items] - to_install = load_tasks_file(os.path.join(base_dir, 'to_install'), logger) - # we do not want to put into rpm transaction what is already installed (it will go to "to_upgrade" bucket) + to_keep = frozenset(config['transaction']['to_keep']) + to_keep = to_keep.union(load_tasks_file( + os.path.join(base_dir, 'to_keep'), logger)) + to_keep = list(to_keep) + + to_remove = frozenset(config['transaction']['to_remove']) + to_remove = to_remove.union(load_tasks_file( + os.path.join(base_dir, 'to_remove'), logger)) + to_remove = list(to_remove) + + to_install = frozenset(config['transaction']['to_install']) + to_install = to_install.union(load_tasks_file( + os.path.join(base_dir, 'to_install'), logger)) + # we do not want to put into rpm transaction what is already installed + # (it will go to "to_upgrade" bucket) to_install_filtered = [pkg for pkg in to_install if pkg not in rpm_names] - filtered = set(to_install) - set(to_install_filtered) + filtered = to_install.difference(to_install_filtered) if filtered: api.current_logger().debug( - 'The following packages from "to_install" file will be ignored as they are already installed:' - '\n- ' + '\n- '.join(filtered)) + 'The following packages from "to_install" file will be ignored as' + ' they are already installed:\n- ' + '\n- '.join(filtered)) return RpmTransactionTasks( to_install=to_install_filtered, - to_keep=load_tasks_file(os.path.join(base_dir, 'to_keep'), logger), - to_remove=load_tasks_file(os.path.join(base_dir, 'to_remove'), logger)) + to_keep=to_keep, + to_remove=to_remove + ) diff --git a/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/tests/test_load_tasks_rpmtransactionconfigtaskscollector.py b/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/tests/test_load_tasks_rpmtransactionconfigtaskscollector.py index 842544bf80..58d77fec62 100644 --- a/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/tests/test_load_tasks_rpmtransactionconfigtaskscollector.py +++ b/repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/tests/test_load_tasks_rpmtransactionconfigtaskscollector.py @@ -1,5 +1,7 @@ import logging +import pytest + from leapp.libraries.actor.rpmtransactionconfigtaskscollector import load_tasks, load_tasks_file from leapp.libraries.stdlib import api from leapp.models import DistributionSignedRPM, RPM @@ -7,7 +9,66 @@ RH_PACKAGER = 'Red Hat, Inc. ' -def test_load_tasks(tmpdir, monkeypatch): +@pytest.mark.parametrize( + ( + 'to_install_file', + 'to_keep_file', + 'to_remove_file', + 'to_install_config', + 'to_keep_config', + 'to_remove_config', + 'to_install', + 'to_keep', + 'to_remove', + ), + ( + ( + 'a\n b\n c \n\n\nc\na\nc\nb', + 'a\n b\n c \n\n\nc\na\nc\nb', + 'a\n b\n c \n\n\nc\na\nc\nb', + [], + [], + [], + frozenset(('a', 'b')), + frozenset(('a', 'b', 'c')), + frozenset(('a', 'b', 'c')), + ), + ( + 'a\n b\n c \n\n\nc\na\nc\nb', + 'a\n b\n c \n\n\nc\na\nc\nb', + 'a\n b\n c \n\n\nc\na\nc\nb', + ['a', 'd'], + ['a', 'd'], + ['a', 'd'], + frozenset(('a', 'b', 'd')), + frozenset(('a', 'b', 'c', 'd')), + frozenset(('a', 'b', 'c', 'd')), + ), + ( + '', + '\n', + '', + ['a', 'b', 'c', 'c', 'a', 'c', 'b'], + ['a', 'b', 'c', 'c', 'a', 'c', 'b'], + ['a', 'b', 'c', 'c', 'a', 'c', 'b'], + frozenset(('a', 'b')), + frozenset(('a', 'b', 'c')), + frozenset(('a', 'b', 'c')), + ), + ) +) +def test_load_tasks(to_install_file, + to_keep_file, + to_remove_file, + to_install_config, + to_keep_config, + to_remove_config, + to_install, + to_keep, + to_remove, + tmpdir, + monkeypatch, + ): def consume_signed_rpms_mocked(*models): installed = [ @@ -18,14 +79,25 @@ def consume_signed_rpms_mocked(*models): monkeypatch.setattr(api, "consume", consume_signed_rpms_mocked) - tmpdir.join('to_install').write('a\n b\n c \n\n\nc\na\nc\nb') - tmpdir.join('to_keep').write('a\n b\n c \n\n\nc\na\nc\nb') - tmpdir.join('to_remove').write('a\n b\n c \n\n\nc\na\nc\nb') - m = load_tasks(tmpdir.strpath, logging) + # Set values in the legacy configuration files + tmpdir.join('to_install').write(to_install_file) + tmpdir.join('to_keep').write(to_keep_file) + tmpdir.join('to_remove').write(to_remove_file) + + # Simulate how the new actor config will come to us + config = { + 'transaction': { + 'to_install': to_install_config, + 'to_keep': to_keep_config, + 'to_remove': to_remove_config, + } + } + + m = load_tasks(config, logging, base_dir=tmpdir.strpath) # c is not going to be in "to_install" as it is already installed - assert set(m.to_install) == set(['a', 'b']) - assert set(m.to_keep) == set(['a', 'b', 'c']) - assert set(m.to_remove) == set(['a', 'b', 'c']) + assert frozenset(m.to_install) == to_install + assert frozenset(m.to_keep) == to_keep + assert frozenset(m.to_remove) == to_remove def test_load_tasks_file(tmpdir): diff --git a/repos/system_upgrade/common/configs/__init__.py b/repos/system_upgrade/common/configs/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/repos/system_upgrade/common/configs/rhui.py b/repos/system_upgrade/common/configs/rhui.py new file mode 100644 index 0000000000..55b96cb211 --- /dev/null +++ b/repos/system_upgrade/common/configs/rhui.py @@ -0,0 +1,81 @@ +""" +Configuration keys for RHUI. + +In case of RHUI in private regions it usual that publicly known RHUI data +is not valid. In such cases it's possible to provide the correct expected +RHUI data to correct the in-place upgrade process. +""" + +from leapp.actors.configs import Config +from leapp.models import fields + + +class RhuiSrcPkg(Config): + section = "rhui" + name = "src_pkg" + type_ = fields.String(default="rhui") + default = "rhui" + description = """ + The name of the source RHUI client RPM (installed on the system). + Default: rhui. + """ + + +class RhuiTargetPkg(Config): + section = "rhui" + name = "target_pkg" + type_ = fields.String(default="rhui") + default = "rhui" + description = """ + The name of the target RHUI client RPM (to be installed on the system). + Default: rhui + """ + + +class RhuiLeappRhuiPkg(Config): + section = "rhui" + name = "leapp_rhui_pkg" + type_ = fields.String(default="leapp-rhui") + default = "leapp-rhui" + description = """ + The name of the leapp-rhui RPM. Default: leapp-rhui + """ + + +class RhuiLeappRhuiPkgRepo(Config): + section = "rhui" + name = "leapp_rhui_pkg_repo" + type_ = fields.String(default="rhel-base") + default = "rhel-base" + description = """ + The repository ID containing the specified leapp-rhui RPM. + Default: rhel-base + """ + + +all_rhui_cfg = (RhuiSrcPkg, RhuiTargetPkg, RhuiLeappRhuiPkg, RhuiLeappRhuiPkg) +# Usage: from configs import rhui +# class MyActor: +# [...] +# configs = all_rhui_cfg + (MyConfig,) + +# TODO: We need to implement fields.Map before this can be enabled +''' +class RhuiFileMap(Config): + section = "rhui" + name = "file_map" + type_ = fields.Map(fields.String()) + description = """ + Define directories to which paritcular files provided by the leapp-rhui + RPM should be installed. The files in 'files_map' are provided by + special Leapp rpms (per cloud) and are supposed to be delivered into the + repos/system_upgrade/common/files/rhui/ directory. + + These files are usually needed to get access to the target system repositories + using RHUI. Typically these are certificates, keys, and repofiles with the + target RHUI repositories. + + The key is the name of the file, the value is the expected directory + where the file should be installed on the upgraded system. + """ +''' diff --git a/repos/system_upgrade/common/configs/rpm.py b/repos/system_upgrade/common/configs/rpm.py new file mode 100644 index 0000000000..c21e94ff03 --- /dev/null +++ b/repos/system_upgrade/common/configs/rpm.py @@ -0,0 +1,64 @@ +""" +Configuration keys for dnf transactions. +""" + +from leapp.actors.config import Config +from leapp.models import fields + + +# * Nested containers? +# * Duplication of default value in type_ and Config. If we eliminate that, we need to extract +# default from the type_ for the documentation. +# * We probably want to allow dicts in Config. But IIRC, dicts were +# specifically excluded for model fields. Do we need something that restricts +# where fields are valid? +# * Test that type validation is strict. For instance, giving an integer like 644 to +# a field.String() is an error. +class Transaction_ToInstall(Config): + section = "transaction" + name = "to_install" + type_ = fields.List(fields.String(), default=[]) + default = [] + description = """ + List of packages to be added to the upgrade transaction. + Signed packages which are already installed will be skipped. + """ + + +class Transaction_ToKeep(Config): + section = "transaction" + name = "to_keep" + type_ = fields.List(fields.String(), default=[ + "leapp", + "python2-leapp", + "python3-leapp", + "leapp-repository", + "snactor", + ]) + default = [ + "leapp", + "python2-leapp", + "python3-leapp", + "leapp-repository", + "snactor", + ] + description = """ + List of packages to be kept in the upgrade transaction. The default is + leapp, python2-leapp, python3-leapp, leapp-repository, snactor. If you + override this, remember to include the default values if applicable. + """ + + +class Transaction_ToRemove(Config): + section = "transaction" + name = "to_remove" + type_ = fields.List(fields.String(), default=[ + "initial-setup", + ]) + default = ["initial-setup"] + description = """ + List of packages to be removed from the upgrade transaction. The default + is initial-setup which should be removed to avoid it asking for EULA + acceptance during upgrade. If you override this, remember to include the + default values if applicable. + """ From cece1e315aab3c556588ebaddc23c9ed513aad04 Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Fri, 20 Sep 2024 02:26:09 -0700 Subject: [PATCH 02/28] Remove rpm transaction configs from repository level. We want this to come from the actor. Depends-On: 870 --- repos/system_upgrade/common/configs/rpm.py | 64 ---------------------- 1 file changed, 64 deletions(-) delete mode 100644 repos/system_upgrade/common/configs/rpm.py diff --git a/repos/system_upgrade/common/configs/rpm.py b/repos/system_upgrade/common/configs/rpm.py deleted file mode 100644 index c21e94ff03..0000000000 --- a/repos/system_upgrade/common/configs/rpm.py +++ /dev/null @@ -1,64 +0,0 @@ -""" -Configuration keys for dnf transactions. -""" - -from leapp.actors.config import Config -from leapp.models import fields - - -# * Nested containers? -# * Duplication of default value in type_ and Config. If we eliminate that, we need to extract -# default from the type_ for the documentation. -# * We probably want to allow dicts in Config. But IIRC, dicts were -# specifically excluded for model fields. Do we need something that restricts -# where fields are valid? -# * Test that type validation is strict. For instance, giving an integer like 644 to -# a field.String() is an error. -class Transaction_ToInstall(Config): - section = "transaction" - name = "to_install" - type_ = fields.List(fields.String(), default=[]) - default = [] - description = """ - List of packages to be added to the upgrade transaction. - Signed packages which are already installed will be skipped. - """ - - -class Transaction_ToKeep(Config): - section = "transaction" - name = "to_keep" - type_ = fields.List(fields.String(), default=[ - "leapp", - "python2-leapp", - "python3-leapp", - "leapp-repository", - "snactor", - ]) - default = [ - "leapp", - "python2-leapp", - "python3-leapp", - "leapp-repository", - "snactor", - ] - description = """ - List of packages to be kept in the upgrade transaction. The default is - leapp, python2-leapp, python3-leapp, leapp-repository, snactor. If you - override this, remember to include the default values if applicable. - """ - - -class Transaction_ToRemove(Config): - section = "transaction" - name = "to_remove" - type_ = fields.List(fields.String(), default=[ - "initial-setup", - ]) - default = ["initial-setup"] - description = """ - List of packages to be removed from the upgrade transaction. The default - is initial-setup which should be removed to avoid it asking for EULA - acceptance during upgrade. If you override this, remember to include the - default values if applicable. - """ From 9f02c14cde82987a1293d593caf5dfacdac1d891 Mon Sep 17 00:00:00 2001 From: Michal Hecko Date: Thu, 3 Oct 2024 16:11:46 +0200 Subject: [PATCH 03/28] rework RHUI config Rework RHUI configuration from 'providing all input parameters to the decision algorithm' into 'here is how the final decision should look like'. --- repos/system_upgrade/common/configs/rhui.py | 82 +++++++++------------ 1 file changed, 35 insertions(+), 47 deletions(-) diff --git a/repos/system_upgrade/common/configs/rhui.py b/repos/system_upgrade/common/configs/rhui.py index 55b96cb211..7b3b2ce8c3 100644 --- a/repos/system_upgrade/common/configs/rhui.py +++ b/repos/system_upgrade/common/configs/rhui.py @@ -10,20 +10,9 @@ from leapp.models import fields -class RhuiSrcPkg(Config): - section = "rhui" - name = "src_pkg" - type_ = fields.String(default="rhui") - default = "rhui" - description = """ - The name of the source RHUI client RPM (installed on the system). - Default: rhui. - """ - - class RhuiTargetPkg(Config): section = "rhui" - name = "target_pkg" + name = "target_client_name" type_ = fields.String(default="rhui") default = "rhui" description = """ @@ -31,51 +20,50 @@ class RhuiTargetPkg(Config): Default: rhui """ - -class RhuiLeappRhuiPkg(Config): +class RhuiCloudProvider(Config): section = "rhui" - name = "leapp_rhui_pkg" - type_ = fields.String(default="leapp-rhui") - default = "leapp-rhui" + name = "cloud_provider" + type_ = fields.String(default="rhui") + default = "provider" description = """ - The name of the leapp-rhui RPM. Default: leapp-rhui - """ + Cloud provider name that should be used internally by leapp. + Leapp recognizes the following cloud providers: + - azure + - azure-sap-apps + - azure-sap-ha + - aws + - aws-sap + - google + - google-sap + + Cloud provider information is used for triggering some provider-specific modifications. The value also + influences how leapp determines target repositories to enable. + """ -class RhuiLeappRhuiPkgRepo(Config): +class RhuiUpgradeFiles(Config): section = "rhui" - name = "leapp_rhui_pkg_repo" - type_ = fields.String(default="rhel-base") - default = "rhel-base" + name = "upgrade_files" + type_ = fields.StringMap(fields.String()) description = """ - The repository ID containing the specified leapp-rhui RPM. - Default: rhel-base - """ + A mapping from source file paths to the destination where should they be + placed in the upgrade container. + Typically, these files should be provided by leapp-rhui- packages. -all_rhui_cfg = (RhuiSrcPkg, RhuiTargetPkg, RhuiLeappRhuiPkg, RhuiLeappRhuiPkg) -# Usage: from configs import rhui -# class MyActor: -# [...] -# configs = all_rhui_cfg + (MyConfig,) + These files are needed to facilitate access to target repositories. Typical examples are: repofile(s), + certificates and keys. + """ -# TODO: We need to implement fields.Map before this can be enabled -''' -class RhuiFileMap(Config): +class RhuiEnabledTargetRepositories(Config): section = "rhui" - name = "file_map" - type_ = fields.Map(fields.String()) + name = "enabled_target_repositories" + type_ = fields.List(fields.String()) description = """ - Define directories to which paritcular files provided by the leapp-rhui - RPM should be installed. The files in 'files_map' are provided by - special Leapp rpms (per cloud) and are supposed to be delivered into the - repos/system_upgrade/common/files/rhui/ directory. + List of target repositories enabled during the upgrade. Similar to executing leapp with --enablerepo. - These files are usually needed to get access to the target system repositories - using RHUI. Typically these are certificates, keys, and repofiles with the - target RHUI repositories. - - The key is the name of the file, the value is the expected directory - where the file should be installed on the upgraded system. + The repositories to be enabled need to be either in the repofiles givin within `upgrade_files` option, + or in repofiles present on the source system. """ -''' + +all_rhui_cfg = (RhuiTargetPkg, RhuiUpgradeFiles, RhuiEnabledTargetRepositories, RhuiCloudProvider) From f9258af0aa8af0c8744cca9382908d00b7878d96 Mon Sep 17 00:00:00 2001 From: Michal Hecko Date: Thu, 3 Oct 2024 16:41:19 +0200 Subject: [PATCH 04/28] add source client back to RHUI config leapp needs those to know what to uninstall during the upgrade --- repos/system_upgrade/common/configs/rhui.py | 45 +++++++++++++++++---- 1 file changed, 38 insertions(+), 7 deletions(-) diff --git a/repos/system_upgrade/common/configs/rhui.py b/repos/system_upgrade/common/configs/rhui.py index 7b3b2ce8c3..ede3b566a8 100644 --- a/repos/system_upgrade/common/configs/rhui.py +++ b/repos/system_upgrade/common/configs/rhui.py @@ -10,16 +10,28 @@ from leapp.models import fields +class RhuiSourcePkg(Config): + section = "rhui" + name = "source_clients" + type_ = fields.List(fields.String(default=["rhui"])) + default = ["rhui"] + description = """ + The name of the source RHUI client RPMs (to be removed from the system). + Default: rhui + """ + + class RhuiTargetPkg(Config): section = "rhui" - name = "target_client_name" - type_ = fields.String(default="rhui") - default = "rhui" + name = "target_clients" + type_ = fields.List(fields.String(default=["rhui"])) + default = ["rhui"] description = """ The name of the target RHUI client RPM (to be installed on the system). Default: rhui """ + class RhuiCloudProvider(Config): section = "rhui" name = "cloud_provider" @@ -30,17 +42,35 @@ class RhuiCloudProvider(Config): Leapp recognizes the following cloud providers: - azure - - azure-sap-apps - - azure-sap-ha - aws - - aws-sap - google - - google-sap Cloud provider information is used for triggering some provider-specific modifications. The value also influences how leapp determines target repositories to enable. """ + +# @Note(mhecko): We likely don't need this. We need the variant primarily to grab files from a correct directory +# in leapp-rhui- folders. +class RhuiCloudVariant(Config): + section = "rhui" + name = "image_variant" + type_ = fields.String(default="ordinary") + default = "ordinary" + description = """ + RHEL variant of the source system - is the source system SAP-specific image? + + Leapp recognizes the following cloud providers: + - ordinary # The source system has not been deployed from a RHEL with SAP image + - sap # RHEL SAP images + - sap-apps # RHEL SAP Apps images (Azure only) + - sap-ha # RHEL HA Apps images (HA only) + + Cloud provider information is used for triggering some provider-specific modifications. The value also + influences how leapp determines target repositories to enable. + """ + + class RhuiUpgradeFiles(Config): section = "rhui" name = "upgrade_files" @@ -55,6 +85,7 @@ class RhuiUpgradeFiles(Config): certificates and keys. """ + class RhuiEnabledTargetRepositories(Config): section = "rhui" name = "enabled_target_repositories" From 52e32defc8dcfe9575a2494b2d823d279cfbbb4b Mon Sep 17 00:00:00 2001 From: Michal Hecko Date: Thu, 3 Oct 2024 16:53:08 +0200 Subject: [PATCH 05/28] check_rhui: read config --- .../common/actors/cloud/checkrhui/actor.py | 6 ++- .../cloud/checkrhui/libraries/checkrhui.py | 54 +++++++++++++++++-- 2 files changed, 54 insertions(+), 6 deletions(-) diff --git a/repos/system_upgrade/common/actors/cloud/checkrhui/actor.py b/repos/system_upgrade/common/actors/cloud/checkrhui/actor.py index 593e73e51f..d6276c08b8 100644 --- a/repos/system_upgrade/common/actors/cloud/checkrhui/actor.py +++ b/repos/system_upgrade/common/actors/cloud/checkrhui/actor.py @@ -1,4 +1,5 @@ from leapp.actors import Actor +from leapp.configs.common.rhui import all_rhui_cfg from leapp.libraries.actor import checkrhui as checkrhui_lib from leapp.models import ( CopyFile, @@ -8,7 +9,8 @@ RequiredTargetUserspacePackages, RHUIInfo, RpmTransactionTasks, - TargetUserSpacePreupgradeTasks + TargetRepositories, + TargetUserSpacePreupgradeTasks, ) from leapp.reporting import Report from leapp.tags import FactsPhaseTag, IPUWorkflowTag @@ -21,6 +23,7 @@ class CheckRHUI(Actor): """ name = 'checkrhui' + config_schemas = all_rhui_cfg consumes = (InstalledRPM,) produces = ( KernelCmdlineArg, @@ -28,6 +31,7 @@ class CheckRHUI(Actor): RequiredTargetUserspacePackages, Report, DNFPluginTask, RpmTransactionTasks, + TargetRepositories, TargetUserSpacePreupgradeTasks, CopyFile, ) diff --git a/repos/system_upgrade/common/actors/cloud/checkrhui/libraries/checkrhui.py b/repos/system_upgrade/common/actors/cloud/checkrhui/libraries/checkrhui.py index 3b2179175e..32bc7d181d 100644 --- a/repos/system_upgrade/common/actors/cloud/checkrhui/libraries/checkrhui.py +++ b/repos/system_upgrade/common/actors/cloud/checkrhui/libraries/checkrhui.py @@ -9,10 +9,12 @@ from leapp.libraries.stdlib import api from leapp.models import ( CopyFile, + CustomTargetRepository, DNFPluginTask, InstalledRPM, RHUIInfo, RpmTransactionTasks, + TargetRepositories, TargetRHUIPostInstallTasks, TargetRHUIPreInstallTasks, TargetRHUISetupInfo, @@ -291,11 +293,11 @@ def produce_rhui_info_to_setup_target(rhui_family, source_setup_desc, target_set api.produce(rhui_info) -def produce_rpms_to_install_into_target(source_setup, target_setup): - to_install = sorted(target_setup.clients - source_setup.clients) - to_remove = sorted(source_setup.clients - target_setup.clients) +def produce_rpms_to_install_into_target(source_clients, target_clients): + to_install = sorted(target_clients - source_clients) + to_remove = sorted(source_clients - target_clients) - api.produce(TargetUserSpacePreupgradeTasks(install_rpms=sorted(target_setup.clients))) + api.produce(TargetUserSpacePreupgradeTasks(install_rpms=sorted(target_clients))) if to_install or to_remove: api.produce(RpmTransactionTasks(to_install=to_install, to_remove=to_remove)) @@ -316,7 +318,47 @@ def inform_about_upgrade_with_rhui_without_no_rhsm(): return False +def emit_rhui_setup_tasks_based_on_config(rhui_config_dict): + files_to_copy_into_overlay = [CopyFile(src=key, dst=value) for key, value in rhui_config_dict['upgrade_files']] + preinstall_tasks = TargetRHUIPreInstallTasks(files_to_copy_into_overlay=files_to_copy_into_overlay) + + target_client_setup_info = TargetRHUISetupInfo( + preinstall_tasks=preinstall_tasks, + postinstall_tasks=TargetRHUIPostInstallTasks(), + bootstrap_target_client=False, # We don't need to install the client into overlay - user provided all files + ) + + rhui_info = RHUIInfo( + provider=rhui_config_dict['cloud_provider'], + variant=rhui_config_dict['image_variant'], + src_client_pkg_names=list(), + target_client_pkg_names=rhui_config_dict['target_clients_names'], + target_client_setup_info=target_client_setup_info + ) + api.produce(rhui_info) + + +def request_configured_repos_to_be_enabled(rhui_config): + custom_repos = [CustomTargetRepository(repoid=repoid) for repoid in rhui_config['enabled_target_repositories']] + if custom_repos: + target_repos = TargetRepositories(custom_repos=custom_repos) + api.produce(target_repos) + + def process(): + rhui_config = api.current_actor().config['rhui'] + expected_fields = ( + 'target_clients', 'source_clients', 'image_variant', + 'cloud_provider', 'upgrade_files', 'enabled_target_repositories' + ) + + if all(expected_field in rhui_config for expected_field in expected_fields): + api.current_logger().info('Skipping RHUI upgrade auto-configuration - using provided config instead.') + emit_rhui_setup_tasks_based_on_config(rhui_config) + produce_rpms_to_install_into_target(set(rhui_config['source_clients']), set(rhui_config['target_clients'])) + request_configured_repos_to_be_enabled(rhui_config) + return + installed_rpm = itertools.chain(*[installed_rpm_msg.items for installed_rpm_msg in api.consume(InstalledRPM)]) installed_pkgs = {rpm.name for rpm in installed_rpm} @@ -342,7 +384,9 @@ def process(): # Instruction on how to access the target content produce_rhui_info_to_setup_target(src_rhui_setup.family, src_rhui_setup.description, target_setup_desc) - produce_rpms_to_install_into_target(src_rhui_setup.description, target_setup_desc) + source_clients = src_rhui_setup.description.clients + target_clients = target_setup_desc.clients + produce_rpms_to_install_into_target(source_clients, target_clients) if src_rhui_setup.family.provider == rhui.RHUIProvider.AWS: # We have to disable Amazon-id plugin in the initramdisk phase as there is no network From cfee78b20364369059189f4594fdfd0aa578b6b1 Mon Sep 17 00:00:00 2001 From: David Kubek Date: Fri, 4 Oct 2024 10:35:36 +0200 Subject: [PATCH 06/28] Fix broken imports --- commands/upgrade/__init__.py | 2 +- repos/system_upgrade/common/configs/rhui.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/commands/upgrade/__init__.py b/commands/upgrade/__init__.py index 0de7148ced..8dbeda0a12 100644 --- a/commands/upgrade/__init__.py +++ b/commands/upgrade/__init__.py @@ -2,7 +2,7 @@ import sys import uuid -from leapp.actor import config as actor_config +from leapp.actors import config as actor_config from leapp.cli.commands import command_utils from leapp.cli.commands.config import get_config from leapp.cli.commands.upgrade import breadcrumbs, util diff --git a/repos/system_upgrade/common/configs/rhui.py b/repos/system_upgrade/common/configs/rhui.py index ede3b566a8..c115afb8f7 100644 --- a/repos/system_upgrade/common/configs/rhui.py +++ b/repos/system_upgrade/common/configs/rhui.py @@ -6,7 +6,7 @@ RHUI data to correct the in-place upgrade process. """ -from leapp.actors.configs import Config +from leapp.actors.config import Config from leapp.models import fields From 549942760e86425832663a57aaf1ed8676561ea2 Mon Sep 17 00:00:00 2001 From: Michal Hecko Date: Sat, 5 Oct 2024 17:56:28 +0200 Subject: [PATCH 07/28] rhui: change how configs are accessed --- .../cloud/checkrhui/libraries/checkrhui.py | 39 ++++++++++----- repos/system_upgrade/common/configs/rhui.py | 48 ++++++++++++++----- 2 files changed, 63 insertions(+), 24 deletions(-) diff --git a/repos/system_upgrade/common/actors/cloud/checkrhui/libraries/checkrhui.py b/repos/system_upgrade/common/actors/cloud/checkrhui/libraries/checkrhui.py index 32bc7d181d..7fbd17d184 100644 --- a/repos/system_upgrade/common/actors/cloud/checkrhui/libraries/checkrhui.py +++ b/repos/system_upgrade/common/actors/cloud/checkrhui/libraries/checkrhui.py @@ -6,6 +6,16 @@ from leapp.exceptions import StopActorExecutionError from leapp.libraries.common import rhsm, rhui from leapp.libraries.common.config import version +import leapp.configs.common.rhui as rhui_config_lib +from leapp.configs.common.rhui import ( # Import all config fields so we are not using their name attributes directly + RhuiEnabledTargetRepositories, + RhuiCloudProvider, + RhuiCloudVariant, + RhuiSourcePkgs, + RhuiTargetPkgs, + RhuiUpgradeFiles, + RhuiUseConfig, +) from leapp.libraries.stdlib import api from leapp.models import ( CopyFile, @@ -21,6 +31,7 @@ TargetUserSpacePreupgradeTasks ) + MatchingSetup = namedtuple('MatchingSetup', ['family', 'description']) @@ -319,7 +330,8 @@ def inform_about_upgrade_with_rhui_without_no_rhsm(): def emit_rhui_setup_tasks_based_on_config(rhui_config_dict): - files_to_copy_into_overlay = [CopyFile(src=key, dst=value) for key, value in rhui_config_dict['upgrade_files']] + config_upgrade_files = rhui_config_dict[RhuiUpgradeFiles.name] + files_to_copy_into_overlay = [CopyFile(src=key, dst=value) for key, value in config_upgrade_files.items()] preinstall_tasks = TargetRHUIPreInstallTasks(files_to_copy_into_overlay=files_to_copy_into_overlay) target_client_setup_info = TargetRHUISetupInfo( @@ -329,33 +341,34 @@ def emit_rhui_setup_tasks_based_on_config(rhui_config_dict): ) rhui_info = RHUIInfo( - provider=rhui_config_dict['cloud_provider'], - variant=rhui_config_dict['image_variant'], + provider=rhui_config_dict[RhuiCloudProvider.name], + variant=rhui_config_dict[RhuiCloudVariant.name], src_client_pkg_names=list(), - target_client_pkg_names=rhui_config_dict['target_clients_names'], + target_client_pkg_names=rhui_config_dict[RhuiTargetPkgs.name], target_client_setup_info=target_client_setup_info ) api.produce(rhui_info) def request_configured_repos_to_be_enabled(rhui_config): - custom_repos = [CustomTargetRepository(repoid=repoid) for repoid in rhui_config['enabled_target_repositories']] + config_repos_to_enable = rhui_config[RhuiEnabledTargetRepositories.name] + custom_repos = [CustomTargetRepository(repoid=repoid) for repoid in config_repos_to_enable] if custom_repos: - target_repos = TargetRepositories(custom_repos=custom_repos) + target_repos = TargetRepositories(custom_repos=custom_repos, rhel_repos=[]) api.produce(target_repos) def process(): - rhui_config = api.current_actor().config['rhui'] - expected_fields = ( - 'target_clients', 'source_clients', 'image_variant', - 'cloud_provider', 'upgrade_files', 'enabled_target_repositories' - ) + rhui_config = api.current_actor().config[rhui_config_lib.RHUI_CONFIG_SECTION] - if all(expected_field in rhui_config for expected_field in expected_fields): + if rhui_config[RhuiUseConfig.name]: api.current_logger().info('Skipping RHUI upgrade auto-configuration - using provided config instead.') emit_rhui_setup_tasks_based_on_config(rhui_config) - produce_rpms_to_install_into_target(set(rhui_config['source_clients']), set(rhui_config['target_clients'])) + + src_clients = set(rhui_config[RhuiSourcePkgs.name]) + target_clients = set(rhui_config[RhuiTargetPkgs.name]) + produce_rpms_to_install_into_target(src_clients, target_clients) + request_configured_repos_to_be_enabled(rhui_config) return diff --git a/repos/system_upgrade/common/configs/rhui.py b/repos/system_upgrade/common/configs/rhui.py index c115afb8f7..1cd9ff0fe7 100644 --- a/repos/system_upgrade/common/configs/rhui.py +++ b/repos/system_upgrade/common/configs/rhui.py @@ -10,8 +10,23 @@ from leapp.models import fields -class RhuiSourcePkg(Config): - section = "rhui" +RHUI_CONFIG_SECTION = 'rhui' + +# @Note(mhecko): We use to distinguish config instantiated from default values that we should ignore +# # Maybe we could make all config values None and detect it that way, but then we cannot +# # give the user an example how the config should look like. +class RhuiUseConfig(Config): + section = RHUI_CONFIG_SECTION + name = "use_config" + type_ = fields.Boolean(default=False) + default = False + description = """ + Use values provided in the configuration file to override leapp's decisions. + """ + + +class RhuiSourcePkgs(Config): + section = RHUI_CONFIG_SECTION name = "source_clients" type_ = fields.List(fields.String(default=["rhui"])) default = ["rhui"] @@ -21,10 +36,10 @@ class RhuiSourcePkg(Config): """ -class RhuiTargetPkg(Config): - section = "rhui" +class RhuiTargetPkgs(Config): + section = RHUI_CONFIG_SECTION name = "target_clients" - type_ = fields.List(fields.String(default=["rhui"])) + type_ = fields.List(fields.String(), default=["rhui"]) default = ["rhui"] description = """ The name of the target RHUI client RPM (to be installed on the system). @@ -33,7 +48,7 @@ class RhuiTargetPkg(Config): class RhuiCloudProvider(Config): - section = "rhui" + section = RHUI_CONFIG_SECTION name = "cloud_provider" type_ = fields.String(default="rhui") default = "provider" @@ -53,7 +68,7 @@ class RhuiCloudProvider(Config): # @Note(mhecko): We likely don't need this. We need the variant primarily to grab files from a correct directory # in leapp-rhui- folders. class RhuiCloudVariant(Config): - section = "rhui" + section = RHUI_CONFIG_SECTION name = "image_variant" type_ = fields.String(default="ordinary") default = "ordinary" @@ -72,9 +87,10 @@ class RhuiCloudVariant(Config): class RhuiUpgradeFiles(Config): - section = "rhui" + section = RHUI_CONFIG_SECTION name = "upgrade_files" type_ = fields.StringMap(fields.String()) + default = dict() description = """ A mapping from source file paths to the destination where should they be placed in the upgrade container. @@ -87,7 +103,7 @@ class RhuiUpgradeFiles(Config): class RhuiEnabledTargetRepositories(Config): - section = "rhui" + section = RHUI_CONFIG_SECTION name = "enabled_target_repositories" type_ = fields.List(fields.String()) description = """ @@ -96,5 +112,15 @@ class RhuiEnabledTargetRepositories(Config): The repositories to be enabled need to be either in the repofiles givin within `upgrade_files` option, or in repofiles present on the source system. """ - -all_rhui_cfg = (RhuiTargetPkg, RhuiUpgradeFiles, RhuiEnabledTargetRepositories, RhuiCloudProvider) + default = list() + + +all_rhui_cfg = ( + RhuiTargetPkgs, + RhuiUpgradeFiles, + RhuiEnabledTargetRepositories, + RhuiCloudProvider, + RhuiCloudVariant, + RhuiSourcePkgs, + RhuiUseConfig +) From ac879afe0cf4fa88577d7ccd11295261c193da5e Mon Sep 17 00:00:00 2001 From: Michal Hecko Date: Sat, 5 Oct 2024 18:06:47 +0200 Subject: [PATCH 08/28] add example rhui config --- etc/leapp/actor_conf.d/rhui.yml | 12 ++++++++++++ packaging/leapp-repository.spec | 1 + 2 files changed, 13 insertions(+) create mode 100644 etc/leapp/actor_conf.d/rhui.yml diff --git a/etc/leapp/actor_conf.d/rhui.yml b/etc/leapp/actor_conf.d/rhui.yml new file mode 100644 index 0000000000..d3ab5cbf9c --- /dev/null +++ b/etc/leapp/actor_conf.d/rhui.yml @@ -0,0 +1,12 @@ +rhui: + use_config : True + source_clients: [ rh-amazon-rhui-client ] + target_clients: [ rh-amazon-rhui-client ] + cloud_provider: aws + upgrade_files: + /usr/share/leapp-repository/repositories/system_upgrade/common/files/rhui/leapp-rhui-aws/rhui-client-config-server-9.crt : /etc/pki/rhui/product/ + /usr/share/leapp-repository/repositories/system_upgrade/common/files/rhui/leapp-rhui-aws/rhui-client-config-server-9.key : /etc/pki/rhui/private/ + /usr/share/leapp-repository/repositories/system_upgrade/common/files/rhui/leapp-rhui-aws/leapp-aws.repo : /etc/yum.repos.d/ + enabled_target_repositories: + - rhel-9-appstream-rhui-rpms + - rhel-9-baseos-rhui-rpms diff --git a/packaging/leapp-repository.spec b/packaging/leapp-repository.spec index 146afc456b..299393e128 100644 --- a/packaging/leapp-repository.spec +++ b/packaging/leapp-repository.spec @@ -243,6 +243,7 @@ install -m 0755 -d %{buildroot}%{_sysconfdir}/leapp/transaction/ install -m 0755 -d %{buildroot}%{_sysconfdir}/leapp/files/ install -m 0644 etc/leapp/transaction/* %{buildroot}%{_sysconfdir}/leapp/transaction install -m 0644 etc/leapp/files/* %{buildroot}%{_sysconfdir}/leapp/files +install -m 0644 etc/leapp/actor_conf.d/* %{buildroot}%{_sysconfdir}/leapp/actor_conf.d # install CLI commands for the leapp utility on the expected path install -m 0755 -d %{buildroot}%{leapp_python_sitelib}/leapp/cli/ From a8f1372d27a8a7c59b4030d02d33d705cc75f006 Mon Sep 17 00:00:00 2001 From: Michal Hecko Date: Sun, 6 Oct 2024 21:01:13 +0200 Subject: [PATCH 09/28] spec: make actor_conf.d before trying to place there stuff --- packaging/leapp-repository.spec | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packaging/leapp-repository.spec b/packaging/leapp-repository.spec index 299393e128..4473d7d455 100644 --- a/packaging/leapp-repository.spec +++ b/packaging/leapp-repository.spec @@ -243,6 +243,9 @@ install -m 0755 -d %{buildroot}%{_sysconfdir}/leapp/transaction/ install -m 0755 -d %{buildroot}%{_sysconfdir}/leapp/files/ install -m 0644 etc/leapp/transaction/* %{buildroot}%{_sysconfdir}/leapp/transaction install -m 0644 etc/leapp/files/* %{buildroot}%{_sysconfdir}/leapp/files + +# Actor configuration +install -m 0755 -d %{buildroot}%{_sysconfdir}/leapp/actor_conf.d/ install -m 0644 etc/leapp/actor_conf.d/* %{buildroot}%{_sysconfdir}/leapp/actor_conf.d # install CLI commands for the leapp utility on the expected path @@ -294,6 +297,7 @@ done; %{_sysconfdir}/leapp/transaction/* %{repositorydir}/* %{leapp_python_sitelib}/leapp/cli/commands/* +%{_sysconfdir}/leapp/actor_conf.d/* %files -n %{lpr_name}-deps From 13508f65c255d39ff97a36268397856e0e40cccb Mon Sep 17 00:00:00 2001 From: Michal Hecko Date: Sun, 6 Oct 2024 21:29:43 +0200 Subject: [PATCH 10/28] cmd(preupgrade): load actor config --- commands/preupgrade/__init__.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/commands/preupgrade/__init__.py b/commands/preupgrade/__init__.py index 5a89069f6a..cb07d1d96e 100644 --- a/commands/preupgrade/__init__.py +++ b/commands/preupgrade/__init__.py @@ -2,6 +2,7 @@ import sys import uuid +from leapp.actors import config as actor_config from leapp.cli.commands import command_utils from leapp.cli.commands.config import get_config from leapp.cli.commands.upgrade import breadcrumbs, util @@ -60,6 +61,13 @@ def preupgrade(args, breadcrumbs): raise CommandError(exc.message) workflow = repositories.lookup_workflow('IPUWorkflow')() + + # Read the Actor Config and validate it against the schemas saved in the configuration. + actor_config_schemas = tuple(actor.config_schemas for actor in repositories.actors) + actor_config_schemas = actor_config.normalize_schemas(actor_config_schemas) + actor_config_path = cfg.get('actor_config', 'path') + actor_config.load(actor_config_path, actor_config_schemas) + util.warn_if_unsupported(configuration) util.process_whitelist_experimental(repositories, workflow, configuration, logger) with beautify_actor_exception(): From f8cbdc9d6fa14871a5b92d6b18c742aeb4d50d72 Mon Sep 17 00:00:00 2001 From: Michal Hecko Date: Tue, 15 Oct 2024 07:56:24 +0200 Subject: [PATCH 11/28] make all rhui config fields required --- etc/leapp/actor_conf.d/rhui.yml | 2 +- .../cloud/checkrhui/libraries/checkrhui.py | 23 ++++++++++++++++++- repos/system_upgrade/common/configs/rhui.py | 10 ++++---- 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/etc/leapp/actor_conf.d/rhui.yml b/etc/leapp/actor_conf.d/rhui.yml index d3ab5cbf9c..71f36e31fc 100644 --- a/etc/leapp/actor_conf.d/rhui.yml +++ b/etc/leapp/actor_conf.d/rhui.yml @@ -1,5 +1,5 @@ rhui: - use_config : True + use_config : False source_clients: [ rh-amazon-rhui-client ] target_clients: [ rh-amazon-rhui-client ] cloud_provider: aws diff --git a/repos/system_upgrade/common/actors/cloud/checkrhui/libraries/checkrhui.py b/repos/system_upgrade/common/actors/cloud/checkrhui/libraries/checkrhui.py index 7fbd17d184..b53b3ba0bf 100644 --- a/repos/system_upgrade/common/actors/cloud/checkrhui/libraries/checkrhui.py +++ b/repos/system_upgrade/common/actors/cloud/checkrhui/libraries/checkrhui.py @@ -3,7 +3,7 @@ from collections import namedtuple from leapp import reporting -from leapp.exceptions import StopActorExecutionError +from leapp.exceptions import StopActorExecution, StopActorExecutionError from leapp.libraries.common import rhsm, rhui from leapp.libraries.common.config import version import leapp.configs.common.rhui as rhui_config_lib @@ -358,12 +358,33 @@ def request_configured_repos_to_be_enabled(rhui_config): api.produce(target_repos) +def stop_with_err_if_config_invalid(config): + required_fields = [ + RhuiEnabledTargetRepositories, + RhuiCloudProvider, + # RhuiCloudVariant, <- this is not required + RhuiSourcePkgs, + RhuiTargetPkgs, + RhuiUpgradeFiles, + ] + + missing_fields = tuple(field for field in required_fields if not config[field.name]) + if missing_fields: + field_names = (field.name for field in missing_fields) + missing_fields_str = ', '.join(field_names) + details = 'The following required RHUI config fields are missing or they are set to an empty value: {}' + details = details.format(missing_fields_str) + raise StopActorExecution('Provided RHUI config is missing values for required fields.', + details={'details': details}) + + def process(): rhui_config = api.current_actor().config[rhui_config_lib.RHUI_CONFIG_SECTION] if rhui_config[RhuiUseConfig.name]: api.current_logger().info('Skipping RHUI upgrade auto-configuration - using provided config instead.') emit_rhui_setup_tasks_based_on_config(rhui_config) + stop_with_err_if_config_invalid(rhui_config) src_clients = set(rhui_config[RhuiSourcePkgs.name]) target_clients = set(rhui_config[RhuiTargetPkgs.name]) diff --git a/repos/system_upgrade/common/configs/rhui.py b/repos/system_upgrade/common/configs/rhui.py index 1cd9ff0fe7..aa20490b01 100644 --- a/repos/system_upgrade/common/configs/rhui.py +++ b/repos/system_upgrade/common/configs/rhui.py @@ -28,8 +28,8 @@ class RhuiUseConfig(Config): class RhuiSourcePkgs(Config): section = RHUI_CONFIG_SECTION name = "source_clients" - type_ = fields.List(fields.String(default=["rhui"])) - default = ["rhui"] + type_ = fields.List(fields.String()) + default = [] description = """ The name of the source RHUI client RPMs (to be removed from the system). Default: rhui @@ -39,8 +39,8 @@ class RhuiSourcePkgs(Config): class RhuiTargetPkgs(Config): section = RHUI_CONFIG_SECTION name = "target_clients" - type_ = fields.List(fields.String(), default=["rhui"]) - default = ["rhui"] + type_ = fields.List(fields.String()) + default = [] description = """ The name of the target RHUI client RPM (to be installed on the system). Default: rhui @@ -51,7 +51,7 @@ class RhuiCloudProvider(Config): section = RHUI_CONFIG_SECTION name = "cloud_provider" type_ = fields.String(default="rhui") - default = "provider" + default = "" description = """ Cloud provider name that should be used internally by leapp. From e437148541a7689c3f210da8090117e40f18ca74 Mon Sep 17 00:00:00 2001 From: Michal Hecko Date: Tue, 15 Oct 2024 07:59:48 +0200 Subject: [PATCH 12/28] rename target repositories field --- etc/leapp/actor_conf.d/rhui.yml | 2 +- .../common/actors/cloud/checkrhui/libraries/checkrhui.py | 6 +++--- repos/system_upgrade/common/configs/rhui.py | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/etc/leapp/actor_conf.d/rhui.yml b/etc/leapp/actor_conf.d/rhui.yml index 71f36e31fc..5c4bba5b5a 100644 --- a/etc/leapp/actor_conf.d/rhui.yml +++ b/etc/leapp/actor_conf.d/rhui.yml @@ -7,6 +7,6 @@ rhui: /usr/share/leapp-repository/repositories/system_upgrade/common/files/rhui/leapp-rhui-aws/rhui-client-config-server-9.crt : /etc/pki/rhui/product/ /usr/share/leapp-repository/repositories/system_upgrade/common/files/rhui/leapp-rhui-aws/rhui-client-config-server-9.key : /etc/pki/rhui/private/ /usr/share/leapp-repository/repositories/system_upgrade/common/files/rhui/leapp-rhui-aws/leapp-aws.repo : /etc/yum.repos.d/ - enabled_target_repositories: + rhui_target_repositories_to_use: - rhel-9-appstream-rhui-rpms - rhel-9-baseos-rhui-rpms diff --git a/repos/system_upgrade/common/actors/cloud/checkrhui/libraries/checkrhui.py b/repos/system_upgrade/common/actors/cloud/checkrhui/libraries/checkrhui.py index b53b3ba0bf..e2047f4a2e 100644 --- a/repos/system_upgrade/common/actors/cloud/checkrhui/libraries/checkrhui.py +++ b/repos/system_upgrade/common/actors/cloud/checkrhui/libraries/checkrhui.py @@ -8,11 +8,11 @@ from leapp.libraries.common.config import version import leapp.configs.common.rhui as rhui_config_lib from leapp.configs.common.rhui import ( # Import all config fields so we are not using their name attributes directly - RhuiEnabledTargetRepositories, RhuiCloudProvider, RhuiCloudVariant, RhuiSourcePkgs, RhuiTargetPkgs, + RhuiTargetRepositoriesToUse, RhuiUpgradeFiles, RhuiUseConfig, ) @@ -351,7 +351,7 @@ def emit_rhui_setup_tasks_based_on_config(rhui_config_dict): def request_configured_repos_to_be_enabled(rhui_config): - config_repos_to_enable = rhui_config[RhuiEnabledTargetRepositories.name] + config_repos_to_enable = rhui_config[RhuiTargetRepositoriesToUse.name] custom_repos = [CustomTargetRepository(repoid=repoid) for repoid in config_repos_to_enable] if custom_repos: target_repos = TargetRepositories(custom_repos=custom_repos, rhel_repos=[]) @@ -360,7 +360,7 @@ def request_configured_repos_to_be_enabled(rhui_config): def stop_with_err_if_config_invalid(config): required_fields = [ - RhuiEnabledTargetRepositories, + RhuiTargetRepositoriesToUse, RhuiCloudProvider, # RhuiCloudVariant, <- this is not required RhuiSourcePkgs, diff --git a/repos/system_upgrade/common/configs/rhui.py b/repos/system_upgrade/common/configs/rhui.py index aa20490b01..ecf165cbbd 100644 --- a/repos/system_upgrade/common/configs/rhui.py +++ b/repos/system_upgrade/common/configs/rhui.py @@ -102,9 +102,9 @@ class RhuiUpgradeFiles(Config): """ -class RhuiEnabledTargetRepositories(Config): +class RhuiTargetRepositoriesToUse(Config): section = RHUI_CONFIG_SECTION - name = "enabled_target_repositories" + name = "rhui_target_repositories_to_use" type_ = fields.List(fields.String()) description = """ List of target repositories enabled during the upgrade. Similar to executing leapp with --enablerepo. @@ -118,7 +118,7 @@ class RhuiEnabledTargetRepositories(Config): all_rhui_cfg = ( RhuiTargetPkgs, RhuiUpgradeFiles, - RhuiEnabledTargetRepositories, + RhuiTargetRepositoriesToUse, RhuiCloudProvider, RhuiCloudVariant, RhuiSourcePkgs, From bea0d31a33c6f2d121b1ddfcc1ae848299cf6d40 Mon Sep 17 00:00:00 2001 From: Michal Hecko Date: Wed, 16 Oct 2024 16:48:36 +0200 Subject: [PATCH 13/28] fix typo in rhui config docs --- repos/system_upgrade/common/configs/rhui.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repos/system_upgrade/common/configs/rhui.py b/repos/system_upgrade/common/configs/rhui.py index ecf165cbbd..1969c74370 100644 --- a/repos/system_upgrade/common/configs/rhui.py +++ b/repos/system_upgrade/common/configs/rhui.py @@ -109,7 +109,7 @@ class RhuiTargetRepositoriesToUse(Config): description = """ List of target repositories enabled during the upgrade. Similar to executing leapp with --enablerepo. - The repositories to be enabled need to be either in the repofiles givin within `upgrade_files` option, + The repositories to be enabled need to be either in the repofiles listed in the `upgrade_files` field, or in repofiles present on the source system. """ default = list() From 70f7d5b85d28a7f8074945859dca3e6b73d65866 Mon Sep 17 00:00:00 2001 From: Michal Hecko Date: Wed, 16 Oct 2024 16:51:58 +0200 Subject: [PATCH 14/28] fix linter complaints in rhui config --- repos/system_upgrade/common/configs/rhui.py | 1 + 1 file changed, 1 insertion(+) diff --git a/repos/system_upgrade/common/configs/rhui.py b/repos/system_upgrade/common/configs/rhui.py index 1969c74370..1eb00e5dbe 100644 --- a/repos/system_upgrade/common/configs/rhui.py +++ b/repos/system_upgrade/common/configs/rhui.py @@ -12,6 +12,7 @@ RHUI_CONFIG_SECTION = 'rhui' + # @Note(mhecko): We use to distinguish config instantiated from default values that we should ignore # # Maybe we could make all config values None and detect it that way, but then we cannot # # give the user an example how the config should look like. From 07a24f756a74de1ab54dab78f1c0c717b541821b Mon Sep 17 00:00:00 2001 From: Michal Hecko Date: Wed, 16 Oct 2024 17:38:36 +0200 Subject: [PATCH 15/28] testutils: add support for configs --- repos/system_upgrade/common/libraries/testutils.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/repos/system_upgrade/common/libraries/testutils.py b/repos/system_upgrade/common/libraries/testutils.py index c538af1a13..a03033c5f8 100644 --- a/repos/system_upgrade/common/libraries/testutils.py +++ b/repos/system_upgrade/common/libraries/testutils.py @@ -4,6 +4,7 @@ from collections import namedtuple from leapp import reporting +from leapp.actors.config import _normalize_config, normalize_schemas from leapp.libraries.common.config import architecture from leapp.models import EnvVar from leapp.utils.deprecation import deprecated @@ -66,10 +67,15 @@ def error(self, *args): def __call__(self): return self +def _make_default_config(actor_config_schema): + """ Make a config dict populated with default values. """ + merged_schema = normalize_schemas((actor_config_schema, )) + return _normalize_config({}, merged_schema) # Will fill default values during normalization + class CurrentActorMocked(object): # pylint:disable=R0904 def __init__(self, arch=architecture.ARCH_X86_64, envars=None, kernel='3.10.0-957.43.1.el7.x86_64', - release_id='rhel', src_ver='7.8', dst_ver='8.1', msgs=None, flavour='default'): + release_id='rhel', src_ver='7.8', dst_ver='8.1', msgs=None, flavour='default', config=None): envarsList = [EnvVar(name=k, value=v) for k, v in envars.items()] if envars else [] version = namedtuple('Version', ['source', 'target'])(src_ver, dst_ver) release = namedtuple('OS_release', ['release_id', 'version_id'])(release_id, src_ver) @@ -82,6 +88,7 @@ def __init__(self, arch=architecture.ARCH_X86_64, envars=None, kernel='3.10.0-95 'configuration', ['architecture', 'kernel', 'leapp_env_vars', 'os_release', 'version', 'flavour'] )(arch, kernel, envarsList, release, version, flavour) self._msgs = msgs or [] + self.config = {} if config is None else config def __call__(self): return self From 7486ce0259a3e4bf64f4167e6e850214c26fd769 Mon Sep 17 00:00:00 2001 From: Michal Hecko Date: Wed, 16 Oct 2024 17:38:59 +0200 Subject: [PATCH 16/28] checkrhui: fix tests feed actor mocks the default configuration --- .../tests/component_test_checkrhui.py | 32 +++++++++++-------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/repos/system_upgrade/common/actors/cloud/checkrhui/tests/component_test_checkrhui.py b/repos/system_upgrade/common/actors/cloud/checkrhui/tests/component_test_checkrhui.py index 27e70eea7f..13f20c1e4d 100644 --- a/repos/system_upgrade/common/actors/cloud/checkrhui/tests/component_test_checkrhui.py +++ b/repos/system_upgrade/common/actors/cloud/checkrhui/tests/component_test_checkrhui.py @@ -4,17 +4,20 @@ import pytest from leapp import reporting +from leapp.configs.common.rhui import all_rhui_cfg from leapp.exceptions import StopActorExecutionError from leapp.libraries.actor import checkrhui as checkrhui_lib from leapp.libraries.common import rhsm, rhui -from leapp.libraries.common.config import mock_configs, version from leapp.libraries.common.rhui import mk_rhui_setup, RHUIFamily -from leapp.libraries.common.testutils import create_report_mocked, CurrentActorMocked, produce_mocked +from leapp.libraries.common.testutils import ( + create_report_mocked, + CurrentActorMocked, + _make_default_config, + produce_mocked +) from leapp.libraries.stdlib import api from leapp.models import ( - CopyFile, InstalledRPM, - RequiredTargetUserspacePackages, RHUIInfo, RPM, RpmTransactionTasks, @@ -23,8 +26,6 @@ TargetRHUISetupInfo, TargetUserSpacePreupgradeTasks ) -from leapp.reporting import Report -from leapp.snactor.fixture import current_actor_context RH_PACKAGER = 'Red Hat, Inc. ' @@ -95,7 +96,8 @@ def mk_cloud_map(variants): ] ) def test_determine_rhui_src_variant(monkeypatch, extra_pkgs, rhui_setups, expected_result): - monkeypatch.setattr(api, 'current_actor', CurrentActorMocked(src_ver='7.9')) + actor = CurrentActorMocked(src_ver='7.9', config=_make_default_config(all_rhui_cfg)) + monkeypatch.setattr(api, 'current_actor', actor) installed_pkgs = {'zip', 'zsh', 'bash', 'grubby'}.union(set(extra_pkgs)) if expected_result and not isinstance(expected_result, RHUIFamily): # An exception @@ -167,7 +169,8 @@ def test_google_specific_customization(provider, should_mutate): ) def test_aws_specific_customization(monkeypatch, rhui_family, target_major, should_mutate): dst_ver = '{major}.0'.format(major=target_major) - monkeypatch.setattr(api, 'current_actor', CurrentActorMocked(dst_ver=dst_ver)) + actor = CurrentActorMocked(dst_ver=dst_ver, config=_make_default_config(all_rhui_cfg)) + monkeypatch.setattr(api, 'current_actor', actor) setup_info = mk_setup_info() checkrhui_lib.customize_rhui_setup_for_aws(rhui_family, setup_info) @@ -215,12 +218,12 @@ def produce_rhui_info_to_setup_target(monkeypatch): def test_produce_rpms_to_install_into_target(monkeypatch): - source_rhui_setup = mk_rhui_setup(clients={'src_pkg'}, leapp_pkg='leapp_pkg') - target_rhui_setup = mk_rhui_setup(clients={'target_pkg'}, leapp_pkg='leapp_pkg') + source_clients = {'src_pkg'} + target_clients = {'target_pkg'} monkeypatch.setattr(api, 'produce', produce_mocked()) - checkrhui_lib.produce_rpms_to_install_into_target(source_rhui_setup, target_rhui_setup) + checkrhui_lib.produce_rpms_to_install_into_target(source_clients, target_clients) assert len(api.produce.model_instances) == 2 userspace_tasks, target_rpm_tasks = api.produce.model_instances[0], api.produce.model_instances[1] @@ -276,7 +279,8 @@ def test_process(monkeypatch, extra_installed_pkgs, skip_rhsm, expected_action): installed_rpms = InstalledRPM(items=installed_pkgs) monkeypatch.setattr(api, 'produce', produce_mocked()) - monkeypatch.setattr(api, 'current_actor', CurrentActorMocked(src_ver='7.9', msgs=[installed_rpms])) + actor = CurrentActorMocked(src_ver='7.9', msgs=[installed_rpms], config=_make_default_config(all_rhui_cfg)) + monkeypatch.setattr(api, 'current_actor', actor) monkeypatch.setattr(reporting, 'create_report', create_report_mocked()) monkeypatch.setattr(rhsm, 'skip_rhsm', lambda: skip_rhsm) monkeypatch.setattr(rhui, 'RHUI_SETUPS', known_setups) @@ -315,11 +319,13 @@ def test_unknown_target_rhui_setup(monkeypatch, is_target_setup_known): installed_rpms = InstalledRPM(items=installed_pkgs) monkeypatch.setattr(api, 'produce', produce_mocked()) - monkeypatch.setattr(api, 'current_actor', CurrentActorMocked(src_ver='7.9', msgs=[installed_rpms])) + actor = CurrentActorMocked(src_ver='7.9', msgs=[installed_rpms], config=_make_default_config(all_rhui_cfg)) + monkeypatch.setattr(api, 'current_actor', actor) monkeypatch.setattr(reporting, 'create_report', create_report_mocked()) monkeypatch.setattr(rhsm, 'skip_rhsm', lambda: True) monkeypatch.setattr(rhui, 'RHUI_SETUPS', known_setups) + if is_target_setup_known: checkrhui_lib.process() assert api.produce.called From 0d044b25c7f296ab715e9aedbba1bbab177e404d Mon Sep 17 00:00:00 2001 From: Michal Hecko Date: Wed, 16 Oct 2024 17:37:26 +0200 Subject: [PATCH 17/28] tmp: install correct leapp branch to run tests --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index a1bb472544..6172566c20 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ pytest==6.2.5; python_version >= '3.6' pyudev==0.22.0 distro==1.5.0 ipaddress==1.0.23 -git+https://github.com/oamg/leapp +git+https://github.com/abadger/leapp.git@actor-config requests # pinning a py27 troublemaking transitive dependency lazy-object-proxy==1.5.2; python_version < '3' From 35dd90075a3e868bac0261949365bf54803865a6 Mon Sep 17 00:00:00 2001 From: Michal Hecko Date: Wed, 16 Oct 2024 17:49:44 +0200 Subject: [PATCH 18/28] checkrhui: fix linter errors in tests --- .../actors/cloud/checkrhui/tests/component_test_checkrhui.py | 1 - 1 file changed, 1 deletion(-) diff --git a/repos/system_upgrade/common/actors/cloud/checkrhui/tests/component_test_checkrhui.py b/repos/system_upgrade/common/actors/cloud/checkrhui/tests/component_test_checkrhui.py index 13f20c1e4d..6998f710f1 100644 --- a/repos/system_upgrade/common/actors/cloud/checkrhui/tests/component_test_checkrhui.py +++ b/repos/system_upgrade/common/actors/cloud/checkrhui/tests/component_test_checkrhui.py @@ -325,7 +325,6 @@ def test_unknown_target_rhui_setup(monkeypatch, is_target_setup_known): monkeypatch.setattr(rhsm, 'skip_rhsm', lambda: True) monkeypatch.setattr(rhui, 'RHUI_SETUPS', known_setups) - if is_target_setup_known: checkrhui_lib.process() assert api.produce.called From 7eaa78afee55fd0603b5168e837761aa275fa029 Mon Sep 17 00:00:00 2001 From: Michal Hecko Date: Wed, 16 Oct 2024 17:55:23 +0200 Subject: [PATCH 19/28] fixup! testutils: add support for configs --- repos/system_upgrade/common/libraries/testutils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/repos/system_upgrade/common/libraries/testutils.py b/repos/system_upgrade/common/libraries/testutils.py index a03033c5f8..afeb360a64 100644 --- a/repos/system_upgrade/common/libraries/testutils.py +++ b/repos/system_upgrade/common/libraries/testutils.py @@ -67,6 +67,7 @@ def error(self, *args): def __call__(self): return self + def _make_default_config(actor_config_schema): """ Make a config dict populated with default values. """ merged_schema = normalize_schemas((actor_config_schema, )) From 418f023745de62caabd19e66c1a30bd4efa26a0c Mon Sep 17 00:00:00 2001 From: Michal Hecko Date: Wed, 16 Oct 2024 17:58:22 +0200 Subject: [PATCH 20/28] fix imports --- .../common/actors/cloud/checkrhui/actor.py | 2 +- .../actors/cloud/checkrhui/libraries/checkrhui.py | 11 +++++------ .../cloud/checkrhui/tests/component_test_checkrhui.py | 2 +- repos/system_upgrade/common/configs/rhui.py | 1 - 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/repos/system_upgrade/common/actors/cloud/checkrhui/actor.py b/repos/system_upgrade/common/actors/cloud/checkrhui/actor.py index d6276c08b8..933ffcb36e 100644 --- a/repos/system_upgrade/common/actors/cloud/checkrhui/actor.py +++ b/repos/system_upgrade/common/actors/cloud/checkrhui/actor.py @@ -10,7 +10,7 @@ RHUIInfo, RpmTransactionTasks, TargetRepositories, - TargetUserSpacePreupgradeTasks, + TargetUserSpacePreupgradeTasks ) from leapp.reporting import Report from leapp.tags import FactsPhaseTag, IPUWorkflowTag diff --git a/repos/system_upgrade/common/actors/cloud/checkrhui/libraries/checkrhui.py b/repos/system_upgrade/common/actors/cloud/checkrhui/libraries/checkrhui.py index e2047f4a2e..6b8a228c3a 100644 --- a/repos/system_upgrade/common/actors/cloud/checkrhui/libraries/checkrhui.py +++ b/repos/system_upgrade/common/actors/cloud/checkrhui/libraries/checkrhui.py @@ -2,11 +2,8 @@ import os from collections import namedtuple -from leapp import reporting -from leapp.exceptions import StopActorExecution, StopActorExecutionError -from leapp.libraries.common import rhsm, rhui -from leapp.libraries.common.config import version import leapp.configs.common.rhui as rhui_config_lib +from leapp import reporting from leapp.configs.common.rhui import ( # Import all config fields so we are not using their name attributes directly RhuiCloudProvider, RhuiCloudVariant, @@ -14,8 +11,11 @@ RhuiTargetPkgs, RhuiTargetRepositoriesToUse, RhuiUpgradeFiles, - RhuiUseConfig, + RhuiUseConfig ) +from leapp.exceptions import StopActorExecution, StopActorExecutionError +from leapp.libraries.common import rhsm, rhui +from leapp.libraries.common.config import version from leapp.libraries.stdlib import api from leapp.models import ( CopyFile, @@ -31,7 +31,6 @@ TargetUserSpacePreupgradeTasks ) - MatchingSetup = namedtuple('MatchingSetup', ['family', 'description']) diff --git a/repos/system_upgrade/common/actors/cloud/checkrhui/tests/component_test_checkrhui.py b/repos/system_upgrade/common/actors/cloud/checkrhui/tests/component_test_checkrhui.py index 6998f710f1..b4b249db72 100644 --- a/repos/system_upgrade/common/actors/cloud/checkrhui/tests/component_test_checkrhui.py +++ b/repos/system_upgrade/common/actors/cloud/checkrhui/tests/component_test_checkrhui.py @@ -10,9 +10,9 @@ from leapp.libraries.common import rhsm, rhui from leapp.libraries.common.rhui import mk_rhui_setup, RHUIFamily from leapp.libraries.common.testutils import ( + _make_default_config, create_report_mocked, CurrentActorMocked, - _make_default_config, produce_mocked ) from leapp.libraries.stdlib import api diff --git a/repos/system_upgrade/common/configs/rhui.py b/repos/system_upgrade/common/configs/rhui.py index 1eb00e5dbe..42fae74c1a 100644 --- a/repos/system_upgrade/common/configs/rhui.py +++ b/repos/system_upgrade/common/configs/rhui.py @@ -9,7 +9,6 @@ from leapp.actors.config import Config from leapp.models import fields - RHUI_CONFIG_SECTION = 'rhui' From 55bdb5c3c98b221483d67e5133cbacfe5acca001 Mon Sep 17 00:00:00 2001 From: Michal Hecko Date: Sun, 20 Oct 2024 16:08:49 +0200 Subject: [PATCH 21/28] userspacegen(rhui): remove repofiles only if now owned by an RPM --- .../libraries/userspacegen.py | 29 ++++++++++++++----- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/repos/system_upgrade/common/actors/targetuserspacecreator/libraries/userspacegen.py b/repos/system_upgrade/common/actors/targetuserspacecreator/libraries/userspacegen.py index cd2d7d6efd..2155e01068 100644 --- a/repos/system_upgrade/common/actors/targetuserspacecreator/libraries/userspacegen.py +++ b/repos/system_upgrade/common/actors/targetuserspacecreator/libraries/userspacegen.py @@ -1110,6 +1110,26 @@ def _get_target_userspace(): return constants.TARGET_USERSPACE.format(get_target_major_version()) +def _remove_injected_repofiles_from_our_rhui_packages(target_userspace_ctx, rhui_setup_info): + target_userspace_path = _get_target_userspace() + for copy in rhui_setup_info.preinstall_tasks.files_to_copy_into_overlay: + dst_in_container = get_copy_location_from_copy_in_task(target_userspace_path, copy) + dst_in_container = dst_in_container.strip('/') + dst_in_host = os.path.join(target_userspace_path, dst_in_container) + + if os.path.isfile(dst_in_host) and dst_in_host.endswith('.repo'): + # The repofile might have been replaced by a new one provided by the RHUI client if names collide + # Performance: Do the query here and not earlier, because we would be running rpm needlessly + try: + target_userspace_ctx.call(['rpm', '-q', '--whatprovides', dst_in_container]) + api.current_logger().debug('Repofile {0} kept as it is owned by some RPM.'.format(dst_in_host)) + except CalledProcessError: + # rpm exists with 1 if the file is not owned by any RPM. We might be catching all kinds of other + # problems here, but still better than always removing repofiles. + api.current_logger().debug('Removing repofile - not owned by any RPM: {0}'.format(dst_in_host)) + os.remove(dst_in_host) + + def _create_target_userspace(context, indata, packages, files, target_repoids): """Create the target userspace.""" target_path = _get_target_userspace() @@ -1129,14 +1149,7 @@ def _create_target_userspace(context, indata, packages, files, target_repoids): ) setup_info = indata.rhui_info.target_client_setup_info if not setup_info.bootstrap_target_client: - target_userspace_path = _get_target_userspace() - for copy in setup_info.preinstall_tasks.files_to_copy_into_overlay: - dst_in_container = get_copy_location_from_copy_in_task(target_userspace_path, copy) - dst_in_container = dst_in_container.strip('/') - dst_in_host = os.path.join(target_userspace_path, dst_in_container) - if os.path.isfile(dst_in_host) and dst_in_host.endswith('.repo'): - api.current_logger().debug('Removing repofile: {0}'.format(dst_in_host)) - os.remove(dst_in_host) + _remove_injected_repofiles_from_our_rhui_packages(context, setup_info) # and do not forget to set the rhsm into the container mode again with mounting.NspawnActions(_get_target_userspace()) as target_context: From 61ae067640e89e4a54723de70b33ed6202124262 Mon Sep 17 00:00:00 2001 From: Michal Hecko Date: Sun, 20 Oct 2024 16:29:37 +0200 Subject: [PATCH 22/28] check_rhui(tests): add config tests --- .../tests/component_test_checkrhui.py | 93 ++++++++++++++++++- 1 file changed, 91 insertions(+), 2 deletions(-) diff --git a/repos/system_upgrade/common/actors/cloud/checkrhui/tests/component_test_checkrhui.py b/repos/system_upgrade/common/actors/cloud/checkrhui/tests/component_test_checkrhui.py index b4b249db72..b6f2038f74 100644 --- a/repos/system_upgrade/common/actors/cloud/checkrhui/tests/component_test_checkrhui.py +++ b/repos/system_upgrade/common/actors/cloud/checkrhui/tests/component_test_checkrhui.py @@ -1,10 +1,18 @@ -from collections import namedtuple +from collections import defaultdict, namedtuple from enum import Enum import pytest from leapp import reporting -from leapp.configs.common.rhui import all_rhui_cfg +from leapp.configs.common.rhui import ( + all_rhui_cfg, + RhuiTargetPkgs, + RhuiCloudProvider, + RhuiSourcePkgs, + RhuiTargetRepositoriesToUse, + RhuiUpgradeFiles, + RhuiUseConfig, +) from leapp.exceptions import StopActorExecutionError from leapp.libraries.actor import checkrhui as checkrhui_lib from leapp.libraries.common import rhsm, rhui @@ -21,6 +29,7 @@ RHUIInfo, RPM, RpmTransactionTasks, + TargetRepositories, TargetRHUIPostInstallTasks, TargetRHUIPreInstallTasks, TargetRHUISetupInfo, @@ -379,3 +388,83 @@ def test_select_chronologically_closest(monkeypatch, setups, desired_minor, expe setup = setups[0] assert setup == expected_setup + + +def test_config_overwrites_everything(monkeypatch): + rhui_config = { + RhuiUseConfig.name: True, + RhuiSourcePkgs.name: ['client_source'], + RhuiTargetPkgs.name: ['client_target'], + RhuiCloudProvider.name: {'aws'}, + RhuiUpgradeFiles.name: { + '/root/file.repo': '/etc/yum.repos.d/' + }, + RhuiTargetRepositoriesToUse.name: [ + 'repoid_to_use' + ] + } + all_config = {'rhui': rhui_config} + + actor = CurrentActorMocked(config=all_config) + monkeypatch.setattr(api, 'current_actor', actor) + + function_calls = defaultdict(int) + def mk_function_probe(fn_name): + def probe(*args, **kwargs): + function_calls[fn_name] += 1 + return probe + + monkeypatch.setattr(checkrhui_lib, + 'emit_rhui_setup_tasks_based_on_config', + mk_function_probe('emit_rhui_setup_tasks_based_on_config')) + monkeypatch.setattr(checkrhui_lib, + 'stop_with_err_if_config_invalid', + mk_function_probe('stop_with_err_if_config_invalid')) + monkeypatch.setattr(checkrhui_lib, + 'produce_rpms_to_install_into_target', + mk_function_probe('produce_rpms_to_install_into_target')) + monkeypatch.setattr(checkrhui_lib, + 'request_configured_repos_to_be_enabled', + mk_function_probe('request_configured_repos_to_be_enabled')) + + checkrhui_lib.process() + + expected_function_calls = { + 'emit_rhui_setup_tasks_based_on_config': 1, + 'stop_with_err_if_config_invalid': 1, + 'produce_rpms_to_install_into_target': 1, + 'request_configured_repos_to_be_enabled': 1, + } + + assert function_calls == expected_function_calls + + +def test_request_configured_repos_to_be_enabled(monkeypatch): + monkeypatch.setattr(api, 'produce', produce_mocked()) + + rhui_config = { + RhuiUseConfig.name: True, + RhuiSourcePkgs.name: ['client_source'], + RhuiTargetPkgs.name: ['client_target'], + RhuiCloudProvider.name: {'aws'}, + RhuiUpgradeFiles.name: { + '/root/file.repo': '/etc/yum.repos.d/' + }, + RhuiTargetRepositoriesToUse.name: [ + 'repoid1', + 'repoid2', + 'repoid3', + ] + } + + checkrhui_lib.request_configured_repos_to_be_enabled(rhui_config) + + assert api.produce.called + assert len(api.produce.model_instances) == 1 + + target_repos = api.produce.model_instances[0] + assert isinstance(target_repos, TargetRepositories) + assert not target_repos.rhel_repos + + custom_repoids = sorted(custom_repo_model.repoid for custom_repo_model in target_repos.custom_repos) + assert custom_repoids == ['repoid1', 'repoid2', 'repoid3'] \ No newline at end of file From f82750e54678a3786c675f19f9d3d53d39c80652 Mon Sep 17 00:00:00 2001 From: Michal Hecko Date: Sun, 20 Oct 2024 16:30:02 +0200 Subject: [PATCH 23/28] check_rhui: error out if config contains nonexisting files --- .../actors/cloud/checkrhui/libraries/checkrhui.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/repos/system_upgrade/common/actors/cloud/checkrhui/libraries/checkrhui.py b/repos/system_upgrade/common/actors/cloud/checkrhui/libraries/checkrhui.py index 6b8a228c3a..8de2b5c8b0 100644 --- a/repos/system_upgrade/common/actors/cloud/checkrhui/libraries/checkrhui.py +++ b/repos/system_upgrade/common/actors/cloud/checkrhui/libraries/checkrhui.py @@ -330,6 +330,21 @@ def inform_about_upgrade_with_rhui_without_no_rhsm(): def emit_rhui_setup_tasks_based_on_config(rhui_config_dict): config_upgrade_files = rhui_config_dict[RhuiUpgradeFiles.name] + + nonexisting_files_to_copy = [] + for source_path in config_upgrade_files: + if not os.path.exists(source_path): + nonexisting_files_to_copy.append(source_path) + + if nonexisting_files_to_copy: + details_lines = ['The following files were not found:'] + # Use .format and put backticks around paths so that weird unicode spaces will be easily seen + details_lines.extend(' - `{0}`'.format(path) for path in nonexisting_files_to_copy) + details = '\n'.join(details_lines) + + reason = 'RHUI config lists nonexisting files in its `{0}` field.'.format(RhuiUpgradeFiles.name) + raise StopActorExecution(reason, details={'details': details}) + files_to_copy_into_overlay = [CopyFile(src=key, dst=value) for key, value in config_upgrade_files.items()] preinstall_tasks = TargetRHUIPreInstallTasks(files_to_copy_into_overlay=files_to_copy_into_overlay) From 50879db99a7787df2eeae468d2843db5e2743b9d Mon Sep 17 00:00:00 2001 From: Michal Hecko Date: Sun, 20 Oct 2024 16:52:08 +0200 Subject: [PATCH 24/28] checkrhui: use config.src_client --- .../actors/cloud/checkrhui/libraries/checkrhui.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/repos/system_upgrade/common/actors/cloud/checkrhui/libraries/checkrhui.py b/repos/system_upgrade/common/actors/cloud/checkrhui/libraries/checkrhui.py index 8de2b5c8b0..64e36e083c 100644 --- a/repos/system_upgrade/common/actors/cloud/checkrhui/libraries/checkrhui.py +++ b/repos/system_upgrade/common/actors/cloud/checkrhui/libraries/checkrhui.py @@ -13,7 +13,7 @@ RhuiUpgradeFiles, RhuiUseConfig ) -from leapp.exceptions import StopActorExecution, StopActorExecutionError +from leapp.exceptions import StopActorExecutionError from leapp.libraries.common import rhsm, rhui from leapp.libraries.common.config import version from leapp.libraries.stdlib import api @@ -343,7 +343,7 @@ def emit_rhui_setup_tasks_based_on_config(rhui_config_dict): details = '\n'.join(details_lines) reason = 'RHUI config lists nonexisting files in its `{0}` field.'.format(RhuiUpgradeFiles.name) - raise StopActorExecution(reason, details={'details': details}) + raise StopActorExecutionError(reason, details={'details': details}) files_to_copy_into_overlay = [CopyFile(src=key, dst=value) for key, value in config_upgrade_files.items()] preinstall_tasks = TargetRHUIPreInstallTasks(files_to_copy_into_overlay=files_to_copy_into_overlay) @@ -357,7 +357,7 @@ def emit_rhui_setup_tasks_based_on_config(rhui_config_dict): rhui_info = RHUIInfo( provider=rhui_config_dict[RhuiCloudProvider.name], variant=rhui_config_dict[RhuiCloudVariant.name], - src_client_pkg_names=list(), + src_client_pkg_names=rhui_config_dict[RhuiSourcePkgs.name], target_client_pkg_names=rhui_config_dict[RhuiTargetPkgs.name], target_client_setup_info=target_client_setup_info ) @@ -372,7 +372,7 @@ def request_configured_repos_to_be_enabled(rhui_config): api.produce(target_repos) -def stop_with_err_if_config_invalid(config): +def stop_with_err_if_config_missing_fields(config): required_fields = [ RhuiTargetRepositoriesToUse, RhuiCloudProvider, @@ -388,8 +388,8 @@ def stop_with_err_if_config_invalid(config): missing_fields_str = ', '.join(field_names) details = 'The following required RHUI config fields are missing or they are set to an empty value: {}' details = details.format(missing_fields_str) - raise StopActorExecution('Provided RHUI config is missing values for required fields.', - details={'details': details}) + raise StopActorExecutionError('Provided RHUI config is missing values for required fields.', + details={'details': details}) def process(): @@ -397,8 +397,8 @@ def process(): if rhui_config[RhuiUseConfig.name]: api.current_logger().info('Skipping RHUI upgrade auto-configuration - using provided config instead.') + stop_with_err_if_config_missing_fields(rhui_config) emit_rhui_setup_tasks_based_on_config(rhui_config) - stop_with_err_if_config_invalid(rhui_config) src_clients = set(rhui_config[RhuiSourcePkgs.name]) target_clients = set(rhui_config[RhuiTargetPkgs.name]) From aea50bd15319e10042df41d1cf1dc9fcab69cff8 Mon Sep 17 00:00:00 2001 From: Michal Hecko Date: Sun, 20 Oct 2024 16:53:01 +0200 Subject: [PATCH 25/28] check_rhui(tests): check invalid files in the config trigger error --- .../tests/component_test_checkrhui.py | 69 +++++++++++++++++-- 1 file changed, 62 insertions(+), 7 deletions(-) diff --git a/repos/system_upgrade/common/actors/cloud/checkrhui/tests/component_test_checkrhui.py b/repos/system_upgrade/common/actors/cloud/checkrhui/tests/component_test_checkrhui.py index b6f2038f74..5cfed36624 100644 --- a/repos/system_upgrade/common/actors/cloud/checkrhui/tests/component_test_checkrhui.py +++ b/repos/system_upgrade/common/actors/cloud/checkrhui/tests/component_test_checkrhui.py @@ -1,5 +1,7 @@ -from collections import defaultdict, namedtuple +from collections import defaultdict +import itertools from enum import Enum +import os import pytest @@ -8,6 +10,7 @@ all_rhui_cfg, RhuiTargetPkgs, RhuiCloudProvider, + RhuiCloudVariant, RhuiSourcePkgs, RhuiTargetRepositoriesToUse, RhuiUpgradeFiles, @@ -395,7 +398,7 @@ def test_config_overwrites_everything(monkeypatch): RhuiUseConfig.name: True, RhuiSourcePkgs.name: ['client_source'], RhuiTargetPkgs.name: ['client_target'], - RhuiCloudProvider.name: {'aws'}, + RhuiCloudProvider.name: 'aws', RhuiUpgradeFiles.name: { '/root/file.repo': '/etc/yum.repos.d/' }, @@ -418,8 +421,8 @@ def probe(*args, **kwargs): 'emit_rhui_setup_tasks_based_on_config', mk_function_probe('emit_rhui_setup_tasks_based_on_config')) monkeypatch.setattr(checkrhui_lib, - 'stop_with_err_if_config_invalid', - mk_function_probe('stop_with_err_if_config_invalid')) + 'stop_with_err_if_config_missing_fields', + mk_function_probe('stop_with_err_if_config_missing_fields')) monkeypatch.setattr(checkrhui_lib, 'produce_rpms_to_install_into_target', mk_function_probe('produce_rpms_to_install_into_target')) @@ -431,7 +434,7 @@ def probe(*args, **kwargs): expected_function_calls = { 'emit_rhui_setup_tasks_based_on_config': 1, - 'stop_with_err_if_config_invalid': 1, + 'stop_with_err_if_config_missing_fields': 1, 'produce_rpms_to_install_into_target': 1, 'request_configured_repos_to_be_enabled': 1, } @@ -446,7 +449,7 @@ def test_request_configured_repos_to_be_enabled(monkeypatch): RhuiUseConfig.name: True, RhuiSourcePkgs.name: ['client_source'], RhuiTargetPkgs.name: ['client_target'], - RhuiCloudProvider.name: {'aws'}, + RhuiCloudProvider.name: 'aws', RhuiUpgradeFiles.name: { '/root/file.repo': '/etc/yum.repos.d/' }, @@ -467,4 +470,56 @@ def test_request_configured_repos_to_be_enabled(monkeypatch): assert not target_repos.rhel_repos custom_repoids = sorted(custom_repo_model.repoid for custom_repo_model in target_repos.custom_repos) - assert custom_repoids == ['repoid1', 'repoid2', 'repoid3'] \ No newline at end of file + assert custom_repoids == ['repoid1', 'repoid2', 'repoid3'] + + +@pytest.mark.parametrize( + ('upgrade_files', 'existing_files'), + ( + (['/root/a', '/root/b'], ['/root/a', '/root/b']), + (['/root/a', '/root/b'], ['/root/b']), + (['/root/a', '/root/b'], []), + ) +) +def test_missing_files_in_config(monkeypatch, upgrade_files, existing_files): + upgrade_files_map = dict((source_path, '/tmp/dummy') for source_path in upgrade_files) + + rhui_config = { + RhuiUseConfig.name: True, + RhuiSourcePkgs.name: ['client_source'], + RhuiTargetPkgs.name: ['client_target'], + RhuiCloudProvider.name: 'aws', + RhuiCloudVariant.name: 'ordinary', + RhuiUpgradeFiles.name: upgrade_files_map, + RhuiTargetRepositoriesToUse.name: [ + 'repoid_to_use' + ] + } + + monkeypatch.setattr(os.path, 'exists', lambda path: path in existing_files) + monkeypatch.setattr(api, 'produce', produce_mocked()) + + should_error = (len(upgrade_files) != len(existing_files)) + if should_error: + with pytest.raises(StopActorExecutionError): + checkrhui_lib.emit_rhui_setup_tasks_based_on_config(rhui_config) + else: + checkrhui_lib.emit_rhui_setup_tasks_based_on_config(rhui_config) + assert api.produce.called + assert len(api.produce.model_instances) == 1 + + rhui_info = api.produce.model_instances[0] + assert isinstance(rhui_info, RHUIInfo) + assert rhui_info.provider == 'aws' + assert rhui_info.variant == 'ordinary' + assert rhui_info.src_client_pkg_names == ['client_source'] + assert rhui_info.target_client_pkg_names == ['client_target'] + + setup_info = rhui_info.target_client_setup_info + assert not setup_info.bootstrap_target_client + + _copies_to_perform = setup_info.preinstall_tasks.files_to_copy_into_overlay + copies_to_perform = sorted((copy.src, copy.dst) for copy in _copies_to_perform) + expected_copies = sorted(zip(upgrade_files, itertools.repeat('/tmp/dummy'))) + + assert copies_to_perform == expected_copies From 892c35e7bc8df162c9a8d6677b0f6740e4174ffd Mon Sep 17 00:00:00 2001 From: Michal Hecko Date: Sun, 20 Oct 2024 18:15:45 +0200 Subject: [PATCH 26/28] fixup! userspacegen(rhui): remove repofiles only if now owned by an RPM --- .../actors/targetuserspacecreator/libraries/userspacegen.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/repos/system_upgrade/common/actors/targetuserspacecreator/libraries/userspacegen.py b/repos/system_upgrade/common/actors/targetuserspacecreator/libraries/userspacegen.py index 2155e01068..8ddb014e87 100644 --- a/repos/system_upgrade/common/actors/targetuserspacecreator/libraries/userspacegen.py +++ b/repos/system_upgrade/common/actors/targetuserspacecreator/libraries/userspacegen.py @@ -1121,7 +1121,8 @@ def _remove_injected_repofiles_from_our_rhui_packages(target_userspace_ctx, rhui # The repofile might have been replaced by a new one provided by the RHUI client if names collide # Performance: Do the query here and not earlier, because we would be running rpm needlessly try: - target_userspace_ctx.call(['rpm', '-q', '--whatprovides', dst_in_container]) + path_with_root = '/' + dst_in_container + target_userspace_ctx.call(['rpm', '-q', '--whatprovides', path_with_root]) api.current_logger().debug('Repofile {0} kept as it is owned by some RPM.'.format(dst_in_host)) except CalledProcessError: # rpm exists with 1 if the file is not owned by any RPM. We might be catching all kinds of other From ddd773a99a65185d6f7b89a979f64ef306c389d0 Mon Sep 17 00:00:00 2001 From: Michal Hecko Date: Sun, 20 Oct 2024 18:18:44 +0200 Subject: [PATCH 27/28] fixup! check_rhui(tests): add config tests --- .../actors/cloud/checkrhui/tests/component_test_checkrhui.py | 1 + 1 file changed, 1 insertion(+) diff --git a/repos/system_upgrade/common/actors/cloud/checkrhui/tests/component_test_checkrhui.py b/repos/system_upgrade/common/actors/cloud/checkrhui/tests/component_test_checkrhui.py index 5cfed36624..11e92ae8e6 100644 --- a/repos/system_upgrade/common/actors/cloud/checkrhui/tests/component_test_checkrhui.py +++ b/repos/system_upgrade/common/actors/cloud/checkrhui/tests/component_test_checkrhui.py @@ -412,6 +412,7 @@ def test_config_overwrites_everything(monkeypatch): monkeypatch.setattr(api, 'current_actor', actor) function_calls = defaultdict(int) + def mk_function_probe(fn_name): def probe(*args, **kwargs): function_calls[fn_name] += 1 From 356a5b3715cd933721a99c8280fe1a04b0dccf7f Mon Sep 17 00:00:00 2001 From: Michal Hecko Date: Sun, 20 Oct 2024 18:24:30 +0200 Subject: [PATCH 28/28] check_rhui(tests): resort imports --- .../cloud/checkrhui/tests/component_test_checkrhui.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/repos/system_upgrade/common/actors/cloud/checkrhui/tests/component_test_checkrhui.py b/repos/system_upgrade/common/actors/cloud/checkrhui/tests/component_test_checkrhui.py index 11e92ae8e6..3ac9c1b883 100644 --- a/repos/system_upgrade/common/actors/cloud/checkrhui/tests/component_test_checkrhui.py +++ b/repos/system_upgrade/common/actors/cloud/checkrhui/tests/component_test_checkrhui.py @@ -1,20 +1,20 @@ -from collections import defaultdict import itertools -from enum import Enum import os +from collections import defaultdict +from enum import Enum import pytest from leapp import reporting from leapp.configs.common.rhui import ( all_rhui_cfg, - RhuiTargetPkgs, RhuiCloudProvider, RhuiCloudVariant, RhuiSourcePkgs, + RhuiTargetPkgs, RhuiTargetRepositoriesToUse, RhuiUpgradeFiles, - RhuiUseConfig, + RhuiUseConfig ) from leapp.exceptions import StopActorExecutionError from leapp.libraries.actor import checkrhui as checkrhui_lib