Add RHUI and RPM Transaction configuration

commands/preupgrade/__init__.py

@@ -2,6 +2,7 @@
 import sys
 import uuid
 
+from leapp.actors import config as actor_config
 from leapp.cli.commands import command_utils
 from leapp.cli.commands.config import get_config
 from leapp.cli.commands.upgrade import breadcrumbs, util
@@ -60,6 +61,13 @@ def preupgrade(args, breadcrumbs):
         raise CommandError(exc.message)
 
     workflow = repositories.lookup_workflow('IPUWorkflow')()
+
+    # Read the Actor Config and validate it against the schemas saved in the configuration.
+    actor_config_schemas = tuple(actor.config_schemas for actor in repositories.actors)
+    actor_config_schemas = actor_config.normalize_schemas(actor_config_schemas)
+    actor_config_path = cfg.get('actor_config', 'path')
+    actor_config.load(actor_config_path, actor_config_schemas)
+
     util.warn_if_unsupported(configuration)
     util.process_whitelist_experimental(repositories, workflow, configuration, logger)
     with beautify_actor_exception():

commands/upgrade/__init__.py

@@ -2,6 +2,7 @@
 import sys
 import uuid
 
+from leapp.actors import config as actor_config
 from leapp.cli.commands import command_utils
 from leapp.cli.commands.config import get_config
 from leapp.cli.commands.upgrade import breadcrumbs, util
@@ -90,6 +91,17 @@ def upgrade(args, breadcrumbs):
     except LeappError as exc:
         raise CommandError(exc.message)
     workflow = repositories.lookup_workflow('IPUWorkflow')(auto_reboot=args.reboot)
+
+    # Read the Actor Config and validate it against the schemas saved in the
+    # configuration.
+    actor_config_schemas = tuple(actor.config_schemas for actor in repositories.actors)
+    actor_config_schemas = actor_config.normalize_schemas(actor_config_schemas)
+    actor_config_path = cfg.get('actor_config', 'path')
+    # Note: actor_config.load() stores the loaded actor config into a global
+    # variable which can then be accessed by functions in that file.  Is this
+    # the right way to store that information?
+    actor_config.load(actor_config_path, actor_config_schemas)
+
     util.process_whitelist_experimental(repositories, workflow, configuration, logger)
     util.warn_if_unsupported(configuration)
     with beautify_actor_exception():

etc/leapp/actor_conf.d/rhui.yml

@@ -0,0 +1,12 @@
+rhui:
+  use_config : True
+  source_clients: [ rh-amazon-rhui-client ]
+  target_clients: [ rh-amazon-rhui-client ]
+  cloud_provider: aws
+  upgrade_files:
+    /usr/share/leapp-repository/repositories/system_upgrade/common/files/rhui/leapp-rhui-aws/rhui-client-config-server-9.crt : /etc/pki/rhui/product/
+    /usr/share/leapp-repository/repositories/system_upgrade/common/files/rhui/leapp-rhui-aws/rhui-client-config-server-9.key : /etc/pki/rhui/private/
+    /usr/share/leapp-repository/repositories/system_upgrade/common/files/rhui/leapp-rhui-aws/leapp-aws.repo : /etc/yum.repos.d/
+  enabled_target_repositories:
+    - rhel-9-appstream-rhui-rpms
+    - rhel-9-baseos-rhui-rpms

packaging/leapp-repository.spec

@@ -244,6 +244,10 @@ install -m 0755 -d %{buildroot}%{_sysconfdir}/leapp/files/
 install -m 0644 etc/leapp/transaction/* %{buildroot}%{_sysconfdir}/leapp/transaction
 install -m 0644 etc/leapp/files/* %{buildroot}%{_sysconfdir}/leapp/files
 
+# Actor configuration
+install -m 0755 -d %{buildroot}%{_sysconfdir}/leapp/actor_conf.d/
+install -m 0644 etc/leapp/actor_conf.d/* %{buildroot}%{_sysconfdir}/leapp/actor_conf.d
+
 # install CLI commands for the leapp utility on the expected path
 install -m 0755 -d %{buildroot}%{leapp_python_sitelib}/leapp/cli/
 cp -r commands %{buildroot}%{leapp_python_sitelib}/leapp/cli/
@@ -293,6 +297,7 @@ done;
 %{_sysconfdir}/leapp/transaction/*
 %{repositorydir}/*
 %{leapp_python_sitelib}/leapp/cli/commands/*
+%{_sysconfdir}/leapp/actor_conf.d/*
 
 
 %files -n %{lpr_name}-deps

repos/system_upgrade/common/actors/cloud/checkrhui/actor.py

@@ -1,4 +1,5 @@
 from leapp.actors import Actor
+from leapp.configs.common.rhui import all_rhui_cfg
 from leapp.libraries.actor import checkrhui as checkrhui_lib
 from leapp.models import (
     CopyFile,
@@ -8,7 +9,8 @@
     RequiredTargetUserspacePackages,
     RHUIInfo,
     RpmTransactionTasks,
-    TargetUserSpacePreupgradeTasks
+    TargetRepositories,
+    TargetUserSpacePreupgradeTasks,
 )
 from leapp.reporting import Report
 from leapp.tags import FactsPhaseTag, IPUWorkflowTag
@@ -21,13 +23,15 @@ class CheckRHUI(Actor):
     """
 
     name = 'checkrhui'
+    config_schemas = all_rhui_cfg
     consumes = (InstalledRPM,)
     produces = (
         KernelCmdlineArg,
         RHUIInfo,
         RequiredTargetUserspacePackages,
         Report, DNFPluginTask,
         RpmTransactionTasks,
+        TargetRepositories,
         TargetUserSpacePreupgradeTasks,
         CopyFile,
     )

repos/system_upgrade/common/actors/cloud/checkrhui/libraries/checkrhui.py

@@ -6,19 +6,32 @@
 from leapp.exceptions import StopActorExecutionError
 from leapp.libraries.common import rhsm, rhui
 from leapp.libraries.common.config import version
+import leapp.configs.common.rhui as rhui_config_lib
+from leapp.configs.common.rhui import (  # Import all config fields so we are not using their name attributes directly
+    RhuiEnabledTargetRepositories,
+    RhuiCloudProvider,
+    RhuiCloudVariant,
+    RhuiSourcePkgs,
+    RhuiTargetPkgs,
+    RhuiUpgradeFiles,
+    RhuiUseConfig,
+)
 from leapp.libraries.stdlib import api
 from leapp.models import (
     CopyFile,
+    CustomTargetRepository,
     DNFPluginTask,
     InstalledRPM,
     RHUIInfo,
     RpmTransactionTasks,
+    TargetRepositories,
     TargetRHUIPostInstallTasks,
     TargetRHUIPreInstallTasks,
     TargetRHUISetupInfo,
     TargetUserSpacePreupgradeTasks
 )
 
+
 MatchingSetup = namedtuple('MatchingSetup', ['family', 'description'])
 
 
@@ -291,11 +304,11 @@ def produce_rhui_info_to_setup_target(rhui_family, source_setup_desc, target_set
     api.produce(rhui_info)
 
 
-def produce_rpms_to_install_into_target(source_setup, target_setup):
-    to_install = sorted(target_setup.clients - source_setup.clients)
-    to_remove = sorted(source_setup.clients - target_setup.clients)
+def produce_rpms_to_install_into_target(source_clients, target_clients):
+    to_install = sorted(target_clients - source_clients)
+    to_remove = sorted(source_clients - target_clients)
 
-    api.produce(TargetUserSpacePreupgradeTasks(install_rpms=sorted(target_setup.clients)))
+    api.produce(TargetUserSpacePreupgradeTasks(install_rpms=sorted(target_clients)))
     if to_install or to_remove:
         api.produce(RpmTransactionTasks(to_install=to_install, to_remove=to_remove))
 
@@ -316,7 +329,49 @@ def inform_about_upgrade_with_rhui_without_no_rhsm():
     return False
 
 
+def emit_rhui_setup_tasks_based_on_config(rhui_config_dict):
+    config_upgrade_files = rhui_config_dict[RhuiUpgradeFiles.name]
+    files_to_copy_into_overlay = [CopyFile(src=key, dst=value) for key, value in config_upgrade_files.items()]
+    preinstall_tasks = TargetRHUIPreInstallTasks(files_to_copy_into_overlay=files_to_copy_into_overlay)
+
+    target_client_setup_info = TargetRHUISetupInfo(
+        preinstall_tasks=preinstall_tasks,
+        postinstall_tasks=TargetRHUIPostInstallTasks(),
+        bootstrap_target_client=False,  # We don't need to install the client into overlay - user provided all files
+    )
+
+    rhui_info = RHUIInfo(
+        provider=rhui_config_dict[RhuiCloudProvider.name],
+        variant=rhui_config_dict[RhuiCloudVariant.name],
+        src_client_pkg_names=list(),
+        target_client_pkg_names=rhui_config_dict[RhuiTargetPkgs.name],
+        target_client_setup_info=target_client_setup_info
+    )
+    api.produce(rhui_info)
+
+
+def request_configured_repos_to_be_enabled(rhui_config):
+    config_repos_to_enable = rhui_config[RhuiEnabledTargetRepositories.name]
+    custom_repos = [CustomTargetRepository(repoid=repoid) for repoid in config_repos_to_enable]
+    if custom_repos:
+        target_repos = TargetRepositories(custom_repos=custom_repos, rhel_repos=[])
+        api.produce(target_repos)
+
+
 def process():
+    rhui_config = api.current_actor().config[rhui_config_lib.RHUI_CONFIG_SECTION]
+
+    if rhui_config[RhuiUseConfig.name]:
+        api.current_logger().info('Skipping RHUI upgrade auto-configuration - using provided config instead.')
+        emit_rhui_setup_tasks_based_on_config(rhui_config)
+
+        src_clients = set(rhui_config[RhuiSourcePkgs.name])
+        target_clients = set(rhui_config[RhuiTargetPkgs.name])
+        produce_rpms_to_install_into_target(src_clients, target_clients)
+
+        request_configured_repos_to_be_enabled(rhui_config)
+        return
+
     installed_rpm = itertools.chain(*[installed_rpm_msg.items for installed_rpm_msg in api.consume(InstalledRPM)])
     installed_pkgs = {rpm.name for rpm in installed_rpm}
 
@@ -342,7 +397,9 @@ def process():
     # Instruction on how to access the target content
     produce_rhui_info_to_setup_target(src_rhui_setup.family, src_rhui_setup.description, target_setup_desc)
 
-    produce_rpms_to_install_into_target(src_rhui_setup.description, target_setup_desc)
+    source_clients = src_rhui_setup.description.clients
+    target_clients = target_setup_desc.clients
+    produce_rpms_to_install_into_target(source_clients, target_clients)
 
     if src_rhui_setup.family.provider == rhui.RHUIProvider.AWS:
         # We have to disable Amazon-id plugin in the initramdisk phase as there is no network

repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/actor.py

@@ -1,10 +1,9 @@
 from leapp.actors import Actor
+from leapp.configs.actor.rpm import Transaction_ToInstall, Transaction_ToKeep, Transaction_ToRemove
 from leapp.libraries.actor.rpmtransactionconfigtaskscollector import load_tasks
 from leapp.models import DistributionSignedRPM, RpmTransactionTasks
 from leapp.tags import FactsPhaseTag, IPUWorkflowTag
 
-CONFIGURATION_BASE_PATH = '/etc/leapp/transaction'
-
 
 class RpmTransactionConfigTasksCollector(Actor):
     """
@@ -13,11 +12,11 @@ class RpmTransactionConfigTasksCollector(Actor):
     After collecting task data from /etc/leapp/transaction directory, a message with relevant data
     will be produced.
     """
-
+    config_schemas = (Transaction_ToInstall, Transaction_ToKeep, Transaction_ToRemove)
     name = 'rpm_transaction_config_tasks_collector'
     consumes = (DistributionSignedRPM,)
     produces = (RpmTransactionTasks,)
     tags = (FactsPhaseTag, IPUWorkflowTag)
 
     def process(self):
-        self.produce(load_tasks(CONFIGURATION_BASE_PATH, self.log))
+        self.produce(load_tasks(self.config, self.log))

repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/configs/rpm.py

@@ -0,0 +1,64 @@
+"""
+Configuration keys for dnf transactions.
+"""
+
+from leapp.actors.config import Config
+from leapp.models import fields
+
+
+# * Nested containers?
+# * Duplication of default value in type_ and Config.  If we eliminate that, we need to extract
+# default from the type_ for the documentation.
+# * We probably want to allow dicts in Config.  But IIRC, dicts were
+# specifically excluded for model fields.  Do we need something that restricts
+# where fields are valid?
+# * Test that type validation is strict.  For instance, giving an integer like 644 to
+# a field.String() is an error.
+class Transaction_ToInstall(Config):
+    section = "transaction"
+    name = "to_install"
+    type_ = fields.List(fields.String(), default=[])
+    default = []
+    description = """
+        List of packages to be added to the upgrade transaction.
+        Signed packages which are already installed will be skipped.
+    """
+
+
+class Transaction_ToKeep(Config):
+    section = "transaction"
+    name = "to_keep"
+    type_ = fields.List(fields.String(), default=[
+        "leapp",
+        "python2-leapp",
+        "python3-leapp",
+        "leapp-repository",
+        "snactor",
+    ])
+    default = [
+        "leapp",
+        "python2-leapp",
+        "python3-leapp",
+        "leapp-repository",
+        "snactor",
+    ]
+    description = """
+        List of packages to be kept in the upgrade transaction. The default is
+        leapp, python2-leapp, python3-leapp, leapp-repository, snactor. If you
+        override this, remember to include the default values if applicable.
+    """
+
+
+class Transaction_ToRemove(Config):
+    section = "transaction"
+    name = "to_remove"
+    type_ = fields.List(fields.String(), default=[
+        "initial-setup",
+    ])
+    default = ["initial-setup"]
+    description = """
+        List of packages to be removed from the upgrade transaction. The default
+        is initial-setup which should be removed to avoid it asking for EULA
+        acceptance during upgrade. If you override this, remember to include the
+        default values if applicable.
+    """

repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/libraries/rpmtransactionconfigtaskscollector.py

@@ -3,6 +3,10 @@
 from leapp.libraries.stdlib import api
 from leapp.models import DistributionSignedRPM, RpmTransactionTasks
 
+# Deprecated.  This is the old, pre-actor config method of customizing which
+# packages to keep, remove, and install.
+_CONFIGURATION_BASE_PATH = '/etc/leapp/transaction'
+
 
 def load_tasks_file(path, logger):
     # Loads the given file and converts it to a deduplicated list of strings that are stripped
@@ -18,21 +22,35 @@ def load_tasks_file(path, logger):
     return []
 
 
-def load_tasks(base_dir, logger):
+def load_tasks(config, logger, base_dir=_CONFIGURATION_BASE_PATH):
     # Loads configuration files to_install, to_keep, and to_remove from the given base directory
     rpms = next(api.consume(DistributionSignedRPM))
     rpm_names = [rpm.name for rpm in rpms.items]
-    to_install = load_tasks_file(os.path.join(base_dir, 'to_install'), logger)
-    # we do not want to put into rpm transaction what is already installed (it will go to "to_upgrade" bucket)
+    to_keep = frozenset(config['transaction']['to_keep'])
+    to_keep = to_keep.union(load_tasks_file(
+        os.path.join(base_dir, 'to_keep'), logger))
+    to_keep = list(to_keep)
+
+    to_remove = frozenset(config['transaction']['to_remove'])
+    to_remove = to_remove.union(load_tasks_file(
+        os.path.join(base_dir, 'to_remove'), logger))
+    to_remove = list(to_remove)
+
+    to_install = frozenset(config['transaction']['to_install'])
+    to_install = to_install.union(load_tasks_file(
+        os.path.join(base_dir, 'to_install'), logger))
+    # we do not want to put into rpm transaction what is already installed
+    # (it will go to "to_upgrade" bucket)
     to_install_filtered = [pkg for pkg in to_install if pkg not in rpm_names]
 
-    filtered = set(to_install) - set(to_install_filtered)
+    filtered = to_install.difference(to_install_filtered)
     if filtered:
         api.current_logger().debug(
-            'The following packages from "to_install" file will be ignored as they are already installed:'
-            '\n- ' + '\n- '.join(filtered))
+            'The following packages from "to_install" file will be ignored as'
+            ' they are already installed:\n- ' + '\n- '.join(filtered))
 
     return RpmTransactionTasks(
         to_install=to_install_filtered,
-        to_keep=load_tasks_file(os.path.join(base_dir, 'to_keep'), logger),
-        to_remove=load_tasks_file(os.path.join(base_dir, 'to_remove'), logger))
+        to_keep=to_keep,
+        to_remove=to_remove
+    )