Initial work to demonstrate an actor config framework.

* Modify the upgrade workflow to load the actor configuration to be available to the actors
  [_] Note: The same sort of code needs to be added to other workflows that need actor config.  For
      instance, the pre-upgrade workflow.
* Define potential config schemas for RHUI and for listing rpm packages to be treated specially
  during transations.
* Add config schemas to the rpmtransactionconfigtaskscollector as a sample of using it.
* The actor config is supposed to live in the actor:
  We decided that we weren't going to share config schema via python import.  Instead, Actors would
  need to copy the schema into themselves.  That way the framework can check whether the schema for
  two Actors is out of sync and force the user to correct the code before running.
  * However, the framework is currently only reading from the configs dir at the repository level,
    not inside the actor.  Need to remove the copy from the repository level once that is fixed.
* Get the rpm package liste from both legacy location and actor config.

Depends-On: 870

fix

commands/upgrade/__init__.py

@@ -2,6 +2,7 @@
 import sys
 import uuid
 
+from leapp.actor import config as actor_config
 from leapp.cli.commands import command_utils
 from leapp.cli.commands.config import get_config
 from leapp.cli.commands.upgrade import breadcrumbs, util
@@ -90,6 +91,17 @@ def upgrade(args, breadcrumbs):
     except LeappError as exc:
         raise CommandError(exc.message)
     workflow = repositories.lookup_workflow('IPUWorkflow')(auto_reboot=args.reboot)
+
+    # Read the Actor Config and validate it against the schemas saved in the
+    # configuration.
+    actor_config_schemas = tuple(actor.config_schemas for actor in repositories.actors)
+    actor_config_schemas = actor_config.normalize_schemas(actor_config_schemas)
+    actor_config_path = cfg.get('actor_config', 'path')
+    # Note: actor_config.load() stores the loaded actor config into a global
+    # variable which can then be accessed by functions in that file.  Is this
+    # the right way to store that information?
+    actor_config.load(actor_config_path, actor_config_schemas)
+
     util.process_whitelist_experimental(repositories, workflow, configuration, logger)
     util.warn_if_unsupported(configuration)
     with beautify_actor_exception():

repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/actor.py

@@ -1,10 +1,9 @@
 from leapp.actors import Actor
+from leapp.configs.actor.rpm import Transaction_ToInstall, Transaction_ToKeep, Transaction_ToRemove
 from leapp.libraries.actor.rpmtransactionconfigtaskscollector import load_tasks
 from leapp.models import DistributionSignedRPM, RpmTransactionTasks
 from leapp.tags import FactsPhaseTag, IPUWorkflowTag
 
-CONFIGURATION_BASE_PATH = '/etc/leapp/transaction'
-
 
 class RpmTransactionConfigTasksCollector(Actor):
     """
@@ -13,11 +12,11 @@ class RpmTransactionConfigTasksCollector(Actor):
     After collecting task data from /etc/leapp/transaction directory, a message with relevant data
     will be produced.
     """
-
+    config_schemas = (Transaction_ToInstall, Transaction_ToKeep, Transaction_ToRemove)
     name = 'rpm_transaction_config_tasks_collector'
     consumes = (DistributionSignedRPM,)
     produces = (RpmTransactionTasks,)
     tags = (FactsPhaseTag, IPUWorkflowTag)
 
     def process(self):
-        self.produce(load_tasks(CONFIGURATION_BASE_PATH, self.log))
+        self.produce(load_tasks(self.config, self.log))

repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/configs/rpm.py

@@ -0,0 +1,64 @@
+"""
+Configuration keys for dnf transactions.
+"""
+
+from leapp.actors.config import Config
+from leapp.models import fields
+
+
+# * Nested containers?
+# * Duplication of default value in type_ and Config.  If we eliminate that, we need to extract
+# default from the type_ for the documentation.
+# * We probably want to allow dicts in Config.  But IIRC, dicts were
+# specifically excluded for model fields.  Do we need something that restricts
+# where fields are valid?
+# * Test that type validation is strict.  For instance, giving an integer like 644 to
+# a field.String() is an error.
+class Transaction_ToInstall(Config):
+    section = "transaction"
+    name = "to_install"
+    type_ = fields.List(fields.String(), default=[])
+    default = []
+    description = """
+        List of packages to be added to the upgrade transaction.
+        Signed packages which are already installed will be skipped.
+    """
+
+
+class Transaction_ToKeep(Config):
+    section = "transaction"
+    name = "to_keep"
+    type_ = fields.List(fields.String(), default=[
+        "leapp",
+        "python2-leapp",
+        "python3-leapp",
+        "leapp-repository",
+        "snactor",
+    ])
+    default = [
+        "leapp",
+        "python2-leapp",
+        "python3-leapp",
+        "leapp-repository",
+        "snactor",
+    ]
+    description = """
+        List of packages to be kept in the upgrade transaction. The default is
+        leapp, python2-leapp, python3-leapp, leapp-repository, snactor. If you
+        override this, remember to include the default values if applicable.
+    """
+
+
+class Transaction_ToRemove(Config):
+    section = "transaction"
+    name = "to_remove"
+    type_ = fields.List(fields.String(), default=[
+        "initial-setup",
+    ])
+    default = ["initial-setup"]
+    description = """
+        List of packages to be removed from the upgrade transaction. The default
+        is initial-setup which should be removed to avoid it asking for EULA
+        acceptance during upgrade. If you override this, remember to include the
+        default values if applicable.
+    """

repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/libraries/rpmtransactionconfigtaskscollector.py

@@ -4,6 +4,11 @@
 from leapp.models import DistributionSignedRPM, RpmTransactionTasks
 
 
+# Deprecated.  This is the old, pre-actor config method of customizing which
+# packages to keep, remove, and install.
+_CONFIGURATION_BASE_PATH = '/etc/leapp/transaction'
+
+
 def load_tasks_file(path, logger):
     # Loads the given file and converts it to a deduplicated list of strings that are stripped
     if os.path.isfile(path):
@@ -18,21 +23,35 @@ def load_tasks_file(path, logger):
     return []
 
 
-def load_tasks(base_dir, logger):
+def load_tasks(config, logger, base_dir=_CONFIGURATION_BASE_PATH):
     # Loads configuration files to_install, to_keep, and to_remove from the given base directory
     rpms = next(api.consume(DistributionSignedRPM))
     rpm_names = [rpm.name for rpm in rpms.items]
-    to_install = load_tasks_file(os.path.join(base_dir, 'to_install'), logger)
-    # we do not want to put into rpm transaction what is already installed (it will go to "to_upgrade" bucket)
+    to_keep = frozenset(config['transaction']['to_keep'])
+    to_keep = to_keep.union(load_tasks_file(
+        os.path.join(base_dir, 'to_keep'), logger))
+    to_keep = list(to_keep)
+
+    to_remove = frozenset(config['transaction']['to_remove'])
+    to_remove = to_remove.union(load_tasks_file(
+        os.path.join(base_dir, 'to_remove'), logger))
+    to_remove = list(to_remove)
+
+    to_install = frozenset(config['transaction']['to_install'])
+    to_install = to_install.union(load_tasks_file(
+        os.path.join(base_dir, 'to_install'), logger))
+    # we do not want to put into rpm transaction what is already installed
+    # (it will go to "to_upgrade" bucket)
     to_install_filtered = [pkg for pkg in to_install if pkg not in rpm_names]
 
-    filtered = set(to_install) - set(to_install_filtered)
+    filtered = to_install.difference(to_install_filtered)
     if filtered:
         api.current_logger().debug(
-            'The following packages from "to_install" file will be ignored as they are already installed:'
-            '\n- ' + '\n- '.join(filtered))
+            'The following packages from "to_install" file will be ignored as'
+            ' they are already installed:\n- ' + '\n- '.join(filtered))
 
     return RpmTransactionTasks(
         to_install=to_install_filtered,
-        to_keep=load_tasks_file(os.path.join(base_dir, 'to_keep'), logger),
-        to_remove=load_tasks_file(os.path.join(base_dir, 'to_remove'), logger))
+        to_keep=to_keep,
+        to_remove=to_remove
+    )

repos/system_upgrade/common/actors/rpmtransactionconfigtaskscollector/tests/test_load_tasks_rpmtransactionconfigtaskscollector.py

@@ -1,13 +1,67 @@
 import logging
 
+import pytest
+
 from leapp.libraries.actor.rpmtransactionconfigtaskscollector import load_tasks, load_tasks_file
 from leapp.libraries.stdlib import api
 from leapp.models import DistributionSignedRPM, RPM
 
 RH_PACKAGER = 'Red Hat, Inc. <http://bugzilla.redhat.com/bugzilla>'
 
-
-def test_load_tasks(tmpdir, monkeypatch):
+@pytest.mark.parametrize(
+    ('to_install_file',
+     'to_keep_file',
+     'to_remove_file',
+     'to_install_config',
+     'to_keep_config',
+     'to_remove_config',
+     'to_install',
+     'to_keep',
+     'to_remove',
+    ),
+    ('a\n b\n  c \n\n\nc\na\nc\nb',
+     'a\n b\n  c \n\n\nc\na\nc\nb',
+     'a\n b\n  c \n\n\nc\na\nc\nb',
+     [],
+     [],
+     [],
+     frozenset(('a', 'b')),
+     frozenset(('a', 'b', 'c')),
+     frozenset(('a', 'b', 'c')),
+     ),
+    ('a\n b\n  c \n\n\nc\na\nc\nb',
+     'a\n b\n  c \n\n\nc\na\nc\nb',
+     'a\n b\n  c \n\n\nc\na\nc\nb',
+     ['a', 'd'],
+     ['a', 'd'],
+     ['a', 'd'],
+     frozenset(('a', 'b', 'd')),
+     frozenset(('a', 'b', 'c', 'd')),
+     frozenset(('a', 'b', 'c', 'd')),
+     ),
+    ('',
+     '\n',
+     '',
+     ['a', 'b', 'c', 'c', 'a', 'c', 'b'],
+     ['a', 'b', 'c', 'c', 'a', 'c', 'b'],
+     ['a', 'b', 'c', 'c', 'a', 'c', 'b'],
+     frozenset(('a', 'b')),
+     frozenset(('a', 'b', 'c')),
+     frozenset(('a', 'b', 'c')),
+     ),
+)
+def test_load_tasks(to_install_file,
+                    to_keep_file,
+                    to_remove_file,
+                    to_install_config,
+                    to_keep_config,
+                    to_remove_config,
+                    to_install,
+                    to_keep,
+                    to_remove,
+                    tmpdir,
+                    monkeypatch,
+                    ):
 
     def consume_signed_rpms_mocked(*models):
         installed = [
@@ -18,14 +72,25 @@ def consume_signed_rpms_mocked(*models):
 
     monkeypatch.setattr(api, "consume", consume_signed_rpms_mocked)
 
-    tmpdir.join('to_install').write('a\n b\n  c \n\n\nc\na\nc\nb')
-    tmpdir.join('to_keep').write('a\n b\n  c \n\n\nc\na\nc\nb')
-    tmpdir.join('to_remove').write('a\n b\n  c \n\n\nc\na\nc\nb')
-    m = load_tasks(tmpdir.strpath, logging)
+    # Set values in the legacy configuration files
+    tmpdir.join('to_install').write(to_install_file)
+    tmpdir.join('to_keep').write(to_keep_file)
+    tmpdir.join('to_remove').write(to_remove_file)
+
+    # Simulate how the new actor config will come to us
+    config = {
+        'transaction': {
+            'to_install': to_install_config,
+            'to_keep': to_keep_config,
+            'to_remove': to_remove_config,
+        }
+    }
+
+    m = load_tasks(config, logging, base_dir=tmpdir)
     # c is not going to be in "to_install" as it is already installed
-    assert set(m.to_install) == set(['a', 'b'])
-    assert set(m.to_keep) == set(['a', 'b', 'c'])
-    assert set(m.to_remove) == set(['a', 'b', 'c'])
+    assert frozenset(m.to_install) == to_install
+    assert frozenset(m.to_keep) == to_keep
+    assert frozenset(m.to_remove) == to_remove
 
 
 def test_load_tasks_file(tmpdir):

repos/system_upgrade/common/configs/rhui.py

@@ -0,0 +1,81 @@
+"""
+Configuration keys for RHUI.
+
+In case of RHUI in private regions it usual that publicly known RHUI data
+is not valid. In such cases it's possible to provide the correct expected
+RHUI data to correct the in-place upgrade process.
+"""
+
+from leapp.actors.configs import Config
+from leapp.models import fields
+
+
+class RhuiSrcPkg(Config):
+    section = "rhui"
+    name = "src_pkg"
+    type_ = fields.String(default="rhui")
+    default = "rhui"
+    description = """
+        The name of the source RHUI client RPM (installed on the system).
+        Default: rhui.
+    """
+
+
+class RhuiTargetPkg(Config):
+    section = "rhui"
+    name = "target_pkg"
+    type_ = fields.String(default="rhui")
+    default = "rhui"
+    description = """
+        The name of the target RHUI client RPM (to be installed on the system).
+        Default: rhui
+    """
+
+
+class RhuiLeappRhuiPkg(Config):
+    section = "rhui"
+    name = "leapp_rhui_pkg"
+    type_ = fields.String(default="leapp-rhui")
+    default = "leapp-rhui"
+    description = """
+    The name of the leapp-rhui RPM.  Default: leapp-rhui
+    """
+
+
+class RhuiLeappRhuiPkgRepo(Config):
+    section = "rhui"
+    name = "leapp_rhui_pkg_repo"
+    type_ = fields.String(default="rhel-base")
+    default = "rhel-base"
+    description = """
+        The repository ID containing the specified leapp-rhui RPM.
+        Default: rhel-base
+    """
+
+
+all_rhui_cfg = (RhuiSrcPkg, RhuiTargetPkg, RhuiLeappRhuiPkg, RhuiLeappRhuiPkg)
+# Usage: from configs import rhui
+#        class MyActor:
+#            [...]
+#            configs = all_rhui_cfg + (MyConfig,)
+
+# TODO: We need to implement fields.Map before this can be enabled
+'''
+class RhuiFileMap(Config):
+    section = "rhui"
+    name = "file_map"
+    type_ = fields.Map(fields.String())
+    description = """
+        Define directories to which paritcular files provided by the leapp-rhui
+        RPM should be installed. The files in 'files_map' are provided by
+        special Leapp rpms (per cloud) and are supposed to be delivered into the
+        repos/system_upgrade/common/files/rhui/<PROVIDER> directory.
+
+        These files are usually needed to get access to the target system repositories
+        using RHUI. Typically these are certificates, keys, and repofiles with the
+        target RHUI repositories.
+
+        The key is the name of the file, the value is the expected directory
+        where the file should be installed on the upgraded system.
+    """
+'''