Merge pull request #33 from cloudlinux/epel

CLOS-3205, CLOS-3230: Rework package conflict actor and add EPEL repo reset

repos/system_upgrade/cloudlinux/actors/clearpackageconflicts/actor.py

@@ -1,102 +1,28 @@
-import os
-import errno
-import shutil
-
 from leapp.actors import Actor
 from leapp.models import InstalledRPM
 from leapp.tags import DownloadPhaseTag, IPUWorkflowTag
 from leapp.libraries.common.cllaunch import run_on_cloudlinux
+from leapp.libraries.actor import clearpackageconflicts
 
 
 class ClearPackageConflicts(Actor):
     """
-    Remove several python package files manually to resolve conflicts between versions of packages to be upgraded.
+    Remove several Python package files manually to resolve conflicts
+    between versions of packages to be upgraded.
+
+    When the corresponding packages are detected,
+    the conflicting files are removed to allow for an upgrade to the new package versions.
+
+    While most packages are handled automatically by the package manager,
+    some specific packages require direct intervention to resolve conflicts
+    between their own versions on different OS releases.
     """
 
     name = "clear_package_conflicts"
     consumes = (InstalledRPM,)
     produces = ()
     tags = (DownloadPhaseTag.Before, IPUWorkflowTag)
-    rpm_lookup = None
-
-    def has_package(self, name):
-        """
-        Check whether the package is installed.
-        Looks only for the package name, nothing else.
-        """
-        if self.rpm_lookup:
-            return name in self.rpm_lookup
-
-    def problem_packages_installed(self, problem_packages):
-        """
-        Check whether any of the problem packages are present in the system.
-        """
-        for pkg in problem_packages:
-            if self.has_package(pkg):
-                self.log.debug("Conflicting package {} detected".format(pkg))
-                return True
-        return False
-
-    def clear_problem_files(self, problem_files, problem_dirs):
-        """
-        Go over the list of problem files and directories and remove them if they exist.
-        They'll be replaced by the new packages.
-        """
-        for p_dir in problem_dirs:
-            try:
-                if os.path.isdir(p_dir):
-                    shutil.rmtree(p_dir)
-                    self.log.debug("Conflicting directory {} removed".format(p_dir))
-            except OSError as e:
-                if e.errno != errno.ENOENT:
-                    raise
-
-        for p_file in problem_files:
-            try:
-                if os.path.isfile(p_file):
-                    os.remove(p_file)
-                    self.log.debug("Conflicting file {} removed".format(p_file))
-            except OSError as e:
-                if e.errno != errno.ENOENT:
-                    raise
-
-    def alt_python37_handle(self):
-        """
-        These alt-python37 packages are conflicting with their own builds for EL8.
-        """
-        problem_packages = [
-            "alt-python37-six",
-            "alt-python37-pytz",
-        ]
-        problem_files = []
-        problem_dirs = [
-            "/opt/alt/python37/lib/python3.7/site-packages/six-1.15.0-py3.7.egg-info",
-            "/opt/alt/python37/lib/python3.7/site-packages/pytz-2017.2-py3.7.egg-info",
-        ]
-
-        if self.problem_packages_installed(problem_packages):
-            self.clear_problem_files(problem_files, problem_dirs)
-
-    def lua_cjson_handle(self):
-        """
-        lua-cjson package is conflicting with the incoming lua-cjson package for EL8.
-        """
-        problem_packages = [
-            "lua-cjson"
-        ]
-        problem_files = [
-            "/usr/lib64/lua/5.1/cjson.so",
-            "/usr/share/lua/5.1/cjson/tests/bench.lua",
-            "/usr/share/lua/5.1/cjson/tests/genutf8.pl",
-            "/usr/share/lua/5.1/cjson/tests/test.lua",
-        ]
-        problem_dirs = []
-
-        if self.problem_packages_installed(problem_packages):
-            self.clear_problem_files(problem_files, problem_dirs)
 
     @run_on_cloudlinux
     def process(self):
-        # todo: (CLOS-3205) investigate why set is needed here
-        self.rpm_lookup = [rpm for rpm in self.consume(InstalledRPM)]
-        self.alt_python37_handle()
+        clearpackageconflicts.process()

repos/system_upgrade/cloudlinux/actors/clearpackageconflicts/libraries/clearpackageconflicts.py

@@ -0,0 +1,90 @@
+import os
+import errno
+import shutil
+
+from leapp.libraries.stdlib import api
+from leapp.models import InstalledRPM
+
+
+def problem_packages_installed(problem_packages, lookup):
+    """
+    Check whether any of the problem packages are present in the system.
+    """
+    for pkg in problem_packages:
+        if pkg in lookup:
+            api.current_logger().debug("Conflicting package {} detected".format(pkg))
+            return True
+    return False
+
+
+def clear_problem_files(problem_files, problem_dirs):
+    """
+    Go over the list of problem files and directories and remove them if they exist.
+    They'll be replaced by the new packages.
+    """
+    for p_dir in problem_dirs:
+        try:
+            if os.path.isdir(p_dir):
+                shutil.rmtree(p_dir)
+                api.current_logger().debug("Conflicting directory {} removed".format(p_dir))
+        except OSError as e:
+            if e.errno != errno.ENOENT:
+                raise
+
+    for p_file in problem_files:
+        try:
+            if os.path.isfile(p_file):
+                os.remove(p_file)
+                api.current_logger().debug("Conflicting file {} removed".format(p_file))
+        except OSError as e:
+            if e.errno != errno.ENOENT:
+                raise
+
+
+def alt_python37_handle(package_lookup):
+    """
+    These alt-python37 packages are conflicting with their own builds for EL8.
+    """
+    problem_packages = [
+        "alt-python37-six",
+        "alt-python37-pytz",
+    ]
+    problem_files = []
+    problem_dirs = [
+        "/opt/alt/python37/lib/python3.7/site-packages/six-1.15.0-py3.7.egg-info",
+        "/opt/alt/python37/lib/python3.7/site-packages/pytz-2017.2-py3.7.egg-info",
+    ]
+
+    if problem_packages_installed(problem_packages, package_lookup):
+        clear_problem_files(problem_files, problem_dirs)
+
+
+def lua_cjson_handle(package_lookup):
+    """
+    lua-cjson package is conflicting with the incoming lua-cjson package for EL8.
+    """
+    problem_packages = [
+        "lua-cjson"
+    ]
+    problem_files = [
+        "/usr/lib64/lua/5.1/cjson.so",
+        "/usr/share/lua/5.1/cjson/tests/bench.lua",
+        "/usr/share/lua/5.1/cjson/tests/genutf8.pl",
+        "/usr/share/lua/5.1/cjson/tests/test.lua",
+    ]
+    problem_dirs = []
+
+    if problem_packages_installed(problem_packages, package_lookup):
+        clear_problem_files(problem_files, problem_dirs)
+
+
+def process():
+    rpm_lookup = set()
+    # Each InstalledRPM is a list of RPM objects.
+    # There's a bunch of other fields, but all that we're interested in here is their names.
+    installed_rpm_messages = api.consume(InstalledRPM)
+    for rpm_list in installed_rpm_messages:
+        rpm_names = [item.name for item in rpm_list.items]
+        rpm_lookup.update(rpm_names)
+
+    alt_python37_handle(rpm_lookup)

repos/system_upgrade/cloudlinux/actors/clearpackageconflicts/tests/unit_test_clearpackageconflicts.py

@@ -0,0 +1,17 @@
+import pytest
+
+# from leapp import reporting
+from leapp.libraries.actor import clearpackageconflicts
+
+
+@pytest.mark.parametrize(
+    "problem_pkgs,lookup,expected_res",
+    (
+        (["cagefs"], {"cagefs", "dnf"}, True),
+        (["lve-utils"], {"lve-utils", "dnf"}, True),
+        (["nonexistent-pkg"], {"cagefs", "dnf"}, False),
+        (["cagefs"], {"lve-utils", "dnf"}, False),
+    ),
+)
+def test_problem_packages_installed(problem_pkgs, lookup, expected_res):
+    assert expected_res == clearpackageconflicts.problem_packages_installed(problem_pkgs, lookup)

repos/system_upgrade/cloudlinux/actors/refreshepel/actor.py

@@ -0,0 +1,68 @@
+from __future__ import print_function
+from operator import is_
+import os
+
+from leapp.actors import Actor
+from leapp.tags import ApplicationsPhaseTag, IPUWorkflowTag
+from leapp.libraries.common.cllaunch import run_on_cloudlinux
+from leapp.libraries.common.backup import backup_and_remove
+from leapp.libraries.common.config.version import get_target_major_version
+
+REPO_DIR = '/etc/yum.repos.d'
+
+
+class RefreshEPEL(Actor):
+    """
+    Check that the EPEL repositories are correctly configured after the upgrade.
+
+    Depending on how the upgrade went, the EPEL repositories might still be targeting the old OS version.
+    This actor checks that the EPEL repositories are correctly configured and if not, it will install the
+    correct EPEL release package and refresh the repositories.
+    """
+
+    name = 'refresh_epel'
+    # We can't depend on InstalledRPM message because by this point
+    # the system is upgraded and the RPMs are not the same as when the data was collected.
+    consumes = ()
+    produces = ()
+    tags = (ApplicationsPhaseTag.After, IPUWorkflowTag)
+
+    def clear_epel_repo_files(self):
+        for repofile in os.listdir(REPO_DIR):
+            if repofile.startswith('epel'):
+                epel_file = os.path.join(REPO_DIR, repofile)
+                backup_and_remove(epel_file)
+
+    def install_epel_release_package(self, target_url):
+        os.system('dnf install {}'.format(target_url))
+        self.log.info('EPEL release package installed: {}'.format(target_url))
+
+    @run_on_cloudlinux
+    def process(self):
+        epel_install_url = 'https://dl.fedoraproject.org/pub/epel/epel-release-latest-{}.noarch.rpm'.format(get_target_major_version())
+
+        target_version = int(get_target_major_version())
+        target_epel_release = epel_install_url.format(target_version)
+
+        # EPEL release package name is 'epel-release' and the version should match the target OS version
+        epel_release_package = 'epel-release'
+
+        is_epel_installed = os.system('rpm -q {}'.format(epel_release_package)) == 0
+        is_correct_version = os.system('rpm -q {}-{}'.format(epel_release_package, target_version)) == 0
+        epel_files_verified = os.system('rpm -V {}'.format(epel_release_package)) == 0
+
+        # It's possible (although unusual) that the correct EPEL release package is installed during the upgrade,
+        # but the EPEL repository files still point to the old OS version.
+        # This was observed on client machines before.
+
+        if (is_epel_installed and not is_correct_version) or not epel_files_verified:
+            # If the EPEL release package is installed but not the correct version, remove it
+            # Same if the files from the package were modified
+            os.system('rpm -e {}'.format(epel_release_package))
+        if not is_epel_installed or not is_correct_version or not epel_files_verified:
+            # Clear the EPEL repository files
+            self.clear_epel_repo_files()
+            # Install the correct EPEL release package
+            self.install_epel_release_package(target_epel_release)
+            # Logging for clarity
+            self.log.info('EPEL release package installation invoked for: {}'.format(target_epel_release))

repos/system_upgrade/cloudlinux/actors/replacerpmnewconfigs/actor.py

@@ -7,12 +7,15 @@
 from leapp import reporting
 from leapp.reporting import Report
 from leapp.libraries.common.cllaunch import run_on_cloudlinux
+from leapp.libraries.common.backup import backup_and_remove, LEAPP_BACKUP_SUFFIX
 
 REPO_DIR = '/etc/yum.repos.d'
-REPO_DELETE_MARKERS = ['cloudlinux', 'imunify', 'epel']
+# These markers are used to identify which repository files should be directly replaced with new versions.
+REPO_DELETE_MARKERS = ['cloudlinux', 'imunify']
+# These markers are used to identify which repository files should be replaced with new versions and backed up.
 REPO_BACKUP_MARKERS = []
+# This suffix is used to identify .rpmnew files that appear after package upgrade.
 RPMNEW = '.rpmnew'
-LEAPP_BACKUP_SUFFIX = '.leapp-backup'
 
 
 class ReplaceRpmnewConfigs(Actor):
@@ -30,32 +33,31 @@ def process(self):
         deleted_repofiles = []
         renamed_repofiles = []
 
-        for reponame in os.listdir(REPO_DIR):
-            if any(mark in reponame for mark in REPO_DELETE_MARKERS) and RPMNEW in reponame:
-                base_reponame = reponame[:-len(RPMNEW)]
-                base_path = os.path.join(REPO_DIR, base_reponame)
-                new_file_path = os.path.join(REPO_DIR, reponame)
+        for rpmnew_filename in os.listdir(REPO_DIR):
+            if any(mark in rpmnew_filename for mark in REPO_DELETE_MARKERS) and rpmnew_filename.endswith(RPMNEW):
+                main_reponame = rpmnew_filename[:-len(RPMNEW)]
+                main_file_path = os.path.join(REPO_DIR, main_reponame)
+                rpmnew_file_path = os.path.join(REPO_DIR, rpmnew_filename)
 
-                os.unlink(base_path)
-                os.rename(new_file_path, base_path)
-                deleted_repofiles.append(base_reponame)
-                self.log.debug('Yum repofile replaced: {}'.format(base_path))
+                os.unlink(main_file_path)
+                os.rename(rpmnew_file_path, main_file_path)
+                deleted_repofiles.append(main_reponame)
+                self.log.debug('Yum repofile replaced: {}'.format(main_file_path))
 
-            if any(mark in reponame for mark in REPO_BACKUP_MARKERS) and RPMNEW in reponame:
-                base_reponame = reponame[:-len(RPMNEW)]
-                base_path = os.path.join(REPO_DIR, base_reponame)
-                new_file_path = os.path.join(REPO_DIR, reponame)
-                backup_path = os.path.join(REPO_DIR, base_reponame + LEAPP_BACKUP_SUFFIX)
+            if any(mark in rpmnew_filename for mark in REPO_BACKUP_MARKERS) and rpmnew_filename.endswith(RPMNEW):
+                main_reponame = rpmnew_filename[:-len(RPMNEW)]
+                main_file_path = os.path.join(REPO_DIR, main_reponame)
+                rpmnew_file_path = os.path.join(REPO_DIR, rpmnew_filename)
 
-                os.rename(base_path, backup_path)
-                os.rename(new_file_path, base_path)
-                renamed_repofiles.append(base_reponame)
-                self.log.debug('Yum repofile replaced with backup: {}'.format(base_path))
+                backup_and_remove(main_file_path)
+                os.rename(rpmnew_file_path, main_file_path)
+                renamed_repofiles.append(main_reponame)
+                self.log.debug('Yum repofile replaced with backup: {}'.format(main_file_path))
 
         # Disable any old repositories.
-        for reponame in os.listdir(REPO_DIR):
-            if LEAPP_BACKUP_SUFFIX in reponame:
-                repofile_path = os.path.join(REPO_DIR, reponame)
+        for repofile_name in os.listdir(REPO_DIR):
+            if LEAPP_BACKUP_SUFFIX in repofile_name:
+                repofile_path = os.path.join(REPO_DIR, repofile_name)
                 for line in fileinput.input(repofile_path, inplace=True):
                     if line.startswith('enabled'):
                         print("enabled = 0")
@@ -66,7 +68,7 @@ def process(self):
             deleted_string = '\n'.join(['{}'.format(repofile_name) for repofile_name in deleted_repofiles])
             replaced_string = '\n'.join(['{}'.format(repofile_name) for repofile_name in renamed_repofiles])
             reporting.create_report([
-                reporting.Title('CloudLinux repository config files replaced by updated versions'),
+                reporting.Title('Repository config files replaced by updated versions'),
                 reporting.Summary(
                     'One or more RPM repository configuration files '
                     'have been replaced with new versions provided by the upgraded packages. '