[ci] Add debian-12 to matrix in packaging and platform jobs (#116172)

Lintian test has been changed to parse the result instead of using exit
code. This was required, because now `mismatched-override` is
non-erasable tag which cannot be ignored for exit code.
Lintian introduced non-backward-compatible format change for overrides
file. Because of that, some overrides are now duplicated in a format for
 older versions.
Additionally, Lintian overrides file has been cleaned up to remove the
tags which are no longer failing.

(cherry picked from commit 2ac267d)

# Conflicts:
#	distribution/packages/src/deb/lintian/elasticsearch
#	qa/os/src/test/java/org/elasticsearch/packaging/util/LintianResultParser.java
#	qa/packaging/src/test/java/org/elasticsearch/packaging/test/DebMetadataTests.java

Author: jozala

.buildkite/pipelines/periodic-packaging.template.yml

@@ -8,6 +8,7 @@ steps:
           setup:
             image:
               - debian-11
+              - debian-12
               - opensuse-leap-15
               - oraclelinux-7
               - oraclelinux-8

.buildkite/pipelines/periodic-packaging.yml

@@ -9,6 +9,7 @@ steps:
           setup:
             image:
               - debian-11
+              - debian-12
               - opensuse-leap-15
               - oraclelinux-7
               - oraclelinux-8

.buildkite/pipelines/periodic-platform-support.yml

@@ -8,6 +8,7 @@ steps:
           setup:
             image:
               - debian-11
+              - debian-12
               - opensuse-leap-15
               - oraclelinux-7
               - oraclelinux-8

.buildkite/pipelines/pull-request/packaging-tests-unix.yml

@@ -11,6 +11,7 @@ steps:
           setup:
             image:
               - debian-11
+              - debian-12
               - opensuse-leap-15
               - oraclelinux-7
               - oraclelinux-8
@@ -38,6 +39,7 @@ steps:
           setup:
             image:
               - debian-11
+              - debian-12
               - opensuse-leap-15
               - oraclelinux-7
               - oraclelinux-8
@@ -65,6 +67,7 @@ steps:
           setup:
             image:
               - debian-11
+              - debian-12
               - opensuse-leap-15
               - oraclelinux-7
               - oraclelinux-8

distribution/packages/build.gradle

@@ -359,7 +359,6 @@ Closure commonDebConfig(boolean oss, boolean jdk, String architecture) {
 
     // versions found on oldest supported distro, centos-6
     requires('bash', '4.1', GREATER | EQUAL)
-    requires('lsb-base', '4', GREATER | EQUAL)
     requires 'libc6'
     requires 'adduser'
 

distribution/packages/src/deb/lintian/elasticsearch

@@ -1,10 +1,10 @@
 # we don't have a changelog, but we put our copyright file
 # under /usr/share/doc/elasticsearch, which triggers this warning
+# Note that this is renamed to `no-changelog` in newer versions of
+# lintian, but we still support Debian 8+, so we can't change this.
 changelog-file-missing-in-native-package
 
 # we intentionally copy our copyright file for all deb packages
-copyright-file-contains-full-apache-2-license
-copyright-should-refer-to-common-license-file-for-apache-2
 copyright-without-copyright-notice
 
 # we still put all our files under /usr/share/elasticsearch even after transition to platform dependent packages
@@ -13,37 +13,59 @@ arch-dependent-file-in-usr-share
 # we have a bundled jdk, so don't use jarwrapper
 missing-dep-on-jarwrapper
 
+# we prefer to not make our config and log files world readable
+non-standard-file-perm 0660 != 0644 [etc/default/elasticsearch]
+non-standard-dir-perm 2750 != 0755 [etc/elasticsearch/]
+non-standard-dir-perm 2750 != 0755 [etc/elasticsearch/jvm.options.d/]
+non-standard-file-perm 0660 != 0644 [etc/elasticsearch/*]
+non-standard-dir-perm 2750 != 0755 [var/lib/elasticsearch/]
+non-standard-dir-perm 2750 != 0755 [var/log/elasticsearch/]
+non-standard-executable-perm 0750 != 0755 [etc/init.d/elasticsearch]
+executable-is-not-world-readable 0750 [etc/init.d/elasticsearch]
+non-standard-file-permissions-for-etc-init.d-script 0750 != 0755 [etc/init.d/elasticsearch]
+
+# the package scripts handle systemd directly and don't need to use deb helpers
+maintainer-script-calls-systemctl
+prerm-calls-updaterc.d elasticsearch [prerm:94]
+
+# we do not automatically enable the service in init.d or systemd
+script-in-etc-init.d-not-registered-via-update-rc.d [etc/init.d/elasticsearch]
+
+# bundled JDK
+embedded-library
+unstripped-binary-or-object [usr/share/elasticsearch/jdk/*]
+
+# the system java version that lintian assumes is far behind what elasticsearch uses
+unknown-java-class-version
+
+# There's no `License` field in Debian control files, but earlier versions
+# of `lintian` were more permissive. Override this warning so that we can
+# run `lintian` on different releases of Debian. The format of this override
+# varies between `lintian` versions.
+unknown-field elasticsearch-*.deb License
+unknown-field License
+
+
+# Below is the copy of some of the above rules in format for Lintian versions <= 2.104 (Debian 11)
+# Override syntax changes between Lintian versions in a non-backwards compatible way, so we handle it with
+# duplication and ignoring some issues in the test code.
+
 # we prefer to not make our config and log files world readable
 non-standard-file-perm etc/default/elasticsearch 0660 != 0644
 non-standard-dir-perm etc/elasticsearch/ 2750 != 0755
 non-standard-dir-perm etc/elasticsearch/jvm.options.d/ 2750 != 0755
 non-standard-file-perm etc/elasticsearch/*
 non-standard-dir-perm var/lib/elasticsearch/ 2750 != 0755
 non-standard-dir-perm var/log/elasticsearch/ 2750 != 0755
+non-standard-executable-perm etc/init.d/elasticsearch 0750 != 0755
 executable-is-not-world-readable etc/init.d/elasticsearch 0750
 non-standard-file-permissions-for-etc-init.d-script etc/init.d/elasticsearch 0750 != 0755
 
-# this lintian tag is simply wrong; contrary to the explanation, debian systemd
-# does actually look at /usr/lib/systemd/system
-systemd-service-file-outside-lib usr/lib/systemd/system/elasticsearch.service
-
 # we do not automatically enable the service in init.d or systemd
 script-in-etc-init.d-not-registered-via-update-rc.d etc/init.d/elasticsearch
 
 # the package scripts handle init.d/systemd directly and don't need to use deb helpers
-maintainer-script-calls-systemctl
 prerm-calls-updaterc.d elasticsearch
 
 # bundled JDK
-embedded-library
-arch-dependent-file-in-usr-share usr/share/elasticsearch/jdk/*
 unstripped-binary-or-object usr/share/elasticsearch/jdk/*
-extra-license-file usr/share/elasticsearch/jdk/legal/*
-hardening-no-pie usr/share/elasticsearch/jdk/bin/*
-hardening-no-pie usr/share/elasticsearch/jdk/lib/*
-
-# the system java version that lintian assumes is far behind what elasticsearch uses
-unknown-java-class-version
-
-# elastic licensed modules contain elastic license
-extra-license-file usr/share/elasticsearch/modules/*

qa/os/src/test/java/org/elasticsearch/packaging/test/DebMetadataTests.java

@@ -11,28 +11,63 @@
 import junit.framework.TestCase;
 
 import org.elasticsearch.packaging.util.Distribution;
-import org.elasticsearch.packaging.util.FileUtils;
+import org.elasticsearch.packaging.util.LintianResultParser;
+import org.elasticsearch.packaging.util.LintianResultParser.Issue;
+import org.elasticsearch.packaging.util.LintianResultParser.Result;
 import org.elasticsearch.packaging.util.Shell;
 import org.junit.BeforeClass;
 
+import java.util.Arrays;
+import java.util.List;
+import java.util.Locale;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 
 import static org.elasticsearch.packaging.util.FileUtils.getDistributionFile;
 import static org.junit.Assume.assumeTrue;
 
 public class DebMetadataTests extends PackagingTestCase {
 
+    private final LintianResultParser lintianParser = new LintianResultParser();
+    private static final List<String> IGNORED_TAGS = Arrays.asList(
+        // Override syntax changes between lintian versions in a non-backwards compatible way, so we have to tolerate these.
+        // Tag mismatched-override is a non-erasable tag which cannot be ignored with overrides, so we handle it here.
+        "mismatched-override",
+        // systemd-service-file-outside-lib has been incorrect and removed in the newer version on Lintian
+        "systemd-service-file-outside-lib"
+    );
+
     @BeforeClass
     public static void filterDistros() {
         assumeTrue("only deb", distribution.packaging == Distribution.Packaging.DEB);
     }
 
     public void test05CheckLintian() {
         String extraArgs = "";
-        if (sh.run("lintian --help").stdout.contains("fail-on-warnings")) {
-            extraArgs = "--fail-on-warnings ";
+        final String helpText = sh.run("lintian --help").stdout;
+        if (helpText.contains("--fail-on-warnings")) {
+            extraArgs = "--fail-on-warnings";
+        } else if (helpText.contains("--fail-on error")) {
+            extraArgs = "--fail-on error,warning";
+        }
+        Shell.Result result = sh.runIgnoreExitCode(
+            String.format(Locale.ROOT, "lintian %s %s", extraArgs, getDistributionFile(distribution()))
+        );
+        Result lintianResult = lintianParser.parse(result.stdout);
+        // Unfortunately Lintian overrides syntax changes between Lintian versions in a non-backwards compatible
+        // way, so we have to manage some exclusions outside the overrides file.
+        if (lintianResult.isSuccess() == false) {
+            List<Issue> importantIssues = lintianResult.issues()
+                .stream()
+                .filter(issue -> IGNORED_TAGS.contains(issue.tag()) == false)
+                .collect(Collectors.toList());
+            if (importantIssues.isEmpty() == false) {
+                fail(
+                    "Issues for DEB package found by Lintian:\n"
+                        + importantIssues.stream().map(Issue::toString).collect(Collectors.joining("\n"))
+                );
+            }
         }
-        sh.run("lintian " + extraArgs + FileUtils.getDistributionFile(distribution()));
     }
 
     public void test06Dependencies() {

qa/os/src/test/java/org/elasticsearch/packaging/util/LintianResultParser.java

@@ -0,0 +1,117 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.packaging.util;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+public class LintianResultParser {
+
+    private static final Logger logger = LogManager.getLogger(LintianResultParser.class);
+    private static final Pattern RESULT_PATTERN = Pattern.compile("(?<severity>[EW]): (?<package>\\S+): (?<tag>\\S+) (?<message>.+)");
+
+    public Result parse(String output) {
+        String[] lines = output.split("\n");
+        List<Issue> issues = Arrays.stream(lines).map(line -> {
+            Matcher matcher = RESULT_PATTERN.matcher(line);
+            if (matcher.matches() == false) {
+                logger.info("Lintian output not matching expected pattern: {}", line);
+                return null;
+            }
+            Severity severity;
+            switch (matcher.group("severity")) {
+                case "E":
+                    severity = Severity.ERROR;
+                    break;
+                case "W":
+                    severity = Severity.WARNING;
+                    break;
+                default:
+                    severity = Severity.UNKNOWN;
+                    break;
+            }
+            return new Issue(severity, matcher.group("tag"), matcher.group("message"));
+        }).filter(Objects::nonNull).collect(Collectors.toList());
+
+        return new Result(issues.stream().noneMatch(it -> it.severity == Severity.ERROR || it.severity == Severity.WARNING), issues);
+    }
+
+    public static final class Result{
+        private final boolean isSuccess;
+        private final List<Issue> issues;
+
+        public Result(boolean isSuccess, List<Issue> issues) {
+            this.isSuccess = isSuccess;
+            this.issues = issues;
+        }
+
+        public boolean isSuccess() {
+            return isSuccess;
+        }
+
+        public List<Issue> issues() {
+            return issues;
+        }
+
+        @Override
+        public String toString() {
+            return "Result{" +
+                "isSuccess=" + isSuccess +
+                ", issues=" + issues +
+                '}';
+        }
+    }
+
+    public static final class Issue {
+        private final Severity severity;
+        private final String tag;
+        private final String message;
+
+        public Issue(Severity severity, String tag, String message) {
+            this.severity = severity;
+            this.tag = tag;
+            this.message = message;
+        }
+
+        public Severity severity() {
+            return severity;
+        }
+
+        public String tag() {
+            return tag;
+        }
+
+        public String message() {
+            return message;
+        }
+
+        @Override
+        public String toString() {
+            return "Issue{" +
+                "severity=" + severity +
+                ", tag='" + tag + '\'' +
+                ", message='" + message + '\'' +
+                '}';
+        }
+    }
+
+    public enum Severity {
+        ERROR,
+        WARNING,
+        UNKNOWN
+    }
+}