[ci] Add debian-12 to matrix in packaging and platform jobs (elastic#116172)

Lintian test has been changed to parse the result instead of using exit
code. This was required, because now `mismatched-override` is
non-erasable tag which cannot be ignored for exit code.
Lintian introduced non-backward-compatible format change for overrides
file. Because of that, some overrides are now duplicated in a format for
 older versions.
Additionally, Lintian overrides file has been cleaned up to remove the
tags which are no longer failing.

Author: jozala

.buildkite/pipelines/periodic-packaging.template.yml

@@ -8,6 +8,7 @@ steps:
           setup:
             image:
               - debian-11
+              - debian-12
               - opensuse-leap-15
               - oraclelinux-7
               - oraclelinux-8

.buildkite/pipelines/periodic-packaging.yml

@@ -9,6 +9,7 @@ steps:
           setup:
             image:
               - debian-11
+              - debian-12
               - opensuse-leap-15
               - oraclelinux-7
               - oraclelinux-8

.buildkite/pipelines/periodic-platform-support.yml

@@ -8,6 +8,7 @@ steps:
           setup:
             image:
               - debian-11
+              - debian-12
               - opensuse-leap-15
               - oraclelinux-7
               - oraclelinux-8

.buildkite/pipelines/pull-request/packaging-tests-unix.yml

@@ -11,6 +11,7 @@ steps:
           setup:
             image:
               - debian-11
+              - debian-12
               - opensuse-leap-15
               - oraclelinux-7
               - oraclelinux-8
@@ -38,6 +39,7 @@ steps:
           setup:
             image:
               - debian-11
+              - debian-12
               - opensuse-leap-15
               - oraclelinux-7
               - oraclelinux-8
@@ -65,6 +67,7 @@ steps:
           setup:
             image:
               - debian-11
+              - debian-12
               - opensuse-leap-15
               - oraclelinux-7
               - oraclelinux-8

distribution/packages/build.gradle

@@ -335,7 +335,6 @@ Closure commonDebConfig(String architecture) {
 
     // versions found on oldest supported distro, centos-6
     requires('bash', '4.1', GREATER | EQUAL)
-    requires('lsb-base', '4', GREATER | EQUAL)
     requires 'libc6'
     requires 'adduser'
 

distribution/packages/src/deb/lintian/elasticsearch

@@ -5,8 +5,6 @@
 changelog-file-missing-in-native-package
 
 # we intentionally copy our copyright file for all deb packages
-copyright-file-contains-full-apache-2-license
-copyright-not-using-common-license-for-apache2
 copyright-without-copyright-notice
 
 # we still put all our files under /usr/share/elasticsearch even after transition to platform dependent packages
@@ -16,37 +14,23 @@ arch-dependent-file-in-usr-share
 missing-dep-on-jarwrapper
 
 # we prefer to not make our config and log files world readable
-non-standard-file-perm etc/default/elasticsearch 0660 != 0644
-non-standard-dir-perm etc/elasticsearch/ 2750 != 0755
-non-standard-dir-perm etc/elasticsearch/jvm.options.d/ 2750 != 0755
-non-standard-file-perm etc/elasticsearch/*
-non-standard-dir-perm var/lib/elasticsearch/ 2750 != 0755
-non-standard-dir-perm var/log/elasticsearch/ 2750 != 0755
-
-# this lintian tag is simply wrong; contrary to the explanation, Debian systemd
-# does actually look at /usr/lib/systemd/system
-systemd-service-file-outside-lib usr/lib/systemd/system/elasticsearch.service
+non-standard-file-perm 0660 != 0644 [etc/default/elasticsearch]
+non-standard-dir-perm 2750 != 0755 [etc/elasticsearch/]
+non-standard-dir-perm 2750 != 0755 [etc/elasticsearch/jvm.options.d/]
+non-standard-file-perm 0660 != 0644 [etc/elasticsearch/*]
+non-standard-dir-perm 2750 != 0755 [var/lib/elasticsearch/]
+non-standard-dir-perm 2750 != 0755 [var/log/elasticsearch/]
 
 # the package scripts handle systemd directly and don't need to use deb helpers
 maintainer-script-calls-systemctl
 
 # bundled JDK
 embedded-library
-unstripped-binary-or-object usr/share/elasticsearch/jdk/*
-extra-license-file usr/share/elasticsearch/jdk/legal/*
-hardening-no-pie usr/share/elasticsearch/jdk/bin/*
-hardening-no-pie usr/share/elasticsearch/jdk/lib/*
+unstripped-binary-or-object [usr/share/elasticsearch/jdk/*]
 
 # the system java version that lintian assumes is far behind what elasticsearch uses
 unknown-java-class-version
 
-# elastic licensed modules contain elastic license
-extra-license-file usr/share/elasticsearch/modules/*
-
-# This dependency appears to have a packaging flaw, and includes a
-# generated source file alongside the compiled version
-jar-contains-source usr/share/elasticsearch/modules/repository-gcs/api-common*.jar *
-
 # There's no `License` field in Debian control files, but earlier versions
 # of `lintian` were more permissive. Override this warning so that we can
 # run `lintian` on different releases of Debian. The format of this override
@@ -58,8 +42,27 @@ unknown-field License
 # indirectly to libc via libdl. This might not be best practice but we
 # don't build them ourselves and the license precludes us modifying them
 # to fix this.
-library-not-linked-against-libc usr/share/elasticsearch/modules/x-pack-ml/platform/linux-x86_64/lib/libmkl_*.so
+library-not-linked-against-libc [usr/share/elasticsearch/modules/x-pack-ml/platform/linux-x86_64/lib/libmkl_*.so*]
+
+
+# Below is the copy of some of the above rules in format for Lintian versions <= 2.104 (Debian 11)
+# Override syntax changes between Lintian versions in a non-backwards compatible way, so we handle it with
+# duplication and ignoring some issues in the test code.
+
+
+# we prefer to not make our config and log files world readable
+non-standard-file-perm etc/default/elasticsearch 0660 != 0644
+non-standard-dir-perm etc/elasticsearch/ 2750 != 0755
+non-standard-dir-perm etc/elasticsearch/jvm.options.d/ 2750 != 0755
+non-standard-file-perm etc/elasticsearch/*
+non-standard-dir-perm var/lib/elasticsearch/ 2750 != 0755
+non-standard-dir-perm var/log/elasticsearch/ 2750 != 0755
 
-# shared-lib-without-dependency-information (now shared-library-lacks-prerequisites) is falsely reported for libvec.so
-# which has no dependencies (not even libc) besides the symbols in the base executable.
-shared-lib-without-dependency-information usr/share/elasticsearch/lib/platform/linux-x64/libvec.so
+# bundled JDK
+unstripped-binary-or-object usr/share/elasticsearch/jdk/*
+
+# Intel MKL libraries are not linked directly to libc. They are linked
+# indirectly to libc via libdl. This might not be best practice but we
+# don't build them ourselves and the license precludes us modifying them
+# to fix this.
+library-not-linked-against-libc usr/share/elasticsearch/modules/x-pack-ml/platform/linux-x86_64/lib/libmkl_*.so*

qa/packaging/src/test/java/org/elasticsearch/packaging/test/DebMetadataTests.java

@@ -12,18 +12,31 @@
 import junit.framework.TestCase;
 
 import org.elasticsearch.packaging.util.Distribution;
-import org.elasticsearch.packaging.util.FileUtils;
+import org.elasticsearch.packaging.util.LintianResultParser;
+import org.elasticsearch.packaging.util.LintianResultParser.Issue;
+import org.elasticsearch.packaging.util.LintianResultParser.Result;
 import org.elasticsearch.packaging.util.Shell;
 import org.junit.BeforeClass;
 
+import java.util.List;
 import java.util.Locale;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 
 import static org.elasticsearch.packaging.util.FileUtils.getDistributionFile;
 import static org.junit.Assume.assumeTrue;
 
 public class DebMetadataTests extends PackagingTestCase {
 
+    private final LintianResultParser lintianParser = new LintianResultParser();
+    private static final List<String> IGNORED_TAGS = List.of(
+        // Override syntax changes between lintian versions in a non-backwards compatible way, so we have to tolerate these.
+        // Tag mismatched-override is a non-erasable tag which cannot be ignored with overrides, so we handle it here.
+        "mismatched-override",
+        // systemd-service-file-outside-lib has been incorrect and removed in the newer version on Lintian
+        "systemd-service-file-outside-lib"
+    );
+
     @BeforeClass
     public static void filterDistros() {
         assumeTrue("only deb", distribution.packaging == Distribution.Packaging.DEB);
@@ -35,15 +48,26 @@ public void test05CheckLintian() {
         if (helpText.contains("--fail-on-warnings")) {
             extraArgs = "--fail-on-warnings";
         } else if (helpText.contains("--fail-on error")) {
-            extraArgs = "--fail-on warning";
-            // Recent lintian versions are picky about malformed or mismatched overrides.
-            // Unfortunately override syntax changes between lintian versions in a non-backwards compatible
-            // way, so we have to tolerate these (or maintain separate override files per lintian version).
-            if (helpText.contains("--suppress-tags")) {
-                extraArgs += " --suppress-tags malformed-override,mismatched-override";
+            extraArgs = "--fail-on error,warning";
+        }
+        Shell.Result result = sh.runIgnoreExitCode(
+            String.format(Locale.ROOT, "lintian %s %s", extraArgs, getDistributionFile(distribution()))
+        );
+        Result lintianResult = lintianParser.parse(result.stdout());
+        // Unfortunately Lintian overrides syntax changes between Lintian versions in a non-backwards compatible
+        // way, so we have to manage some exclusions outside the overrides file.
+        if (lintianResult.isSuccess() == false) {
+            List<Issue> importantIssues = lintianResult.issues()
+                .stream()
+                .filter(issue -> IGNORED_TAGS.contains(issue.tag()) == false)
+                .toList();
+            if (importantIssues.isEmpty() == false) {
+                fail(
+                    "Issues for DEB package found by Lintian:\n"
+                        + importantIssues.stream().map(Record::toString).collect(Collectors.joining("\n"))
+                );
             }
         }
-        sh.run(String.format(Locale.ROOT, "lintian %s %s", extraArgs, FileUtils.getDistributionFile(distribution())));
     }
 
     public void test06Dependencies() {

qa/packaging/src/test/java/org/elasticsearch/packaging/util/LintianResultParser.java

@@ -0,0 +1,54 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.packaging.util;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Objects;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class LintianResultParser {
+
+    private static final Logger logger = LogManager.getLogger(LintianResultParser.class);
+    private static final Pattern RESULT_PATTERN = Pattern.compile("(?<severity>[EW]): (?<package>\\S+): (?<tag>\\S+) (?<message>.+)");
+
+    public Result parse(String output) {
+        String[] lines = output.split("\n");
+        List<Issue> issues = Arrays.stream(lines).map(line -> {
+            Matcher matcher = RESULT_PATTERN.matcher(line);
+            if (matcher.matches() == false) {
+                logger.info("Lintian output not matching expected pattern: {}", line);
+                return null;
+            }
+            Severity severity = switch (matcher.group("severity")) {
+                case "E" -> Severity.ERROR;
+                case "W" -> Severity.WARNING;
+                default -> Severity.UNKNOWN;
+            };
+            return new Issue(severity, matcher.group("tag"), matcher.group("message"));
+        }).filter(Objects::nonNull).toList();
+
+        return new Result(issues.stream().noneMatch(it -> it.severity == Severity.ERROR || it.severity == Severity.WARNING), issues);
+    }
+
+    public record Result(boolean isSuccess, List<Issue> issues) {}
+
+    public record Issue(Severity severity, String tag, String message) {}
+
+    enum Severity {
+        ERROR,
+        WARNING,
+        UNKNOWN
+    }
+}