Toolkit: provide mechanism to auto-install build deps when building on azl

.github/workflows/check-prerequisites.yml

@@ -0,0 +1,44 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+# This action checks that the licenses.json file is up-to-date
+# and that the LICENSES-MAP.md file is up-to-date
+name: Prerequisites Check
+
+on:
+  push:
+    branches: [main, dev, 1.0*, 2.0*, 3.0*, fasttrack/*]
+  pull_request:
+    branches: [main, dev, 1.0*, 2.0*, 3.0*, fasttrack/*]
+
+jobs:
+  check:
+    name: Prerequisites Check
+    runs-on: ubuntu-latest
+
+    steps:
+      # Checkout the branch of our repo that triggered this action
+      - name: Workflow trigger checkout
+        uses: actions/checkout@v4
+
+      - name: Run prerequisites checking script for Ubuntu
+        run: |
+          if [ ! -f ./toolkit/docs/building/prerequisites-src.json ]; then
+            echo "prerequisites-src.json not found"
+            exit 0
+          fi
+          ./toolkit/scripts/prerequisites.sh \
+            -m ./toolkit/docs/building/prerequisites-ubuntu.md \
+            -s ./toolkit/docs/building/prerequisites-src.json \
+            -d "ubuntu"
+
+      - name: Run prerequisites checking script for AzureLinux
+        run: |
+          if [ ! -f ./toolkit/docs/building/prerequisites-src.json ]; then
+            echo "prerequisites-src.json not found"
+            exit 0
+          fi
+          ./toolkit/scripts/prerequisites.sh \
+            -m ./toolkit/docs/building/prerequisites-mariner.md \
+            -s ./toolkit/docs/building/prerequisites-src.json \
+            -d "azurelinux"

toolkit/docs/building/prerequisites-mariner.md

@@ -16,9 +16,9 @@ sudo tdnf -y install \
     diffutils \
     dosfstools \
     gawk \
-    glibc-devel \
     genisoimage \
     git \
+    glibc-devel \
     golang \
     jq \
     kernel-headers \

toolkit/docs/building/prerequisites-src.json

@@ -0,0 +1,133 @@
+{
+    "_comment": "This JSON file lists the prerequisites for building the source code in the CBL-Mariner toolkit.",
+    "_comment2": "check-prerequisites.yml and 'make install-azurelinux-prereqs' use this file as the canonical truth",
+    "_comment3": "The 'name' field is the name of the package to install, followed by the actual package name for each supported OS",
+    "packages": [
+        {
+            "name": "acl",
+            "azurelinux": "acl",
+            "ubuntu": "acl"
+        },
+        {
+            "name": "binutils",
+            "azurelinux": "binutils"
+        },
+        {
+            "name": "cdrkit",
+            "azurelinux": "cdrkit"
+        },
+        {
+            "name": "curl",
+            "azurelinux": "curl",
+            "ubuntu": "curl"
+        },
+        {
+            "name": "diffutils",
+            "azurelinux": "diffutils",
+            "ubuntu": "diffutils"
+        },
+        {
+            "name": "dosfstools",
+            "azurelinux": "dosfstools"
+        },
+        {
+            "name": "gawk",
+            "azurelinux": "gawk",
+            "ubuntu": "gawk"
+        },
+        {
+            "name": "glibc-devel",
+            "azurelinux": "glibc-devel"
+        },
+        {
+            "name": "genisoimage",
+            "azurelinux": "genisoimage",
+            "ubuntu": "genisoimage"
+        },
+        {
+            "name": "git",
+            "azurelinux": "git",
+            "ubuntu": "git"
+        },
+        {
+            "name": "golang",
+            "azurelinux": "golang",
+            "ubuntu": "golang-1.21-go"
+        },
+        {
+            "name": "jq",
+            "azurelinux": "jq",
+            "ubuntu": "jq"
+        },
+        {
+            "name": "kernel-headers",
+            "azurelinux": "kernel-headers"
+        },
+        {
+            "name": "make",
+            "azurelinux": "make",
+            "ubuntu": "make"
+        },
+        {
+            "name": "moby-cli",
+            "azurelinux": "moby-cli"
+        },
+        {
+            "name": "moby-engine",
+            "azurelinux": "moby-engine"
+        },
+        {
+            "name": "openssl",
+            "azurelinux": "openssl",
+            "ubuntu": "openssl"
+        },
+        {
+            "name": "parted",
+            "azurelinux": "parted",
+            "ubuntu": "parted"
+        },
+        {
+            "name": "pigz",
+            "azurelinux": "pigz",
+            "ubuntu": "pigz"
+        },
+        {
+            "name": "qemu-img",
+            "azurelinux": "qemu-img",
+            "ubuntu": "qemu-utils"
+        },
+        {
+            "name": "rpm",
+            "azurelinux": "rpm",
+            "ubuntu": "rpm"
+        },
+        {
+            "name": "rpm-build",
+            "azurelinux": "rpm-build"
+        },
+        {
+            "name": "sudo",
+            "azurelinux": "sudo"
+        },
+        {
+            "name": "tar",
+            "azurelinux": "tar",
+            "ubuntu": "tar"
+        },
+        {
+            "name": "wget",
+            "azurelinux": "wget",
+            "ubuntu": "wget"
+        },
+        {
+            "name": "xfsprogs",
+            "azurelinux": "xfsprogs",
+            "ubuntu": "xfsprogs"
+        },
+        {
+            "name": "zstd",
+            "azurelinux": "zstd",
+            "ubuntu": "zstd"
+        }
+    ]
+}

toolkit/docs/building/prerequisites-ubuntu.md

@@ -20,9 +20,9 @@ sudo apt -y install \
     golang-1.21-go \
     jq \
     make \
+    openssl \
     parted \
     pigz \
-    openssl \
     qemu-utils \
     rpm \
     tar \

toolkit/scripts/prerequisites.sh

@@ -0,0 +1,119 @@
+#!/bin/bash
+
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+set -e
+
+prereq_md_path=""
+prereq_json_path=""
+
+function print_help {
+    echo "Pre-requisite installer. Installs azurelinux pre-requisites for the toolkit."
+    echo "Usage:"
+    echo '[MANDATORY] -m PATH -> path to the pre-requisite md file'
+    echo '[MANDATORY] -s PATH -> path to the src json file'
+    echo '[MANDATORY] -d DISTRO -> One of "azurelinux" or "ubuntu"'
+    echo '[OPTIONAL]  -p -> print the canonical pre-requisite list and exit'
+    echo '[OPTIONAL]  -h -> print this help dialogue and exit'
+}
+
+do_print=false
+while getopts "hpm:s:d:" OPTIONS; do
+    case ${OPTIONS} in
+        m ) prereq_md_path=$OPTARG ;;
+        s ) prereq_json_path=$OPTARG ;;
+        d ) distro=$OPTARG ;;
+        p ) do_print="true" ;;
+        h ) print_help; exit 0 ;;
+        ? ) echo -e "ERROR: INVALID OPTION.\n\n" >&2; print_help; exit 1 ;;
+    esac
+done
+
+if [ -z "$prereq_json_path" ]; then
+    echo "ERROR: Missing src json file path." >&2
+    print_help
+    exit 1
+fi
+
+if [ ! -f "$prereq_json_path" ]; then
+    echo "ERROR: Src json file not found at $prereq_json_path." >&2
+    exit 1
+fi
+
+if [ -z "$distro" ]; then
+    echo "ERROR: Missing distro." >&2
+    print_help
+    exit 1
+fi
+
+if [ "$distro" != "azurelinux" ] && [ "$distro" != "ubuntu" ]; then
+    echo "ERROR: Invalid distro. Must be one of 'azurelinux' or 'ubuntu'." >&2
+    print_help
+    exit 1
+fi
+
+# Load the pre-requisite source json for the distro of interest
+# Json is of the form:
+# {
+#   "packages": [
+#       {
+#           "name": "pkg",
+#           "azurelinux": "pkg-b",
+#           "ubuntu": "pkg-a"
+#       },
+#       ...
+#   ]
+#}
+# A distro specific entry may not exist for every package, so get the entries that do exist, and
+# select the distro specific value. Convert the result to a space separated string.
+src_prereqs=$(jq -r --arg distro "$distro" '[.packages[] | select(has($distro)) | .[$distro]] | sort | join(" ")' "$prereq_json_path")
+
+# For print mode, just print the pre-requisites and exit early.
+if "$do_print"; then
+    echo "$src_prereqs"
+    exit 0
+fi
+
+if [ -z "$prereq_md_path" ]; then
+    echo "ERROR: Missing pre-requisite file path." >&2
+    print_help
+    exit 1
+fi
+
+if [ ! -f "$prereq_md_path" ]; then
+    echo "ERROR: Pre-requisite file not found at $prereq_md_path." >&2
+    exit 1
+fi
+
+# Get the current entry in the .md file:
+# 1) Remove all line continuations (backslash followed by newline) and replace with a space
+current_prereqs="$(sed -e ':a' -e 'N' -e '$!ba' -e 's/\\\n/ /g' "$prereq_md_path")"
+
+# 2) Remove all leading and trailing whitespace
+current_prereqs=$(echo "$current_prereqs" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')
+
+# 3) Find the line that starts with "sudo <pkg manager> -y install ..."
+if [[ "$distro" == "azurelinux" ]]; then
+    current_prereqs=$(echo "$current_prereqs" | grep -oP 'sudo tdnf -y install \K.*')
+elif [[ "$distro" == "ubuntu" ]]; then
+    current_prereqs=$(echo "$current_prereqs" | grep -oP 'sudo apt -y install \K.*')
+fi
+
+# 4) Grab the package list from the line
+current_prereqs=$(echo "$current_prereqs" | xargs)
+
+# Compare the current and source pre-requisites. Enforce a 1:1 match including order for consistency.
+if [ "$current_prereqs" != "$src_prereqs" ]; then
+    echo "ERROR: Pre-requisites do not match. Please update the pre-requisite file."
+    echo ""
+    echo "Comparing: $prereq_json_path"
+    echo "with:      $prereq_md_path"
+    echo ""
+    echo ".json file:   $src_prereqs"
+    echo ".md file:     $current_prereqs"
+    exit 1
+else
+    echo "Pre-requisites match."
+    exit 0
+fi

toolkit/scripts/toolkit.mk

@@ -127,3 +127,9 @@ $(valid_arch_spec_names): $(go-specarchchecker) $(chroot_worker) $(local_specs)
 		--log-level=$(LOG_LEVEL) \
 		--log-file="$(valid_arch_spec_names_logs_path)" \
 		--log-color="$(LOG_COLOR)"
+
+##help:target:install-azurelinux-prereqs=Install Azure Linux Prerequisites
+install-azurelinux-prereqs:
+	@echo "Installing Azure Linux Prerequisites"
+	prereqs="$$( $(SCRIPTS_DIR)/prerequisites.sh -s $(toolkit_root)/docs/building/prerequisites-src.json -d azurelinux -p )" && \
+	tdnf -y install $$prereqs