From 4c85516de8e3107aff2c288c08fd30b3b2688bbe Mon Sep 17 00:00:00 2001 From: Ihor Solodrai Date: Thu, 24 Oct 2024 15:43:04 -0700 Subject: [PATCH] Factor out tar-artifacts action (#148) Rework tar-artifact.sh script, making it suitable for use as an independent github action. Use tar -rf (append) to accumulate files of interest before compressing with zstd. Control groups of artifacts to include (e.g. selftests/bpf, selftests/sched_ext) with environment variables. --- .github/scripts/tar-artifact.sh | 107 ----------------------------- .github/workflows/kernel-build.yml | 26 ++++--- tar-artifacts/README.md | 34 +++++++++ tar-artifacts/action.yml | 26 +++++++ tar-artifacts/tar-artifacts.sh | 79 +++++++++++++++++++++ 5 files changed, 156 insertions(+), 116 deletions(-) delete mode 100644 .github/scripts/tar-artifact.sh create mode 100644 tar-artifacts/README.md create mode 100644 tar-artifacts/action.yml create mode 100755 tar-artifacts/tar-artifacts.sh diff --git a/.github/scripts/tar-artifact.sh b/.github/scripts/tar-artifact.sh deleted file mode 100644 index 549acf9..0000000 --- a/.github/scripts/tar-artifact.sh +++ /dev/null @@ -1,107 +0,0 @@ -#!/bin/bash - -set -eux - -# Assumptions: -# - $(pwd) is the root of kernel repo we're tarring -# - zstd is installed by default in the runner images - -if [ ! -d "${KBUILD_OUTPUT:-}" ]; then - echo "KBUILD_OUTPUT must be a directory" - exit 1 -fi - -arch="${1}" -toolchain="${2}" - -# Convert a platform (as returned by uname -m) to the kernel -# arch (as expected by ARCH= env). -platform_to_kernel_arch() { - case $1 in - s390x) - echo "s390" - ;; - aarch64) - echo "arm64" - ;; - riscv64) - echo "riscv" - ;; - x86_64) - echo "x86" - ;; - *) - echo "$1" - ;; - esac -} - -# Remove intermediate object files that we have no use for. Ideally -# we'd just exclude them from tar below, but it does not provide -# options to express the precise constraints. -find tools/testing/selftests/ -name "*.o" -a ! -name "*.bpf.o" -print0 | \ - xargs --null --max-args=10000 rm - -# Strip debug information, which is excessively large (consuming -# bandwidth) while not actually being used (the kernel does not use -# DWARF to symbolize stacktraces). -"${arch}"-linux-gnu-strip --strip-debug "${KBUILD_OUTPUT}"/vmlinux - -image_name=$(make ARCH="$(platform_to_kernel_arch "${arch}")" -s image_name) -kbuild_output_file_list=( - ".config" - "${image_name}" - "include/config/auto.conf" - "include/generated/autoconf.h" - "vmlinux" -) - -# While we are preparing the tarball, move $KBUILD_OUTPUT to a tmp -# location just in case it's inside the repo root -tmp=$(mktemp -d) -mv "${KBUILD_OUTPUT}" "${tmp}" -stashed_kbuild_output=${tmp}/$(basename "${KBUILD_OUTPUT}") - -# Note: ${local_kbuild_output} must point to ./kbuild-output because -# of the tar command at the bottom. -local_kbuild_output=$(realpath kbuild-output) -mkdir -p "${local_kbuild_output}" - -for file in "${kbuild_output_file_list[@]}"; do - mkdir -p "$(dirname "${local_kbuild_output}/${file}")" - cp -a "${stashed_kbuild_output}/${file}" "${local_kbuild_output}/${file}" -done - -additional_file_list=() -if [[ -n "${ARCHIVE_MAKE_HELPERS}" ]]; then - # Package up a bunch of additional infrastructure to support running - # 'make kernelrelease' and bpf tool checks later on. - mapfile -t additional_file_list < <(find . -iname Makefile) - additional_file_list+=( - "scripts/" - "tools/testing/selftests/bpf/" - "tools/include/" - "tools/bpf/bpftool/" - ) -fi - -mkdir -p selftests -cp -r tools/testing/selftests/bpf selftests -if [[ -n "${BUILD_SCHED_EXT_SELFTESTS}" ]]; then - cp -r tools/testing/selftests/sched_ext selftests -fi - -tar -cf - \ - kbuild-output \ - "${additional_file_list[@]}" \ - --exclude '*.cmd' \ - --exclude '*.d' \ - --exclude '*.h' \ - --exclude '*.output' \ - selftests/ \ - | zstd -T0 -19 -o "vmlinux-${arch}-${toolchain}.tar.zst" - -# Cleanup and restore the original KBUILD_OUTPUT -# We have to put KBUILD_OUTPUT back to its original location for actions/cache -rm -rf "${local_kbuild_output}" -mv "${stashed_kbuild_output}" "${KBUILD_OUTPUT}" diff --git a/.github/workflows/kernel-build.yml b/.github/workflows/kernel-build.yml index 35c22b3..071e9ff 100644 --- a/.github/workflows/kernel-build.yml +++ b/.github/workflows/kernel-build.yml @@ -45,17 +45,17 @@ jobs: runs-on: ${{ fromJSON(inputs.runs_on) }} timeout-minutes: 100 env: - ARCHIVE_MAKE_HELPERS: ${{ github.repository != 'kernel-patches/bpf' && 'true' || '' }} - KERNEL: ${{ inputs.kernel }} - REPO_ROOT: ${{ github.workspace }} - REPO_PATH: "" - KBUILD_OUTPUT: ${{ github.workspace }}/kbuild-output + ARTIFACTS_ARCHIVE: "vmlinux-${{ inputs.arch }}-${{ inputs.toolchain_full }}.tar.zst" BASE_BRANCH: >- ${{ github.event_name == 'push' && github.ref_name || github.base_ref || 'bpf-next' }} BUILD_SCHED_EXT_SELFTESTS: ${{ inputs.arch == 'x86_64' || inputs.arch == 'aarch64' && 'true' || '' }} + KBUILD_OUTPUT: ${{ github.workspace }}/kbuild-output + KERNEL: ${{ inputs.kernel }} + REPO_PATH: "" + REPO_ROOT: ${{ github.workspace }} steps: - uses: actions/checkout@v4 # We fetch an actual bit of history here to facilitate incremental @@ -140,9 +140,17 @@ jobs: max-make-jobs: 32 llvm-version: ${{ inputs.llvm-version }} - name: Tar artifacts - working-directory: ${{ env.REPO_ROOT }} - run: | - bash .github/scripts/tar-artifact.sh ${{ inputs.arch }} ${{ inputs.toolchain_full }} + id: tar-artifacts + uses: ./tar-artifacts + env: + ARCHIVE_BPF_SELFTESTS: 'true' + ARCHIVE_MAKE_HELPERS: ${{ github.repository != 'kernel-patches/bpf' && 'true' || '' }} + ARCHIVE_SCHED_EXT_SELFTESTS: ${{ env.BUILD_SCHED_EXT_SELFTESTS }} + with: + arch: ${{ inputs.arch }} + archive: ${{ env.ARTIFACTS_ARCHIVE }} + kbuild-output: ${{ env.KBUILD_OUTPUT }} + repo-root: ${{ env.REPO_ROOT }} - if: ${{ github.event_name != 'push' }} name: Remove KBUILD_OUTPUT content shell: bash @@ -155,4 +163,4 @@ jobs: with: name: vmlinux-${{ inputs.arch }}-${{ inputs.toolchain_full }}${{ inputs.release && '-release' || '' }} if-no-files-found: error - path: vmlinux-${{ inputs.arch }}-${{ inputs.toolchain_full }}.tar.zst + path: ${{ env.ARTIFACTS_ARCHIVE }} diff --git a/tar-artifacts/README.md b/tar-artifacts/README.md new file mode 100644 index 0000000..baa6f09 --- /dev/null +++ b/tar-artifacts/README.md @@ -0,0 +1,34 @@ +# Tar build artifacts + +This action creates a tarball with kbuild-output and other build +artifacts necessary to run the selftests. + +The action is expected to be executed by a workflow with access to the +Linux kernel repository. + +## Required inputs + +* `arch` - Kernel build architecture, required to find image_name +* `archive` - path to the produced .zst archive +* `kbuild-output` - Path to the kernel build output +* `repo-root` - Path to the root of the Linux kernel repository + +# Archive options + +Essential content of the directory passed via `kbuild-output` input is +always included in the tarball. + +For selftests artifacts the script checks environment variables to +determine what to include. These are handled as bash flags: +emptystring means false, any other value means true. + +* `ARCHIVE_BPF_SELFTESTS` - add `tools/testing/selftests/bpf` binaries + under `selftests/bpf` in the tarball +* `ARCHIVE_MAKE_HELPERS` - add all the Linux repo makefiles and other + scripts +* `ARCHIVE_SCHED_EXT_SELFTESTS` - add + `tools/testing/selftests/sched_ext` binaries under + `selftests/sched_ext` in the tarball + + + diff --git a/tar-artifacts/action.yml b/tar-artifacts/action.yml new file mode 100644 index 0000000..373adf5 --- /dev/null +++ b/tar-artifacts/action.yml @@ -0,0 +1,26 @@ +name: 'Tar build artifacts' +inputs: + arch: + description: 'Target arch of the kernel, required for finding the image' + required: true + archive: + description: 'Path to the output archive' + required: true + kbuild-output: + description: 'Path to the kernel build output for archiving' + required: true + repo-root: + description: "Path to the root of the kernel repository" + required: true + +runs: + using: "composite" + steps: + - name: Run tar-artifacts.sh + env: + KBUILD_OUTPUT: ${{ inputs.kbuild-output }} + REPO_ROOT: ${{ inputs.repo-root }} + ARCH: ${{ inputs.arch }} + shell: bash + run: + ${GITHUB_ACTION_PATH}/tar-artifacts.sh ${{ inputs.archive }} diff --git a/tar-artifacts/tar-artifacts.sh b/tar-artifacts/tar-artifacts.sh new file mode 100755 index 0000000..ad10150 --- /dev/null +++ b/tar-artifacts/tar-artifacts.sh @@ -0,0 +1,79 @@ +#!/bin/bash + +set -eux -o pipefail + +if [ ! -d "${REPO_ROOT:-}" ]; then + echo "REPO_ROOT must be a directory: ${REPO_ROOT}" + exit 1 +fi + +if [ ! -d "${KBUILD_OUTPUT:-}" ]; then + echo "KBUILD_OUTPUT must be a directory: ${KBUILD_OUTPUT}" + exit 1 +fi + +zst_tarball="$1" +arch="${ARCH}" + +ARCHIVE_BPF_SELFTESTS="${ARCHIVE_BPF_SELFTESTS:-true}" +ARCHIVE_MAKE_HELPERS="${ARCHIVE_MAKE_HELPERS:-}" +ARCHIVE_SCHED_EXT_SELFTESTS="${ARCHIVE_SCHED_EXT_SELFTESTS:-}" + +tarball=$(mktemp ./artifacts.XXXXXXXX.tar) + +source "${GITHUB_ACTION_PATH}/../helpers.sh" + +# Strip debug information, which is excessively large (consuming +# bandwidth) while not actually being used (the kernel does not use +# DWARF to symbolize stacktraces). +"${arch}"-linux-gnu-strip --strip-debug "${KBUILD_OUTPUT}"/vmlinux + +image_name=$(make -C ${REPO_ROOT} ARCH="$(platform_to_kernel_arch "${arch}")" -s image_name) +kbuild_output_file_list=( + ".config" + "${image_name}" + "include/config/auto.conf" + "include/generated/autoconf.h" + "vmlinux" +) + +tar -rf "${tarball}" -C "${KBUILD_OUTPUT}" \ + --transform "s,^,kbuild-output/," \ + "${kbuild_output_file_list[@]}" + +# In case artifacts are restored not to the kernel repo root, +# package up a bunch of additional infrastructure to support running +# 'make kernelrelease' and bpf tool checks later on. +if [[ -n "${ARCHIVE_MAKE_HELPERS}" ]]; then + find "${REPO_ROOT}" -iname Makefile -printf '%P\n' \ + | tar -rf "${tarball}" -C "${REPO_ROOT}" -T - + tar -rf "${tarball}" -C "${REPO_ROOT}" \ + --exclude '*.o' \ + --exclude '*.d' \ + "scripts/" \ + "tools/testing/selftests/bpf/" \ + "tools/include/" \ + "tools/bpf/bpftool/" +fi + +if [[ -n "${ARCHIVE_BPF_SELFTESTS}" ]]; then + # add .bpf.o files + find "${REPO_ROOT}/tools/testing/selftests/bpf" \ + -name "*.bpf.o" -printf 'selftests/bpf/%P\n' \ + | tar -rf "${tarball}" -C "${REPO_ROOT}/tools/testing" -T - + # add other relevant files + tar -rf "${tarball}" -C "${REPO_ROOT}/tools/testing" \ + --exclude '*.cmd' \ + --exclude '*.d' \ + --exclude '*.h' \ + --exclude '*.o' \ + --exclude '*.output' \ + selftests/bpf/ +fi + +if [[ -n "${ARCHIVE_SCHED_EXT_SELFTESTS}" ]]; then + tar -rf "${tarball}" -C "${REPO_ROOT}/tools/testing" selftests/sched_ext/ +fi + +zstd -T0 -19 -i "${tarball}" -o "${zst_tarball}" +