From 2b34eda7849a99cedbfac55d8ae026cf886f7715 Mon Sep 17 00:00:00 2001 From: John Lee Date: Fri, 26 Jul 2024 10:37:30 -0400 Subject: [PATCH] CI: update to CentOS Stream 9 and start integrating universal target (#6080) --- docker/build/cicd.centos.Dockerfile | 28 ------------------- ...os8.Dockerfile => cicd.centos9.Dockerfile} | 11 ++------ package-deploy.yaml | 4 +-- package-test.yaml | 4 +-- package.yaml | 4 +-- scripts/release/README.md | 8 +++--- scripts/release/build/stage/build/task.sh | 4 +-- .../release/common/docker/centos.Dockerfile | 8 ------ ...{centos8.Dockerfile => centos9.Dockerfile} | 5 ++-- scripts/release/common/setup.sh | 3 +- ...ntos8_image.sh => ensure_centos9_image.sh} | 6 ++-- scripts/release/mule/package/rpm/package.sh | 2 +- scripts/release/prod/rpm/run_centos.sh | 4 +-- scripts/release/test/rpm/run_centos.sh | 4 +-- scripts/release/test/util/test_package.sh | 6 ++-- test/muleCI/mule.yaml | 24 ++++------------ .../test_linux_amd64_compatibility.sh | 6 ++-- 17 files changed, 38 insertions(+), 93 deletions(-) delete mode 100644 docker/build/cicd.centos.Dockerfile rename docker/build/{cicd.centos8.Dockerfile => cicd.centos9.Dockerfile} (79%) delete mode 100644 scripts/release/common/docker/centos.Dockerfile rename scripts/release/common/docker/{centos8.Dockerfile => centos9.Dockerfile} (77%) rename scripts/release/mule/common/{ensure_centos8_image.sh => ensure_centos9_image.sh} (69%) diff --git a/docker/build/cicd.centos.Dockerfile b/docker/build/cicd.centos.Dockerfile deleted file mode 100644 index f292e3d220..0000000000 --- a/docker/build/cicd.centos.Dockerfile +++ /dev/null @@ -1,28 +0,0 @@ -ARG ARCH="amd64" - -FROM ${ARCH}/centos:7 -ARG GOLANG_VERSION -ARG ARCH="amd64" -RUN yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm && \ - yum update -y && \ - yum install -y autoconf wget awscli git gnupg2 nfs-utils python3-devel expect jq \ - libtool gcc-c++ libstdc++-devel libstdc++-static rpmdevtools createrepo rpm-sign bzip2 which ShellCheck \ - libffi-devel openssl-devel -WORKDIR /root -RUN wget https://dl.google.com/go/go${GOLANG_VERSION}.linux-${ARCH%v*}.tar.gz \ - && tar -xvf go${GOLANG_VERSION}.linux-${ARCH%v*}.tar.gz && \ - mv go /usr/local -ENV GOROOT=/usr/local/go \ - GOPATH=$HOME/go \ - ARCH_TYPE=${ARCH} -RUN mkdir -p $GOPATH/src/github.com/algorand -COPY . $GOPATH/src/github.com/algorand/go-algorand -ENV PATH=$GOPATH/bin:$GOROOT/bin:$PATH \ - GOPROXY=https://proxy.golang.org,https://pkg.go.dev,https://goproxy.io,direct -WORKDIR $GOPATH/src/github.com/algorand/go-algorand -RUN git config --global --add safe.directory '*' -RUN make clean -RUN rm -rf $GOPATH/src/github.com/algorand/go-algorand && \ - mkdir -p $GOPATH/src/github.com/algorand/go-algorand -RUN echo "vm.max_map_count = 262144" >> /etc/sysctl.conf -CMD ["/bin/bash"] diff --git a/docker/build/cicd.centos8.Dockerfile b/docker/build/cicd.centos9.Dockerfile similarity index 79% rename from docker/build/cicd.centos8.Dockerfile rename to docker/build/cicd.centos9.Dockerfile index 76ec3e9cc5..e0d53e467e 100644 --- a/docker/build/cicd.centos8.Dockerfile +++ b/docker/build/cicd.centos9.Dockerfile @@ -1,18 +1,13 @@ ARG ARCH="amd64" -FROM quay.io/centos/centos:stream8 +FROM quay.io/centos/centos:stream9 ARG GOLANG_VERSION ARG ARCH="amd64" -RUN dnf update rpm -y && \ - dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \ +RUN dnf install -y epel-release epel-next-release && dnf config-manager --set-enabled crb && \ dnf update -y && \ dnf install -y autoconf wget awscli git gnupg2 nfs-utils python3-devel expect jq \ libtool gcc-c++ libstdc++-devel rpmdevtools createrepo rpm-sign bzip2 which \ - libffi-devel openssl-devel -RUN dnf install -y epel-release && \ - dnf update && \ - dnf -y --enablerepo=powertools install libstdc++-static && \ - dnf -y install make + libffi-devel openssl-devel libstdc++-static RUN echo "${BOLD}Downloading and installing binaries...${RESET}" && \ curl -Of https://shellcheck.storage.googleapis.com/shellcheck-v0.7.0.linux.x86_64.tar.xz && \ tar -C /usr/local/bin/ -xf shellcheck-v0.7.0.linux.x86_64.tar.xz --no-anchored 'shellcheck' --strip=1 diff --git a/package-deploy.yaml b/package-deploy.yaml index 9b67a2fe6c..b4da6c2eec 100644 --- a/package-deploy.yaml +++ b/package-deploy.yaml @@ -45,8 +45,8 @@ agents: workDir: $HOME/projects/go-algorand - name: rpm - dockerFilePath: docker/build/cicd.centos8.Dockerfile - image: algorand/go-algorand-ci-linux-centos8 + dockerFilePath: docker/build/cicd.centos9.Dockerfile + image: algorand/go-algorand-ci-linux-centos9 version: scripts/configure_dev-deps.sh buildArgs: - GOLANG_VERSION=`./scripts/get_golang_version.sh` diff --git a/package-test.yaml b/package-test.yaml index e338197ff1..cd526dda68 100644 --- a/package-test.yaml +++ b/package-test.yaml @@ -16,8 +16,8 @@ agents: workDir: $HOME/projects/go-algorand - name: rpm - dockerFilePath: docker/build/cicd.centos.Dockerfile - image: algorand/mule-linux-centos + dockerFilePath: docker/build/cicd.centos9.Dockerfile + image: algorand/mule-linux-centos9 version: scripts/configure_dev-deps.sh buildArgs: - GOLANG_VERSION=`./scripts/get_golang_version.sh` diff --git a/package.yaml b/package.yaml index d5d8de0d97..0cc1d588eb 100644 --- a/package.yaml +++ b/package.yaml @@ -11,8 +11,8 @@ agents: workDir: $HOME/projects/go-algorand - name: rpm - dockerFilePath: docker/build/cicd.centos.Dockerfile - image: algorand/go-algorand-ci-linux-centos + dockerFilePath: docker/build/cicd.centos9.Dockerfile + image: algorand/go-algorand-ci-linux-centos9 version: scripts/configure_dev-deps.sh buildArgs: - GOLANG_VERSION=`./scripts/get_golang_version.sh` diff --git a/scripts/release/README.md b/scripts/release/README.md index f3fa543f13..534bc70faa 100644 --- a/scripts/release/README.md +++ b/scripts/release/README.md @@ -50,7 +50,7 @@ This section briefly describes the expected outcomes of the current build pipeli 1. build - 1. Build (compile) the binaries in a Centos 7 & 8 docker container that will then be used by both `deb` and `rpm` packaging. + 1. Build (compile) the binaries in a Centos 9 docker container that will then be used by both `deb` and `rpm` packaging. 1. Docker containers will package `deb` and `rpm` artifacts inside of Ubuntu 20.04 and Centos 7 & 8, respectively. @@ -69,9 +69,9 @@ This section briefly describes the expected outcomes of the current build pipeli - The signatures are correct. - The packages are built from the correct branch and channel and are the correct version. This done by running `algod -v`. + This is done for the following docker containers: - - centos:7 - - quay.io/centos/centos:stream8 - - fedora:38 + - quay.io/centos/centos:stream9 + - fedora:39 + - fedora:40 - ubuntu:20.04 - ubuntu:22.04 diff --git a/scripts/release/build/stage/build/task.sh b/scripts/release/build/stage/build/task.sh index fdb96d6b27..944aeb7b34 100755 --- a/scripts/release/build/stage/build/task.sh +++ b/scripts/release/build/stage/build/task.sh @@ -30,8 +30,8 @@ else echo ${BUILD_NUMBER} > "${REPO_ROOT}"/buildnumber.dat fi -# Run RPM build in Centos 7 & 8 Docker container -sg docker "docker build -t algocentosbuild - < $HOME/go/src/github.com/algorand/go-algorand/scripts/release/common/docker/centos.Dockerfile" +# Run RPM build in Centos 9 Docker container +sg docker "docker build -t algocentosbuild - < $HOME/go/src/github.com/algorand/go-algorand/scripts/release/common/docker/centos9.Dockerfile" sg docker "docker run --rm --env-file ${HOME}/build_env_docker --mount type=bind,src=${HOME},dst=/root/subhome algocentosbuild /root/subhome/go/src/github.com/algorand/go-algorand/scripts/release/build/rpm/build.sh" echo diff --git a/scripts/release/common/docker/centos.Dockerfile b/scripts/release/common/docker/centos.Dockerfile deleted file mode 100644 index a23b446ca1..0000000000 --- a/scripts/release/common/docker/centos.Dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -FROM centos:7 - -WORKDIR /root -RUN yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm -RUN yum install -y autoconf awscli curl git gnupg2 nfs-utils python36 expect jq libtool gcc-c++ libstdc++-devel libstdc++-static rpmdevtools createrepo rpm-sign bzip2 which ShellCheck - -ENTRYPOINT ["/bin/bash"] - diff --git a/scripts/release/common/docker/centos8.Dockerfile b/scripts/release/common/docker/centos9.Dockerfile similarity index 77% rename from scripts/release/common/docker/centos8.Dockerfile rename to scripts/release/common/docker/centos9.Dockerfile index cf5474cfe7..1151201edb 100644 --- a/scripts/release/common/docker/centos8.Dockerfile +++ b/scripts/release/common/docker/centos9.Dockerfile @@ -1,7 +1,8 @@ -FROM quay.io/centos/centos:stream8 +FROM quay.io/centos/centos:stream9 WORKDIR /root -RUN dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \ +RUN dnf install -y epel-release epel-next-release && dnf config-manager --set-enabled crb && \ + dnf update -y && \ dnf install -y autoconf awscli curl git gnupg2 nfs-utils python36 expect jq libtool gcc-c++ libstdc++-devel rpmdevtools createrepo rpm-sign bzip2 which && \ dnf -y --enablerepo=powertools install libstdc++-static diff --git a/scripts/release/common/setup.sh b/scripts/release/common/setup.sh index 75683262d5..6124e0b95f 100755 --- a/scripts/release/common/setup.sh +++ b/scripts/release/common/setup.sh @@ -104,8 +104,7 @@ else fi sudo usermod -a -G docker ubuntu -sg docker "docker pull centos:7" -sg docker "docker pull quay.io/centos/centos:stream8" +sg docker "docker pull quay.io/centos/centos:stream9" sg docker "docker pull ubuntu:22.04" cat << EOF >> "${HOME}/.bashrc" diff --git a/scripts/release/mule/common/ensure_centos8_image.sh b/scripts/release/mule/common/ensure_centos9_image.sh similarity index 69% rename from scripts/release/mule/common/ensure_centos8_image.sh rename to scripts/release/mule/common/ensure_centos9_image.sh index 1ebd3475fe..bb03624c1b 100755 --- a/scripts/release/mule/common/ensure_centos8_image.sh +++ b/scripts/release/mule/common/ensure_centos9_image.sh @@ -2,9 +2,9 @@ set -exo pipefail -# Ensure the centos8 docker image is built and available +# Ensure the centos docker image is built and available -DOCKER_IMAGE="algorand/go-algorand-ci-linux-centos8:amd64-$(sha1sum scripts/configure_dev-deps.sh | cut -f1 -d' ')" +DOCKER_IMAGE="algorand/go-algorand-ci-linux-centos9:amd64-$(sha1sum scripts/configure_dev-deps.sh | cut -f1 -d' ')" MATCH=${DOCKER_IMAGE/:*/} echo "Checking for RPM image" @@ -13,5 +13,5 @@ if docker images $DOCKER_IMAGE | grep -qs $MATCH > /dev/null 2>&1; then else echo "RPM image doesn't exist, building" docker build --platform=linux/amd64 --build-arg ARCH=amd64 \ - --build-arg GOLANG_VERSION=$(./scripts/get_golang_version.sh) -t $DOCKER_IMAGE -f docker/build/cicd.centos8.Dockerfile . + --build-arg GOLANG_VERSION=$(./scripts/get_golang_version.sh) -t $DOCKER_IMAGE -f docker/build/cicd.centos9.Dockerfile . fi diff --git a/scripts/release/mule/package/rpm/package.sh b/scripts/release/mule/package/rpm/package.sh index f3e3dde17f..7d63872f5c 100755 --- a/scripts/release/mule/package/rpm/package.sh +++ b/scripts/release/mule/package/rpm/package.sh @@ -50,7 +50,7 @@ find tmp/node_pkgs -name "*${CHANNEL}*linux*${VERSION}*.tar.gz" | cut -d '/' -f3 -e "s,@REQUIRED_ALGORAND_PKG@,$REQUIRED_ALGORAND_PACKAGE," \ > "$TEMPDIR/$ALGORAND_PACKAGE_NAME.spec" - rpmbuild --buildroot "$HOME/foo" --define "_rpmdir $RPMTMP" --define "RELEASE_GENESIS_PROCESS xtrue" --define "LICENSE_FILE ./COPYING" -bb "$TEMPDIR/$ALGORAND_PACKAGE_NAME.spec" --target $ARCH_UNAME + rpmbuild --buildroot "$HOME/foo" --define "_rpmdir $RPMTMP" --define "RELEASE_GENESIS_PROCESS \"xtrue\"" --define "LICENSE_FILE ./COPYING" -bb "$TEMPDIR/$ALGORAND_PACKAGE_NAME.spec" --target $ARCH_UNAME cp -p "$RPMTMP"/*/*.rpm "./tmp/node_pkgs/$OS_TYPE/$ARCH_TYPE" echo "${RPMTMP}" diff --git a/scripts/release/prod/rpm/run_centos.sh b/scripts/release/prod/rpm/run_centos.sh index 9426da5ea1..abb2b73f9e 100755 --- a/scripts/release/prod/rpm/run_centos.sh +++ b/scripts/release/prod/rpm/run_centos.sh @@ -5,8 +5,8 @@ set -ex . "${HOME}"/build_env -# Run RPM build in Centos 7 & 8 Docker container -sg docker "docker build -t algocentosbuild - < ${HOME}/go/src/github.com/algorand/go-algorand/scripts/release/common/docker/centos.Dockerfile" +# Run RPM build in Centos 9 Docker container +sg docker "docker build -t algocentosbuild - < ${HOME}/go/src/github.com/algorand/go-algorand/scripts/release/common/docker/centos9.Dockerfile" sg docker "docker run --rm --env-file ${HOME}/build_env_docker --mount type=bind,src=/run/user/1000/gnupg/S.gpg-agent,dst=/root/S.gpg-agent --mount type=bind,src=${HOME}/prodrepo,dst=/root/prodrepo --mount type=bind,src=${HOME}/keys,dst=/root/keys --mount type=bind,src=${HOME},dst=/root/subhome algocentosbuild /root/subhome/go/src/github.com/algorand/go-algorand/scripts/release/prod/rpm/snapshot.sh" diff --git a/scripts/release/test/rpm/run_centos.sh b/scripts/release/test/rpm/run_centos.sh index 88e0a6be4c..d206a085d7 100755 --- a/scripts/release/test/rpm/run_centos.sh +++ b/scripts/release/test/rpm/run_centos.sh @@ -14,8 +14,8 @@ if [ "$CHANNEL" = beta ]; then exit 0 fi -# Run RPM build in Centos 7 & 8 Docker container -sg docker "docker build -t algocentosbuild - < ${HOME}/go/src/github.com/algorand/go-algorand/scripts/release/common/docker/centos.Dockerfile" +# Run RPM build in Centos 9 Docker container +sg docker "docker build -t algocentosbuild - < ${HOME}/go/src/github.com/algorand/go-algorand/scripts/release/common/docker/centos9.Dockerfile" cat <"${HOME}"/dummyrepo/algodummy.repo [algodummy] diff --git a/scripts/release/test/util/test_package.sh b/scripts/release/test/util/test_package.sh index 61c93b84b4..c8dd206c8c 100755 --- a/scripts/release/test/util/test_package.sh +++ b/scripts/release/test/util/test_package.sh @@ -8,9 +8,9 @@ set -ex . "${HOME}"/build_env OS_LIST=( - centos:7 - quay.io/centos/centos:stream8 - fedora:38 + quay.io/centos/centos:stream9 + fedora:39 + fedora:40 ubuntu:20.04 ubuntu:22.04 ) diff --git a/test/muleCI/mule.yaml b/test/muleCI/mule.yaml index 5022cba792..e1ce86b9f8 100644 --- a/test/muleCI/mule.yaml +++ b/test/muleCI/mule.yaml @@ -15,23 +15,9 @@ agents: - GOLANG_VERSION=`./scripts/get_golang_version.sh` - ARCH=amd64 - GOARCH=amd64 - - name: cicd.centos.amd64 - dockerFilePath: docker/build/cicd.centos.Dockerfile - image: algorand/go-algorand-ci-linux-centos - version: scripts/configure_dev-deps.sh - arch: amd64 - env: - - TRAVIS_BRANCH=${GIT_BRANCH} - - NETWORK=$NETWORK - - VERSION=$VERSION - - BUILD_NUMBER=$BUILD_NUMBER - - GOHOSTARCH=amd64 - buildArgs: - - GOLANG_VERSION=`./scripts/get_golang_version.sh` - - ARCH=amd64 - - name: cicd.centos8.amd64 - dockerFilePath: docker/build/cicd.centos8.Dockerfile - image: algorand/go-algorand-ci-linux-centos8 + - name: cicd.centos9.amd64 + dockerFilePath: docker/build/cicd.centos9.Dockerfile + image: algorand/go-algorand-ci-linux-centos9 version: scripts/configure_dev-deps.sh arch: amd64 env: @@ -106,12 +92,12 @@ tasks: - task: docker.Make name: archive - agent: cicd.centos8.amd64 + agent: cicd.centos9.amd64 target: archive - task: docker.Make name: rpm.amd64 - agent: cicd.centos.amd64 + agent: cicd.centos9.amd64 target: mule-package-rpm - task: docker.Make diff --git a/test/platform/test_linux_amd64_compatibility.sh b/test/platform/test_linux_amd64_compatibility.sh index 7e91c8728e..2ab8d4c990 100755 --- a/test/platform/test_linux_amd64_compatibility.sh +++ b/test/platform/test_linux_amd64_compatibility.sh @@ -7,9 +7,9 @@ BLUE_FG=$(tput setaf 4 2>/dev/null) END_FG_COLOR=$(tput sgr0 2>/dev/null) OS_LIST=( - centos:7 - quay.io/centos/centos:stream8 - fedora:38 + quay.io/centos/centos:stream9 + fedora:39 + fedora:40 ubuntu:20.04 ubuntu:22.04 )