From 83f294c22370974b3fcf3bcff758ddff264432ac Mon Sep 17 00:00:00 2001 From: everpcpc Date: Tue, 27 Feb 2024 16:39:31 +0800 Subject: [PATCH] chore: change build image to debian/alpine (#14755) --- .github/actions/build_linux/action.yml | 7 +++ .../actions/build_linux_sanitizer/action.yml | 7 +++ .github/workflows/build_tool.yml | 58 +++---------------- .github/workflows/reuse.linux.yml | 5 +- docker/build-tool/alpine/Dockerfile | 30 ++++++++++ docker/build-tool/{base => debian}/Dockerfile | 0 docker/build-tool/gnu/Dockerfile | 15 ----- docker/build-tool/musl/Dockerfile | 37 ------------ docker/build-tool/musl/linker.sh | 18 ------ scripts/setup/dev_setup.sh | 6 +- 10 files changed, 58 insertions(+), 125 deletions(-) create mode 100644 docker/build-tool/alpine/Dockerfile rename docker/build-tool/{base => debian}/Dockerfile (100%) delete mode 100644 docker/build-tool/gnu/Dockerfile delete mode 100644 docker/build-tool/musl/Dockerfile delete mode 100644 docker/build-tool/musl/linker.sh diff --git a/.github/actions/build_linux/action.yml b/.github/actions/build_linux/action.yml index 1878382a85d4..3465052be8c1 100644 --- a/.github/actions/build_linux/action.yml +++ b/.github/actions/build_linux/action.yml @@ -27,6 +27,13 @@ runs: using: "composite" steps: - name: Setup Build Tool + if: endsWith(inputs.target, '-linux-gnu') + uses: ./.github/actions/setup_build_tool + with: + target: base + bypass_env_vars: RUSTFLAGS,RUST_LOG + - name: Setup Build Tool + if: endsWith(inputs.target, '-linux-musl') uses: ./.github/actions/setup_build_tool with: target: ${{ inputs.target }} diff --git a/.github/actions/build_linux_sanitizer/action.yml b/.github/actions/build_linux_sanitizer/action.yml index a13868d2d056..b24cd1a1c339 100644 --- a/.github/actions/build_linux_sanitizer/action.yml +++ b/.github/actions/build_linux_sanitizer/action.yml @@ -16,6 +16,13 @@ runs: using: "composite" steps: - name: Setup Build Tool + if: endsWith(inputs.target, '-linux-gnu') + uses: ./.github/actions/setup_build_tool + with: + target: base + bypass_env_vars: RUSTFLAGS,RUST_LOG + - name: Setup Build Tool + if: endsWith(inputs.target, '-linux-musl') uses: ./.github/actions/setup_build_tool with: target: ${{ inputs.target }} diff --git a/.github/workflows/build_tool.yml b/.github/workflows/build_tool.yml index 98ac7342b159..3d438c900d3e 100644 --- a/.github/workflows/build_tool.yml +++ b/.github/workflows/build_tool.yml @@ -13,48 +13,13 @@ permissions: contents: read jobs: - base: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - uses: ./.github/actions/setup_docker - id: login - with: - repo: build-tool - ecr_role_arn: ${{ secrets.ECR_ROLE_ARN }} - dockerhub_user: ${{ secrets.DOCKERHUB_USERNAME }} - dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Get rust toolchain version - id: toolchain - run: | - version=$(awk -F'[ ="]+' '$1 == "channel" { print $2 }' scripts/setup/rust-toolchain.toml) - echo "TOOLCHAIN=${version}" >> $GITHUB_OUTPUT - - - name: Build and publish databend build base image - uses: docker/build-push-action@v5 - with: - pull: true - push: true - tags: | - ${{ steps.login.outputs.dockerhub_repo }}:base - ${{ steps.login.outputs.dockerhub_repo }}:base-${{ steps.toolchain.outputs.TOOLCHAIN }} - context: ./scripts/setup - file: ./docker/build-tool/base/Dockerfile - platforms: linux/amd64,linux/arm64 - build: runs-on: ubuntu-latest - needs: base strategy: matrix: - arch: - - x86_64 - - aarch64 - libc: - - gnu - - musl + os: + - debian + - alpine steps: - uses: actions/checkout@v4 @@ -69,7 +34,6 @@ jobs: - name: Get rust toolchain version id: toolchain run: | - # version=$(taplo get -f scripts/setup/rust-toolchain.toml '.toolchain.channel') version=$(awk -F'[ ="]+' '$1 == "channel" { print $2 }' scripts/setup/rust-toolchain.toml) echo "TOOLCHAIN=${version}" >> $GITHUB_OUTPUT @@ -79,19 +43,15 @@ jobs: pull: true push: true tags: | - ${{ steps.login.outputs.dockerhub_repo }}:${{ matrix.arch }}-unknown-linux-${{ matrix.libc }} - ${{ steps.login.outputs.dockerhub_repo }}:${{ matrix.arch }}-unknown-linux-${{ matrix.libc }}-${{ steps.toolchain.outputs.TOOLCHAIN }} - ${{ steps.login.outputs.ecr_repo }}:${{ matrix.arch }}-unknown-linux-${{ matrix.libc }} - ${{ steps.login.outputs.ecr_repo }}:${{ matrix.arch }}-unknown-linux-${{ matrix.libc }}-${{ steps.toolchain.outputs.TOOLCHAIN }} - context: ./docker/build-tool/${{ matrix.libc }}/ - file: ./docker/build-tool/${{ matrix.libc }}/Dockerfile - build-args: | - ARCH=${{ matrix.arch }} - RUST_TOOLCHAIN=${{ steps.toolchain.outputs.TOOLCHAIN }} + ${{ steps.login.outputs.dockerhub_repo }}:${{ matrix.os }} + ${{ steps.login.outputs.dockerhub_repo }}:${{ matrix.os }}-${{ steps.toolchain.outputs.TOOLCHAIN }} + context: ./scripts/setup + file: ./docker/build-tool/${{ matrix.os }}/Dockerfile + platforms: linux/amd64,linux/arm64 dev: runs-on: ubuntu-latest - needs: base + needs: build steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/reuse.linux.yml b/.github/workflows/reuse.linux.yml index d9c61774b2f7..563ae94d2a9b 100644 --- a/.github/workflows/reuse.linux.yml +++ b/.github/workflows/reuse.linux.yml @@ -73,12 +73,11 @@ jobs: artifacts: all build_address_sanitizer: - runs-on: [ self-hosted, X64, Linux, 8c16g, "${{ inputs.runner_provider }}" ] + runs-on: [ self-hosted, ARM64, Linux, 8c16g, "${{ inputs.runner_provider }}" ] strategy: fail-fast: false matrix: include: - # - { arch: x86_64, libc: gnu } - { arch: aarch64, libc: gnu } steps: - uses: actions/checkout@v4 @@ -97,9 +96,7 @@ jobs: fail-fast: false matrix: include: - # - { arch: x86_64, libc: musl } - { arch: aarch64, libc: musl } - # - { arch: aarch64, libc: gnu } steps: - uses: actions/checkout@v4 with: diff --git a/docker/build-tool/alpine/Dockerfile b/docker/build-tool/alpine/Dockerfile new file mode 100644 index 000000000000..766c249d21e1 --- /dev/null +++ b/docker/build-tool/alpine/Dockerfile @@ -0,0 +1,30 @@ +FROM alpine:3.19 + +ENV PATH=/root/.cargo/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + +ENV LANGUAGE en_US +ENV LANG en_US.UTF-8 +ENV LC_ALL en_US.UTF-8 + +RUN apk --update add --no-cache bash + +COPY rust-toolchain.toml /build/rust-toolchain.toml +COPY dev_setup.sh /build/scripts/setup/dev_setup.sh +RUN chmod +x /build/scripts/setup/dev_setup.sh && \ + /build/scripts/setup/dev_setup.sh -yb && \ + rm -rf /root/.cargo/git && \ + rm -rf /root/.cargo/registry && \ + mkdir -p /opt/rust && \ + mv /root/.cargo /opt/rust/cargo && \ + mv /root/.rustup /opt/rust/rustup + +RUN curl -sSfLo /tmp/mold.tar.gz https://github.com/rui314/mold/releases/download/v2.4.0/mold-2.4.0-$(uname -m)-linux.tar.gz && \ + tar --strip-components=1 -xzf /tmp/mold.tar.gz -C /usr/local && \ + rm -rf /tmp/mold.tar.gz && \ + ln -sf /usr/local/bin/mold /usr/bin/$(uname -m)-linux-gnu-ld + +ENV RUSTUP_HOME /opt/rust/rustup +ENV CARGO_HOME /opt/rust/cargo +ENV PATH /opt/rust/cargo/bin:/opt/java/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + +VOLUME [ "/workspace", "/opt/rust/cargo/registry", "/opt/rust/cargo/git" ] diff --git a/docker/build-tool/base/Dockerfile b/docker/build-tool/debian/Dockerfile similarity index 100% rename from docker/build-tool/base/Dockerfile rename to docker/build-tool/debian/Dockerfile diff --git a/docker/build-tool/gnu/Dockerfile b/docker/build-tool/gnu/Dockerfile deleted file mode 100644 index 4909d9321286..000000000000 --- a/docker/build-tool/gnu/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -ARG RUST_TOOLCHAIN -FROM datafuselabs/build-tool:base-${RUST_TOOLCHAIN} - -ENV PKG_CONFIG_ALLOW_CROSS 1 -RUN dpkg --add-architecture arm64 && \ - apt-get update -yq && \ - apt-get install -yq libc6-arm64-cross libc6-dev-arm64-cross gcc-aarch64-linux-gnu g++-aarch64-linux-gnu && \ - apt-get install -yq libdbus-1-dev libdbus-1-dev:arm64 && \ - apt-get install -yq libssl-dev libssl-dev:arm64 zlib1g-dev zlib1g-dev:arm64 liblzma-dev liblzma-dev:arm64 && \ - apt-get install -yq libsqlite3-dev libsqlite3-dev:arm64 - -ARG ARCH -RUN rustup target add ${ARCH}-unknown-linux-gnu -RUN ln -sf /usr/local/bin/mold /usr/bin/${ARCH}-linux-gnu-ld -RUN printf "[target.${ARCH}-unknown-linux-gnu]\nlinker = \"${ARCH}-linux-gnu-gcc\"\n" > ${CARGO_HOME}/config diff --git a/docker/build-tool/musl/Dockerfile b/docker/build-tool/musl/Dockerfile deleted file mode 100644 index 2c73fb1b3ca9..000000000000 --- a/docker/build-tool/musl/Dockerfile +++ /dev/null @@ -1,37 +0,0 @@ -ARG RUST_TOOLCHAIN -FROM datafuselabs/build-tool:base-${RUST_TOOLCHAIN} - -ARG ARCH -ENV MUSL_TARGET ${ARCH}-linux-musl -ENV MUSL_RUST_TARGET ${ARCH}-unknown-linux-musl - -# TODO:(everpcpc) change to tagged version after next release -# curl -sSfLo /tmp/musl-cross-make.tar.gz https://github.com/richfelker/musl-cross-make/archive/v0.9.9.tar.gz && \ -# tar -xzf /tmp/musl-cross-make.tar.gz -C /tmp -RUN curl -sSfLo /tmp/musl-cross-make.tar.gz https://github.com/richfelker/musl-cross-make/archive/fe915821b652a7fa37b34a596f47d8e20bc72338.zip && \ - unzip /tmp/musl-cross-make.tar.gz -d /tmp && \ - cd /tmp/musl-cross-make-* && \ - make install "-j$(nproc)" \ - OUTPUT=/usr/local/ \ - TARGET=${MUSL_TARGET} \ - DL_CMD='curl --retry 3 -sSfL -C - -o' && \ - rm -rf /tmp/musl-cross-make-* && \ - rm -f /tmp/musl-cross-make.tar.gz - -ENV C_INCLUDE_PATH /usr/local/${MUSL_TARGET}/include/ -RUN ln -s ${MUSL_TARGET}-gcc /usr/local/bin/musl-gcc - -RUN rustup target add ${MUSL_RUST_TARGET} - -# needed by z3 -RUN ln -s ${MUSL_TARGET}-g++ /usr/local/bin/musl-g++ -RUN ln -s ${MUSL_TARGET}-ar /usr/local/bin/musl-ar - -# HACK: something wrong with python detection in cmake for z3 -RUN ln -s python3.9 /usr/bin/python3.10 - -# HACK: to link with libstdc++ statically -# ref: https://github.com/rust-lang/rust/issues/36710#issuecomment-364623950 -COPY linker.sh /usr/local/bin/linker -RUN chmod +x /usr/local/bin/linker -RUN printf "[target.${MUSL_RUST_TARGET}]\nlinker = \"linker\"\n" >${CARGO_HOME}/config diff --git a/docker/build-tool/musl/linker.sh b/docker/build-tool/musl/linker.sh deleted file mode 100644 index a8ee80d9f326..000000000000 --- a/docker/build-tool/musl/linker.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -args=() - -for arg in "$@"; do - if [[ $arg = *"Bdynamic"* ]]; then - args+=() # we do not want this arg - elif [[ $arg = *"crti.o"* ]]; then - args+=("$arg" "-Bstatic") - elif [[ $arg = *"crtn.o"* ]]; then - args+=("-lgcc" "-lgcc_eh" "-lc" "/usr/local/lib/gcc/${MUSL_TARGET}/9.4.0/crtendS.o" "$arg") - else - args+=("$arg") - fi -done - -echo "RUNNING WITH ARGS: ${args[@]}" -mold -run /usr/local/bin/${MUSL_TARGET}-g++ "${args[@]}" diff --git a/scripts/setup/dev_setup.sh b/scripts/setup/dev_setup.sh index cb85d10dcfac..2df005e669cd 100755 --- a/scripts/setup/dev_setup.sh +++ b/scripts/setup/dev_setup.sh @@ -120,6 +120,9 @@ function install_ziglang { brew) install_pkg zig "$PACKAGE_MANAGER" ;; + apk) + echo "TODO: install ziglang for alpine" + ;; *) echo "Unable to install ziglang with package manager: $PACKAGE_MANAGER" exit 1 @@ -201,7 +204,7 @@ function install_protobuf { echo "==> installing protobuf compiler..." case "$PACKAGE_MANAGER" in - brew) + brew | apk) install_pkg protobuf "$PACKAGE_MANAGER" ;; *) @@ -343,7 +346,6 @@ function install_sqlite3 { ;; apk) install_pkg sqlite-dev "$PACKAGE_MANAGER" - install_pkg sqlite "$PACKAGE_MANAGER" ;; yum | dnf) install_pkg sqlite-devel "$PACKAGE_MANAGER"