From 08658b124d7f4e93463485980fe074c02920eb61 Mon Sep 17 00:00:00 2001 From: Philip Hyunsu Cho Date: Wed, 26 Jun 2024 01:43:15 -0700 Subject: [PATCH 1/4] [CI] Add CI pipeline to build libxgboost4j.so targeting Linux ARM64 (#10487) --- dev/prepare_jvm_release.py | 25 ++++------- .../build-jvm-linux-arm64-manylinux2014.sh | 29 +++++++++++++ .../build-jvm-linux-x86_64-manylinux2014.sh | 29 +++++++++++++ tests/buildkite/build-jvm-macos-m1.sh | 41 +++++++++++++++++++ tests/buildkite/pipeline-mac-m1.yml | 5 +++ tests/buildkite/pipeline.yml | 10 +++++ tests/buildkite/test-macos-m1-clang11.sh | 28 +------------ .../Dockerfile.jvm_manylinux2014_aarch64 | 17 ++++++++ .../Dockerfile.jvm_manylinux2014_x86_64 | 17 ++++++++ 9 files changed, 158 insertions(+), 43 deletions(-) create mode 100644 tests/buildkite/build-jvm-linux-arm64-manylinux2014.sh create mode 100644 tests/buildkite/build-jvm-linux-x86_64-manylinux2014.sh create mode 100644 tests/buildkite/build-jvm-macos-m1.sh create mode 100644 tests/ci_build/Dockerfile.jvm_manylinux2014_aarch64 create mode 100644 tests/ci_build/Dockerfile.jvm_manylinux2014_x86_64 diff --git a/dev/prepare_jvm_release.py b/dev/prepare_jvm_release.py index bcd119a892cb..af338c93cb55 100644 --- a/dev/prepare_jvm_release.py +++ b/dev/prepare_jvm_release.py @@ -134,6 +134,7 @@ def main(): print("====Creating directories to hold native binaries====") for os_ident, arch in [ ("linux", "x86_64"), + ("linux", "aarch64"), ("windows", "x86_64"), ("macos", "x86_64"), ("macos", "aarch64"), @@ -156,6 +157,14 @@ def main(): url=f"{nightly_bucket_prefix}/{git_branch}/libxgboost4j/xgboost4j_{commit_hash}.dll", filename="xgboost4j/src/main/resources/lib/windows/x86_64/xgboost4j.dll", ) + retrieve( + url=f"{nightly_bucket_prefix}/{git_branch}/libxgboost4j/libxgboost4j_linux_x86_64_{commit_hash}.so", + filename="xgboost4j/src/main/resources/lib/linux/x86_64/libxgboost4j.so", + ) + retrieve( + url=f"{nightly_bucket_prefix}/{git_branch}/libxgboost4j/libxgboost4j_linux_arm64_{commit_hash}.so", + filename="xgboost4j/src/main/resources/lib/linux/aarch64/libxgboost4j.so", + ) retrieve( url=f"{nightly_bucket_prefix}/{git_branch}/libxgboost4j/libxgboost4j_{commit_hash}.dylib", filename="xgboost4j/src/main/resources/lib/macos/x86_64/libxgboost4j.dylib", @@ -166,22 +175,6 @@ def main(): ) with tempfile.TemporaryDirectory() as tempdir: - # libxgboost4j.so for Linux x86_64, CPU only - zip_path = os.path.join(tempdir, "xgboost4j_2.12.jar") - extract_dir = os.path.join(tempdir, "xgboost4j") - retrieve( - url=f"{maven_repo_prefix}/xgboost4j_2.12/{version}/" - f"xgboost4j_2.12-{version}.jar", - filename=zip_path, - ) - os.mkdir(extract_dir) - with zipfile.ZipFile(zip_path, "r") as t: - t.extractall(extract_dir) - cp( - os.path.join(extract_dir, "lib", "linux", "x86_64", "libxgboost4j.so"), - "xgboost4j/src/main/resources/lib/linux/x86_64/libxgboost4j.so", - ) - # libxgboost4j.so for Linux x86_64, GPU support zip_path = os.path.join(tempdir, "xgboost4j-gpu_2.12.jar") extract_dir = os.path.join(tempdir, "xgboost4j-gpu") diff --git a/tests/buildkite/build-jvm-linux-arm64-manylinux2014.sh b/tests/buildkite/build-jvm-linux-arm64-manylinux2014.sh new file mode 100644 index 000000000000..f72183676e42 --- /dev/null +++ b/tests/buildkite/build-jvm-linux-arm64-manylinux2014.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +set -euo pipefail + +source tests/buildkite/conftest.sh + +command_wrapper="tests/ci_build/ci_build.sh jvm_manylinux2014_aarch64" + +# Build XGBoost4J binary +echo "--- Build libxgboost4j.so (targeting glibc 2.17)" +set -x +mkdir build +$command_wrapper bash -c \ + "cd build && cmake .. -DJVM_BINDINGS=ON -DUSE_OPENMP=ON && make -j$(nproc)" +ldd lib/libxgboost4j.so +objdump -T lib/libxgboost4j.so | grep GLIBC_ | sed 's/.*GLIBC_\([.0-9]*\).*/\1/g' | sort -Vu + +echo "--- Upload libxgboost4j.so" +pushd lib +libname=libxgboost4j_linux_arm64_${BUILDKITE_COMMIT}.so +mv -v libxgboost4j.so ${libname} +buildkite-agent artifact upload ${libname} +#if [[ ($is_pull_request == 0) && ($is_release_branch == 1) ]] +#then + aws s3 cp ${libname} \ + s3://xgboost-nightly-builds/${BRANCH_NAME}/libxgboost4j/ \ + --acl public-read --no-progress +#fi +popd diff --git a/tests/buildkite/build-jvm-linux-x86_64-manylinux2014.sh b/tests/buildkite/build-jvm-linux-x86_64-manylinux2014.sh new file mode 100644 index 000000000000..3c299bb36a01 --- /dev/null +++ b/tests/buildkite/build-jvm-linux-x86_64-manylinux2014.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +set -euo pipefail + +source tests/buildkite/conftest.sh + +command_wrapper="tests/ci_build/ci_build.sh jvm_manylinux2014_x86_64" + +# Build XGBoost4J binary +echo "--- Build libxgboost4j.so (targeting glibc 2.17)" +set -x +mkdir build +$command_wrapper bash -c \ + "cd build && cmake .. -GNinja -DJVM_BINDINGS=ON -DUSE_OPENMP=ON && ninja -v" +ldd lib/libxgboost4j.so +objdump -T lib/libxgboost4j.so | grep GLIBC_ | sed 's/.*GLIBC_\([.0-9]*\).*/\1/g' | sort -Vu + +echo "--- Upload libxgboost4j.so" +pushd lib +libname=libxgboost4j_linux_x86_64_${BUILDKITE_COMMIT}.so +mv -v libxgboost4j.so ${libname} +buildkite-agent artifact upload ${libname} +#if [[ ($is_pull_request == 0) && ($is_release_branch == 1) ]] +#then + aws s3 cp ${libname} \ + s3://xgboost-nightly-builds/${BRANCH_NAME}/libxgboost4j/ \ + --acl public-read --no-progress +#fi +popd diff --git a/tests/buildkite/build-jvm-macos-m1.sh b/tests/buildkite/build-jvm-macos-m1.sh new file mode 100644 index 000000000000..469b0786ee37 --- /dev/null +++ b/tests/buildkite/build-jvm-macos-m1.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +set -euo pipefail + +source tests/buildkite/conftest.sh + +# Display system info +echo "--- Display system information" +set -x +system_profiler SPSoftwareDataType +sysctl -n machdep.cpu.brand_string +uname -m +set +x + +# Build XGBoost4J binary +echo "--- Build libxgboost4j.dylib" +set -x +mkdir build +pushd build +export JAVA_HOME=$(/usr/libexec/java_home) +cmake .. -GNinja -DJVM_BINDINGS=ON -DUSE_OPENMP=ON -DCMAKE_OSX_DEPLOYMENT_TARGET=10.15 +ninja -v +popd +rm -rf build +otool -L lib/libxgboost.dylib +set +x + +echo "--- Upload libxgboost4j.dylib" +set -x +pushd lib +libname=libxgboost4j_m1_${BUILDKITE_COMMIT}.dylib +mv -v libxgboost4j.dylib ${libname} +buildkite-agent artifact upload ${libname} +#if [[ ($is_pull_request == 0) && ($is_release_branch == 1) ]] +#then + aws s3 cp ${libname} \ + s3://xgboost-nightly-builds/${BRANCH_NAME}/libxgboost4j/ \ + --acl public-read --no-progress +#fi +popd +set +x diff --git a/tests/buildkite/pipeline-mac-m1.yml b/tests/buildkite/pipeline-mac-m1.yml index 7e4a664acf6b..57b1b1d12010 100644 --- a/tests/buildkite/pipeline-mac-m1.yml +++ b/tests/buildkite/pipeline-mac-m1.yml @@ -1,6 +1,11 @@ steps: - block: ":rocket: Run this test job" if: build.pull_request.id != null || build.branch =~ /^dependabot\// + - label: ":macos: Build libxgboost4j.dylib for MacOS M1" + command: "tests/buildkite/build-jvm-macos-m1.sh" + key: mac-m1-jvm + agents: + queue: mac-mini-m1 - label: ":macos: Build and Test XGBoost for MacOS M1 with Clang 11" command: "tests/buildkite/test-macos-m1-clang11.sh" key: mac-m1-appleclang11 diff --git a/tests/buildkite/pipeline.yml b/tests/buildkite/pipeline.yml index 73a60517a672..acdb71dba529 100644 --- a/tests/buildkite/pipeline.yml +++ b/tests/buildkite/pipeline.yml @@ -56,6 +56,16 @@ steps: key: build-jvm-packages agents: queue: linux-amd64-cpu + - label: ":console: Build libxgboost4j.so for Linux ARM64 (targeting glibc 2.17)" + command: "tests/buildkite/build-jvm-linux-arm64-manylinux2014.sh" + key: build-jvm-linux-arm64-manylinux2014 + agents: + queue: linux-arm64-cpu + - label: ":console: Build libxgboost4j.so for Linux x86_64 (targeting glibc 2.17)" + command: "tests/buildkite/build-jvm-linux-x86_64-manylinux2014.sh" + key: build-jvm-linux-x86_64-manylinux2014 + agents: + queue: linux-amd64-cpu - label: ":console: Build JVM package doc" command: "tests/buildkite/build-jvm-doc.sh" key: build-jvm-doc diff --git a/tests/buildkite/test-macos-m1-clang11.sh b/tests/buildkite/test-macos-m1-clang11.sh index a3f1eab6c589..6824cb7b14b4 100755 --- a/tests/buildkite/test-macos-m1-clang11.sh +++ b/tests/buildkite/test-macos-m1-clang11.sh @@ -12,33 +12,6 @@ sysctl -n machdep.cpu.brand_string uname -m set +x -# Build XGBoost4J binary -echo "--- Build libxgboost4j.dylib" -set -x -mkdir build -pushd build -export JAVA_HOME=$(/usr/libexec/java_home) -cmake .. -GNinja -DJVM_BINDINGS=ON -DUSE_OPENMP=ON -DCMAKE_OSX_DEPLOYMENT_TARGET=10.15 -ninja -v -popd -rm -rf build -otool -L lib/libxgboost.dylib -set +x - -echo "--- Upload Python wheel" -set -x -pushd lib -mv -v libxgboost4j.dylib libxgboost4j_m1_${BUILDKITE_COMMIT}.dylib -buildkite-agent artifact upload libxgboost4j_m1_${BUILDKITE_COMMIT}.dylib -if [[ ($is_pull_request == 0) && ($is_release_branch == 1) ]] -then - aws s3 cp libxgboost4j_m1_${BUILDKITE_COMMIT}.dylib \ - s3://xgboost-nightly-builds/${BRANCH_NAME}/libxgboost4j/ \ - --acl public-read --no-progress -fi -popd -set +x - # Ensure that XGBoost can be built with Clang 11 echo "--- Build and Test XGBoost with MacOS M1, Clang 11" set -x @@ -49,3 +22,4 @@ cmake .. -GNinja -DCMAKE_C_COMPILER=${LLVM11_PATH}/bin/clang \ -DCMAKE_CXX_COMPILER=${LLVM11_PATH}/bin/clang++ -DGOOGLE_TEST=ON \ -DUSE_DMLC_GTEST=ON ninja -v +./testxgboost diff --git a/tests/ci_build/Dockerfile.jvm_manylinux2014_aarch64 b/tests/ci_build/Dockerfile.jvm_manylinux2014_aarch64 new file mode 100644 index 000000000000..1442a7644110 --- /dev/null +++ b/tests/ci_build/Dockerfile.jvm_manylinux2014_aarch64 @@ -0,0 +1,17 @@ +FROM quay.io/pypa/manylinux2014_aarch64 + +RUN yum update -y && yum install -y java-1.8.0-openjdk-devel + +# Install lightweight sudo (not bound to TTY) +ENV GOSU_VERSION 1.10 +RUN set -ex; \ + curl -o /usr/local/bin/gosu -L "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-arm64" && \ + chmod +x /usr/local/bin/gosu && \ + gosu nobody true + +# Default entry-point to use if running locally +# It will preserve attributes of created files +COPY entrypoint.sh /scripts/ + +WORKDIR /workspace +ENTRYPOINT ["/scripts/entrypoint.sh"] diff --git a/tests/ci_build/Dockerfile.jvm_manylinux2014_x86_64 b/tests/ci_build/Dockerfile.jvm_manylinux2014_x86_64 new file mode 100644 index 000000000000..7626e252db0a --- /dev/null +++ b/tests/ci_build/Dockerfile.jvm_manylinux2014_x86_64 @@ -0,0 +1,17 @@ +FROM quay.io/pypa/manylinux2014_x86_64 + +RUN yum update -y && yum install -y java-1.8.0-openjdk-devel ninja-build + +# Install lightweight sudo (not bound to TTY) +ENV GOSU_VERSION 1.10 +RUN set -ex; \ + curl -o /usr/local/bin/gosu -L "https://github.com/tianon/gosu/releases/download/$GOSU_VERSION/gosu-amd64" && \ + chmod +x /usr/local/bin/gosu && \ + gosu nobody true + +# Default entry-point to use if running locally +# It will preserve attributes of created files +COPY entrypoint.sh /scripts/ + +WORKDIR /workspace +ENTRYPOINT ["/scripts/entrypoint.sh"] From 5efc9795512a3acf84f83609d478d79971253bd1 Mon Sep 17 00:00:00 2001 From: Hyunsu Cho Date: Wed, 26 Jun 2024 06:21:46 -0700 Subject: [PATCH 2/4] [CI] [Hotfix] Make S3 upload conditional --- tests/buildkite/build-jvm-linux-arm64-manylinux2014.sh | 6 +++--- tests/buildkite/build-jvm-linux-x86_64-manylinux2014.sh | 6 +++--- tests/buildkite/build-jvm-macos-m1.sh | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/buildkite/build-jvm-linux-arm64-manylinux2014.sh b/tests/buildkite/build-jvm-linux-arm64-manylinux2014.sh index f72183676e42..e7fec780b956 100644 --- a/tests/buildkite/build-jvm-linux-arm64-manylinux2014.sh +++ b/tests/buildkite/build-jvm-linux-arm64-manylinux2014.sh @@ -20,10 +20,10 @@ pushd lib libname=libxgboost4j_linux_arm64_${BUILDKITE_COMMIT}.so mv -v libxgboost4j.so ${libname} buildkite-agent artifact upload ${libname} -#if [[ ($is_pull_request == 0) && ($is_release_branch == 1) ]] -#then +if [[ ($is_pull_request == 0) && ($is_release_branch == 1) ]] +then aws s3 cp ${libname} \ s3://xgboost-nightly-builds/${BRANCH_NAME}/libxgboost4j/ \ --acl public-read --no-progress -#fi +fi popd diff --git a/tests/buildkite/build-jvm-linux-x86_64-manylinux2014.sh b/tests/buildkite/build-jvm-linux-x86_64-manylinux2014.sh index 3c299bb36a01..46a819a016d3 100644 --- a/tests/buildkite/build-jvm-linux-x86_64-manylinux2014.sh +++ b/tests/buildkite/build-jvm-linux-x86_64-manylinux2014.sh @@ -20,10 +20,10 @@ pushd lib libname=libxgboost4j_linux_x86_64_${BUILDKITE_COMMIT}.so mv -v libxgboost4j.so ${libname} buildkite-agent artifact upload ${libname} -#if [[ ($is_pull_request == 0) && ($is_release_branch == 1) ]] -#then +if [[ ($is_pull_request == 0) && ($is_release_branch == 1) ]] +then aws s3 cp ${libname} \ s3://xgboost-nightly-builds/${BRANCH_NAME}/libxgboost4j/ \ --acl public-read --no-progress -#fi +fi popd diff --git a/tests/buildkite/build-jvm-macos-m1.sh b/tests/buildkite/build-jvm-macos-m1.sh index 469b0786ee37..1d2e5e8703bc 100644 --- a/tests/buildkite/build-jvm-macos-m1.sh +++ b/tests/buildkite/build-jvm-macos-m1.sh @@ -31,11 +31,11 @@ pushd lib libname=libxgboost4j_m1_${BUILDKITE_COMMIT}.dylib mv -v libxgboost4j.dylib ${libname} buildkite-agent artifact upload ${libname} -#if [[ ($is_pull_request == 0) && ($is_release_branch == 1) ]] -#then +if [[ ($is_pull_request == 0) && ($is_release_branch == 1) ]] +then aws s3 cp ${libname} \ s3://xgboost-nightly-builds/${BRANCH_NAME}/libxgboost4j/ \ --acl public-read --no-progress -#fi +fi popd set +x From 4b88dfff24f3912350dafc6c9acd2fa157a9313a Mon Sep 17 00:00:00 2001 From: Philip Hyunsu Cho Date: Wed, 26 Jun 2024 14:08:49 -0700 Subject: [PATCH 3/4] [CI] Temporarily pin pylint to 3.2.3 (#10494) * [CI] Temporarily pin pylint to 3.2.3 * Add quotes * Correct env --- tests/ci_build/conda_env/python_lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ci_build/conda_env/python_lint.yml b/tests/ci_build/conda_env/python_lint.yml index dc5105a84fab..4d6249f1ebe9 100644 --- a/tests/ci_build/conda_env/python_lint.yml +++ b/tests/ci_build/conda_env/python_lint.yml @@ -3,7 +3,7 @@ channels: - conda-forge dependencies: - python=3.8 -- pylint +- pylint<3.2.4 # https://github.com/pylint-dev/pylint/issues/9751 - wheel - setuptools - mypy>=0.981 From bed3695bebe5559a2418f4c005406e67069786f1 Mon Sep 17 00:00:00 2001 From: Philip Hyunsu Cho Date: Wed, 26 Jun 2024 16:35:23 -0700 Subject: [PATCH 4/4] [jvm-packages] Honor skip.native.build option in xgboost4j-gpu (#10496) --- jvm-packages/xgboost4j-gpu/pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/jvm-packages/xgboost4j-gpu/pom.xml b/jvm-packages/xgboost4j-gpu/pom.xml index bd26acd688cd..600c5ffeb9cd 100644 --- a/jvm-packages/xgboost4j-gpu/pom.xml +++ b/jvm-packages/xgboost4j-gpu/pom.xml @@ -106,6 +106,7 @@ ${use.cuda} ${user.dir} + ${skip.native.build}