Move all commands to separate script files

.github/workflows/freebsd.yml

@@ -27,8 +27,5 @@ jobs:
             pkg install -y cmake git ninja googletest
 
           run: |
-            mkdir build
-            cd build
-            cmake .. -GNinja -DGOOGLE_TEST=ON
-            ninja -v
-            ./testxgboost
+            bash ops/script/build_via_cmake.sh
+            ./build/testxgboost

.github/workflows/jvm_tests.yml

@@ -232,7 +232,6 @@ jobs:
         run: |
           cd lib/
           Rename-Item -Path xgboost4j.dll -NewName xgboost4j_${{ github.sha }}.dll
-          dir
           python -m awscli s3 cp xgboost4j_${{ github.sha }}.dll `
             s3://xgboost-nightly-builds/${{ env.BRANCH_NAME }}/libxgboost4j/ `
             --acl public-read --region us-west-2

.github/workflows/main.yml

@@ -268,14 +268,14 @@ jobs:
           KEY: ${{ matrix.artifact_from }}
       - run: bash ops/pipeline/test-cpp-gpu.sh ${{ matrix.suite }}
 
-  test-python:
+  test-python-wheel:
     name: Run Python tests (${{ matrix.description }})
     needs: [build-cuda, build-cpu-arm64]
     runs-on:
       - runs-on
       - runner=${{ matrix.runner }}
       - run-id=${{ github.run_id }}
-      - tag=main-test-python-${{ matrix.description }}
+      - tag=main-test-python-wheel-${{ matrix.description }}
     strategy:
       fail-fast: false
       max-parallel: 2
@@ -329,4 +329,4 @@ jobs:
           COMMAND: download
           KEY: ${{ matrix.artifact_from }}
       - name: Run Python tests, ${{ matrix.description }}
-        run: bash ops/pipeline/test-python.sh ${{ matrix.suite }} ${{ matrix.container }}
+        run: bash ops/pipeline/test-python-wheel.sh ${{ matrix.suite }} ${{ matrix.container }}

.github/workflows/python_tests.yml

@@ -14,9 +14,13 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  python-sdist-test-on-Linux:
-    runs-on: ubuntu-latest
-    name: Test installing XGBoost Python source package
+  python-sdist-test:
+    runs-on: ${{ matrix.os }}
+    name: Test installing Python XGBoost from the source distribution (${{ matrix.os }})
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-13, windows-latest, ubuntu-latest]
     steps:
       - uses: actions/checkout@v4
         with:
@@ -32,53 +36,12 @@ jobs:
         run: |
           conda info
           conda list
-      - name: Build and install XGBoost
+      - name: Install extra package for MacOS
         run: |
-          cd python-package
-          python --version
-          python -m build --sdist
-          pip install -v ./dist/xgboost-*.tar.gz --config-settings use_openmp=False
-          cd ..
-          python -c 'import xgboost'
-
-  python-sdist-test:
-    # Use system toolchain instead of conda toolchain for macos and windows.
-    # MacOS has linker error if clang++ from conda-forge is used
-    runs-on: ${{ matrix.os }}
-    name: Test installing XGBoost Python source package on ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [macos-13, windows-latest]
-        python-version: ["3.10"]
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: 'true'
-      - name: Install osx system dependencies
+          mamba install -c conda-forge llvm-openmp
         if: matrix.os == 'macos-13'
-        run: |
-          brew install ninja libomp
-      - uses: conda-incubator/setup-miniconda@v3
-        with:
-          auto-update-conda: true
-          python-version: ${{ matrix.python-version }}
-          activate-environment: test
-      - name: Install build
-        run: |
-          conda install -c conda-forge python-build
-      - name: Display Conda env
-        run: |
-          conda info
-          conda list
       - name: Build and install XGBoost
-        run: |
-          cd python-package
-          python --version
-          python -m build --sdist
-          pip install -v ./dist/xgboost-*.tar.gz
-          cd ..
-          python -c 'import xgboost'
+        run: bash ops/pipeline/test-python-sdist.sh
 
   python-tests-on-macos:
     name: Test XGBoost Python package on macos-13
@@ -102,31 +65,7 @@ jobs:
           conda info
           conda list
 
-      - name: Build XGBoost on macos
-        run: |
-          brew install ninja
-
-          mkdir build
-          cd build
-          # Set prefix, to use OpenMP library from Conda env
-          # See https://github.com/dmlc/xgboost/issues/7039#issuecomment-1025038228
-          # to learn why we don't use libomp from Homebrew.
-          cmake .. -GNinja -DCMAKE_PREFIX_PATH=$CONDA_PREFIX -DBUILD_DEPRECATED_CLI=ON
-          ninja
-
-      - name: Install Python package
-        run: |
-          cd python-package
-          python --version
-          pip install -v .
-
-      - name: Test Python package
-        run: |
-          pytest -s -v -rxXs --durations=0 ./tests/python
-
-      - name: Test Dask Interface
-        run: |
-          pytest -s -v -rxXs --durations=0 ./tests/test_distributed/test_with_dask
+      - run: bash test-python-macos.sh
 
   python-system-installation-on-ubuntu:
     name: Test XGBoost Python package System Installation on Ubuntu
@@ -135,32 +74,8 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: 'true'
-
       - name: Set up Python 3.10
         uses: actions/setup-python@v5
         with:
           python-version: "3.10"
-
-      - name: Install ninja
-        run: |
-          sudo apt-get update && sudo apt-get install -y ninja-build
-
-      - name: Build XGBoost on Ubuntu
-        run: |
-          mkdir build
-          cd build
-          cmake .. -GNinja
-          ninja
-
-      - name: Copy lib to system lib
-        run: |
-          cp lib/* "$(python -c 'import sys; print(sys.base_prefix)')/lib"
-
-      - name: Install XGBoost in Virtual Environment
-        run: |
-          cd python-package
-          pip install virtualenv
-          virtualenv venv
-          source venv/bin/activate && \
-            pip install -v . --config-settings use_system_libxgboost=True && \
-            python -c 'import xgboost'
+      - run: bash test-python-with-sysprefix.sh

.github/workflows/sycl_tests.yml

@@ -38,15 +38,9 @@ jobs:
         conda list
     - name: Build and install XGBoost
       run: |
-        mkdir build
-        cd build
-        cmake .. -DGOOGLE_TEST=ON -DUSE_DMLC_GTEST=ON -DPLUGIN_SYCL=ON -DCMAKE_CXX_COMPILER=g++ \
-          -DCMAKE_C_COMPILER=gcc -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX -GNinja
-        ninja
+        bash ops/script/build_via_cmake.sh --conda-env=linux_sycl_test -DPLUGIN_SYCL=ON
     - name: Run gtest
-      run: |
-        cd build
-        ./testxgboost
+      run: ./build/testxgboost
 
   python-sycl-tests-on-ubuntu:
     name: Test XGBoost Python package with SYCL
@@ -56,31 +50,22 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: 'true'
-
       - uses: conda-incubator/setup-miniconda@v3
         with:
           miniforge-variant: Miniforge3
           miniforge-version: latest
           activate-environment: linux_sycl_test
           environment-file: ops/conda_env/linux_sycl_test.yml
           use-mamba: true
-
       - name: Display Conda env
         run: |
           conda info
           conda list
-      - name: Build XGBoost on Ubuntu
-        run: |
-          mkdir build
-          cd build
-          cmake .. -DPLUGIN_SYCL=ON -DCMAKE_CXX_COMPILER=g++ -DCMAKE_C_COMPILER=gcc \
-            -DCMAKE_PREFIX_PATH=$CONDA_PREFIX -GNinja
-          ninja
-      - name: Install Python package
+      - name: Build and install XGBoost
         run: |
+          bash ops/script/build_via_cmake.sh --conda-env=linux_sycl_test -DPLUGIN_SYCL=ON
           cd python-package
           python --version
           pip install -v .
       - name: Test Python package
-        run: |
-          pytest -s -v -rxXs --durations=0 ./tests/python-sycl/
+        run: pytest -s -v -rxXs --durations=0 ./tests/python-sycl/

ops/conda_env/sdist_test.yml

@@ -9,5 +9,3 @@ dependencies:
 - cmake
 - ninja
 - python-build
-- c-compiler
-- cxx-compiler

ops/pipeline/build-test-jvm-packages-impl.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 ## Build and test JVM packages.
+## Companion script for build-test-jvm-packages.sh.
 ##
 ## Note. This script takes in all inputs via environment variables.
 

ops/pipeline/classify-git-branch.sh

@@ -1,6 +1,5 @@
 #!/bin/bash
-
-## Detect whether the current git branch is a pull request or a release branch.
+## Detect whether the current git branch is a pull request or a release branch
 
 set -euo pipefail
 

ops/pipeline/deploy-jvm-packages-impl.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
-## Deploy JVM packages to xgboost-maven-repo S3 bucket
+## Deploy JVM packages to S3 bucket
+## Companion script for ops/pipeline/deploy-jvm-packages.sh
 
 set -euox pipefail
 

ops/pipeline/deploy-jvm-packages.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+## Deploy JVM packages to S3 bucket
 
 set -euox pipefail
 

ops/pipeline/test-python-macos.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+## Test XGBoost Python wheel on MacOS
+
+set -euox pipefail
+
+brew install ninja
+
+mkdir build
+cd build
+# Set prefix, to use OpenMP library from Conda env
+# See https://github.com/dmlc/xgboost/issues/7039#issuecomment-1025038228
+# to learn why we don't use libomp from Homebrew.
+cmake .. -GNinja -DCMAKE_PREFIX_PATH=$CONDA_PREFIX -DBUILD_DEPRECATED_CLI=ON
+ninja
+
+cd python-package
+python --version
+pip install -v .
+
+pytest -s -v -rxXs --durations=0 ./tests/python
+pytest -s -v -rxXs --durations=0 ./tests/test_distributed/test_with_dask

ops/pipeline/test-python-sdist.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+## Test installing Python XGBoost from source distribution
+
+set -euox pipefail
+
+cd python-package
+python --version
+python -m build --sdist
+pip install -v ./dist/xgboost-*.tar.gz
+cd ..
+python -c 'import xgboost'

ops/pipeline/test-python-impl.sh → ops/pipeline/test-python-wheel-impl.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+## Companion script for ops/pipeline/test-python-wheel.sh
 
 set -eo pipefail
 

ops/pipeline/test-python.sh → ops/pipeline/test-python-wheel.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+## Test XGBoost Python wheel on the Linux platform
 
 set -euo pipefail
 
@@ -20,4 +21,4 @@ fi
 
 python3 ops/docker_run.py --container-id "${container_id}" ${gpu_option} \
   --run-args='--shm-size=4g' \
-  -- bash ops/pipeline/test-python-impl.sh "${suite}"
+  -- bash ops/pipeline/test-python-wheel-impl.sh "${suite}"

ops/pipeline/test-python-with-sysprefix.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+## Test if Python XGBoost can be configured to use libxgboost.so from the system prefix
+
+set -euox pipefail
+
+sudo apt-get update && sudo apt-get install -y ninja-build
+
+mkdir build
+cd build
+cmake .. -GNinja
+ninja
+
+# Copy libxgboost.so to system prefix
+cp -v lib/* "$(python -c 'import sys; print(sys.base_prefix)')/lib"
+
+# Now configure Python XGBoost to use libxgboost.so from the system prefix
+cd python-package
+pip install virtualenv
+virtualenv venv
+source venv/bin/activate && \
+  pip install -v . --config-settings use_system_libxgboost=True && \
+  python -c 'import xgboost'