diff --git a/.github/workflows/jvm_tests.yml b/.github/workflows/jvm_tests.yml index 8318749b9a0f..fc2f21d68043 100644 --- a/.github/workflows/jvm_tests.yml +++ b/.github/workflows/jvm_tests.yml @@ -247,10 +247,10 @@ jobs: matrix: variant: - name: cpu - container_id: xgb-ci.jvm + image_repo: xgb-ci.jvm artifact_from: build-test-jvm-packages - name: gpu - container_id: xgb-ci.jvm_gpu_build + image_repo: xgb-ci.jvm_gpu_build artifact_from: build-jvm-gpu scala_version: ['2.12', '2.13'] steps: @@ -272,4 +272,4 @@ jobs: - name: Deploy JVM packages to S3 run: | bash ops/pipeline/deploy-jvm-packages.sh ${{ matrix.variant.name }} \ - ${{ matrix.variant.container_id }} ${{ matrix.scala_version }} + ${{ matrix.variant.image_repo }} ${{ matrix.scala_version }} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 81abe556dd93..332606cece7b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -220,22 +220,22 @@ jobs: matrix: include: - description: single-gpu - container: xgb-ci.gpu + image_repo: xgb-ci.gpu suite: gpu runner: linux-amd64-gpu artifact_from: build-cuda - description: multiple-gpu - container: xgb-ci.gpu + image_repo: xgb-ci.gpu suite: mgpu runner: linux-amd64-mgpu artifact_from: build-cuda - description: cpu-amd64 - container: xgb-ci.cpu + image_repo: xgb-ci.cpu suite: cpu runner: linux-amd64-cpu artifact_from: build-cuda - description: cpu-arm64 - container: xgb-ci.aarch64 + image_repo: xgb-ci.aarch64 suite: cpu-arm64 runner: linux-arm64-cpu artifact_from: build-cpu-arm64 @@ -257,4 +257,4 @@ jobs: mv -v wheelhouse/xgboost . chmod +x ./xgboost - name: Run Python tests, ${{ matrix.description }} - run: bash ops/pipeline/test-python-wheel.sh ${{ matrix.suite }} ${{ matrix.container }} + run: bash ops/pipeline/test-python-wheel.sh ${{ matrix.suite }} ${{ matrix.image_repo }} diff --git a/doc/contrib/ci.rst b/doc/contrib/ci.rst index edc144326cdc..3b3542dcd9cc 100644 --- a/doc/contrib/ci.rst +++ b/doc/contrib/ci.rst @@ -44,16 +44,17 @@ To make changes to the CI container, carry out the following steps: 4. Submit a pull request to `dmlc/xgboost-devops `_ with the proposed changes to the Dockerfile. Make note of the pull request number. Example: ``#204`` 5. Clone `dmlc/xgboost `_ and update all references to the - old container to point to the new container. More specifically, all Docker tags of format - ``492475357299.dkr.ecr.us-west-2.amazonaws.com/[container_id]:main`` should have the last - component replaced with ``PR-#``, where ``#`` is the pull request number. For the example above, + old container to point to the new container. More specifically, all container image URIs of form + ``492475357299.dkr.ecr.us-west-2.amazonaws.com/[image_repo]:main`` should have its image tag + (last component) replaced with ``PR-#``, where ``#`` is the pull request number. + For the example above, we'd replace ``492475357299.dkr.ecr.us-west-2.amazonaws.com/xgb-ci.gpu:main`` with ``492475357299.dkr.ecr.us-west-2.amazonaws.com/xgb-ci.gpu:PR-204``. 6. Now submit a pull request to `dmlc/xgboost `_. The CI will run tests using the new container. Verify that all tests pass. 7. Merge the pull request in ``dmlc/xgboost-devops``. Wait until the CI completes on the ``main`` branch. -8. Go back to the the pull request for ``dmlc/xgboost`` and change the container references back - to ``:main``. +8. Go back to the the pull request for ``dmlc/xgboost`` and revise all the container references to use + the old tag ``:main``. 9. Merge the pull request in ``dmlc/xgboost``. .. _build_run_docker_locally: @@ -83,11 +84,12 @@ and invoke ``containers/docker_build.sh`` as follows: # For local testing, set them to "main" export GITHUB_SHA="main" export BRANCH_NAME="main" - bash containers/docker_build.sh CONTAINER_ID + bash containers/docker_build.sh IMAGE_REPO -where ``CONTAINER_ID`` identifies for the container. The wrapper script will look up the YAML file -``containers/ci_container.yml``. For example, when ``CONTAINER_ID`` is set to ``xgb-ci.gpu``, -the script will use the corresponding entry from ``containers/ci_container.yml``: +where ``IMAGE_REPO`` is the name of the container image. The wrapper script will look up the +YAML file ``containers/ci_container.yml``. For example, when ``IMAGE_REPO`` is set to +``xgb-ci.gpu``, the script will use the corresponding entry from +``containers/ci_container.yml``: .. code-block:: yaml @@ -113,10 +115,11 @@ the build arguments are: The build arguments provide inputs to the ``ARG`` instructions in the Dockerfile. -When ``containers/docker_build.sh`` completes, you will have access to the container with tag -``492475357299.dkr.ecr.us-west-2.amazonaws.com/[container_id]:main``. The prefix -``492475357299.dkr.ecr.us-west-2.amazonaws.com/`` was added so that the container could -later be uploaded to AWS Elastic Container Registry (ECR), a private Docker registry. +When ``containers/docker_build.sh`` completes, you will have access to the container with the +(fully qualified) URI ``492475357299.dkr.ecr.us-west-2.amazonaws.com/[image_repo]:main``. +The prefix ``492475357299.dkr.ecr.us-west-2.amazonaws.com/`` was added so that +the container could later be uploaded to AWS Elastic Container Registry (ECR), +a private Docker registry. ----------------------------------------- To run commands within a Docker container @@ -126,7 +129,7 @@ Invoke ``ops/docker_run.py`` from the main ``dmlc/xgboost`` repo as follows: .. code-block:: bash python3 ops/docker_run.py \ - --container-tag 492475357299.dkr.ecr.us-west-2.amazonaws.com/[container_id]:main \ + --image-uri 492475357299.dkr.ecr.us-west-2.amazonaws.com/[image_repo]:[image_tag] \ [--use-gpus] \ -- "command to run inside the container" @@ -138,12 +141,12 @@ For example: # Run without GPU python3 ops/docker_run.py \ - --container-tag 492475357299.dkr.ecr.us-west-2.amazonaws.com/xgb-ci.cpu:main \ + --image-uri 492475357299.dkr.ecr.us-west-2.amazonaws.com/xgb-ci.cpu:main \ -- bash ops/pipeline/build-cpu-impl.sh cpu # Run with NVIDIA GPU python3 ops/docker_run.py \ - --container-tag 492475357299.dkr.ecr.us-west-2.amazonaws.com/xgb-ci.gpu:main \ + --image-uri 492475357299.dkr.ecr.us-west-2.amazonaws.com/xgb-ci.gpu:main \ --use-gpus \ -- bash ops/pipeline/test-python-wheel-impl.sh gpu @@ -154,7 +157,7 @@ Optionally, you can specify ``--run-args`` to pass extra arguments to ``docker r # Allocate extra space in /dev/shm to enable NCCL # Also run the container with elevated privileges python3 ops/docker_run.py \ - --container-tag 492475357299.dkr.ecr.us-west-2.amazonaws.com/xgb-ci.gpu:main \ + --image-uri 492475357299.dkr.ecr.us-west-2.amazonaws.com/xgb-ci.gpu:main \ --use-gpus \ --run-args='--shm-size=4g --privileged' \ -- bash ops/pipeline/test-python-wheel-impl.sh gpu @@ -171,7 +174,7 @@ Examples: useful tasks for local development export DOCKER_REGISTRY=492475357299.dkr.ecr.us-west-2.amazonaws.com python3 ops/docker_run.py \ - --container-tag ${DOCKER_REGISTRY}/xgb-ci.gpu_build_rockylinux8:main \ + --image-uri ${DOCKER_REGISTRY}/xgb-ci.gpu_build_rockylinux8:main \ -- ops/pipeline/build-cuda-impl.sh * Run Python tests @@ -180,7 +183,7 @@ Examples: useful tasks for local development export DOCKER_REGISTRY=492475357299.dkr.ecr.us-west-2.amazonaws.com python3 ops/docker_run.py \ - --container-tag ${DOCKER_REGISTRY}/xgb-ci.cpu:main \ + --image-uri ${DOCKER_REGISTRY}/xgb-ci.cpu:main \ -- ops/pipeline/test-python-wheel-impl.sh cpu * Run Python tests with GPU algorithm @@ -189,7 +192,7 @@ Examples: useful tasks for local development export DOCKER_REGISTRY=492475357299.dkr.ecr.us-west-2.amazonaws.com python3 ops/docker_run.py \ - --container-tag ${DOCKER_REGISTRY}/xgb-ci.gpu:main \ + --image-uri ${DOCKER_REGISTRY}/xgb-ci.gpu:main \ --use-gpus \ -- ops/pipeline/test-python-wheel-impl.sh gpu @@ -199,7 +202,7 @@ Examples: useful tasks for local development export DOCKER_REGISTRY=492475357299.dkr.ecr.us-west-2.amazonaws.com python3 ops/docker_run.py \ - --container-tag ${DOCKER_REGISTRY}/xgb-ci.gpu:main \ + --image-uri ${DOCKER_REGISTRY}/xgb-ci.gpu:main \ --use-gpus \ --run-args='--shm-size=4g' \ -- ops/pipeline/test-python-wheel-impl.sh mgpu @@ -212,7 +215,7 @@ Examples: useful tasks for local development export DOCKER_REGISTRY=492475357299.dkr.ecr.us-west-2.amazonaws.com export SCALA_VERSION=2.12 # Specify Scala version (2.12 or 2.13) python3 ops/docker_run.py \ - --container-tag ${DOCKER_REGISTRY}/xgb-ci.jvm:main \ + --image-uri ${DOCKER_REGISTRY}/xgb-ci.jvm:main \ --run-args "-e SCALA_VERSION" \ -- ops/pipeline/build-test-jvm-packages-impl.sh @@ -224,7 +227,7 @@ Examples: useful tasks for local development export SCALA_VERSION=2.12 # Specify Scala version (2.12 or 2.13) export USE_CUDA=1 python3 ops/docker_run.py \ - --container-tag ${DOCKER_REGISTRY}/xgb-ci.jvm_gpu_build:main \ + --image-uri ${DOCKER_REGISTRY}/xgb-ci.jvm_gpu_build:main \ --use-gpus \ --run-args "-e SCALA_VERSION -e USE_CUDA --shm-size=4g" \ -- ops/pipeline/build-test-jvm-packages-impl.sh @@ -456,7 +459,7 @@ For example, when you run ``bash containers/docker_build.sh xgb-ci.gpu``, the lo # docker_build.sh calls docker_build.py... python3 containers/docker_build.py --container-def gpu \ - --container-tag 492475357299.dkr.ecr.us-west-2.amazonaws.com/xgb-ci.gpu:main \ + --image-uri 492475357299.dkr.ecr.us-west-2.amazonaws.com/xgb-ci.gpu:main \ --build-arg CUDA_VERSION_ARG=12.4.1 --build-arg NCCL_VERSION_ARG=2.23.4-1 \ --build-arg RAPIDS_VERSION_ARG=24.10 @@ -480,14 +483,14 @@ Here is an example with ``docker_run.py``: # Run without GPU python3 ops/docker_run.py \ - --container-tag 492475357299.dkr.ecr.us-west-2.amazonaws.com/xgb-ci.cpu:main \ + --image-uri 492475357299.dkr.ecr.us-west-2.amazonaws.com/xgb-ci.cpu:main \ -- bash ops/pipeline/build-cpu-impl.sh cpu # Run with NVIDIA GPU # Allocate extra space in /dev/shm to enable NCCL # Also run the container with elevated privileges python3 ops/docker_run.py \ - --container-tag 492475357299.dkr.ecr.us-west-2.amazonaws.com/xgb-ci.gpu:main \ + --image-uri 492475357299.dkr.ecr.us-west-2.amazonaws.com/xgb-ci.gpu:main \ --use-gpus \ --run-args='--shm-size=4g --privileged' \ -- bash ops/pipeline/test-python-wheel-impl.sh gpu diff --git a/ops/docker_run.py b/ops/docker_run.py index 06f9d6cc8dc8..949f7fb7807d 100644 --- a/ops/docker_run.py +++ b/ops/docker_run.py @@ -53,7 +53,7 @@ def fancy_print_cli_args(*, cli_args: list[str]) -> None: def docker_run( *, - container_tag: str, + image_uri: str, command_args: list[str], use_gpus: bool, workdir: pathlib.Path, @@ -71,7 +71,7 @@ def docker_run( itertools.chain.from_iterable([["-e", f"{k}={v}"] for k, v in user_ids.items()]) ) docker_run_cli_args.extend(extra_args) - docker_run_cli_args.append(container_tag) + docker_run_cli_args.append(image_uri) docker_run_cli_args.extend(command_args) cli_args = ["docker", "run"] + docker_run_cli_args @@ -90,7 +90,7 @@ def main(*, args: argparse.Namespace) -> None: run_args.append("-it") docker_run( - container_tag=args.container_tag, + image_uri=args.image_uri, command_args=args.command_args, use_gpus=args.use_gpus, workdir=args.workdir, @@ -102,18 +102,18 @@ def main(*, args: argparse.Namespace) -> None: if __name__ == "__main__": parser = argparse.ArgumentParser( usage=( - f"{sys.argv[0]} --container-tag CONTAINER_TAG [--use-gpus] [--interactive] " + f"{sys.argv[0]} --image-uri IMAGE_URI [--use-gpus] [--interactive] " "[--workdir WORKDIR] [--run-args RUN_ARGS] -- COMMAND_ARG " "[COMMAND_ARG ...]" ), description="Run tasks inside a Docker container", ) parser.add_argument( - "--container-tag", + "--image-uri", type=str, required=True, help=( - "Container tag to identify the container, e.g. " + "Fully qualified image URI to identify the container, e.g. " "492475357299.dkr.ecr.us-west-2.amazonaws.com/xgb-ci.gpu:main" ), ) diff --git a/ops/pipeline/build-cpu-arm64.sh b/ops/pipeline/build-cpu-arm64.sh index 9801790baaaa..e3e0cebb22c8 100755 --- a/ops/pipeline/build-cpu-arm64.sh +++ b/ops/pipeline/build-cpu-arm64.sh @@ -13,17 +13,17 @@ source ops/pipeline/classify-git-branch.sh source ops/pipeline/get-docker-registry-details.sh WHEEL_TAG=manylinux_2_28_aarch64 -CONTAINER_TAG=${DOCKER_REGISTRY_URL}/xgb-ci.aarch64:main +IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.aarch64:main echo "--- Build CPU code targeting ARM64" set -x python3 ops/docker_run.py \ - --container-tag ${CONTAINER_TAG} \ + --image-uri ${IMAGE_URI} \ -- ops/pipeline/build-cpu-arm64-impl.sh echo "--- Audit binary wheel to ensure it's compliant with ${WHEEL_TAG} standard" python3 ops/docker_run.py \ - --container-tag ${CONTAINER_TAG} \ + --image-uri ${IMAGE_URI} \ -- auditwheel repair --only-plat \ --plat ${WHEEL_TAG} python-package/dist/*.whl python3 -m wheel tags --python-tag py3 --abi-tag none --platform ${WHEEL_TAG} --remove \ diff --git a/ops/pipeline/build-cpu.sh b/ops/pipeline/build-cpu.sh index 04fd4944eae7..2443006bd7c2 100755 --- a/ops/pipeline/build-cpu.sh +++ b/ops/pipeline/build-cpu.sh @@ -6,7 +6,7 @@ set -euo pipefail source ops/pipeline/classify-git-branch.sh source ops/pipeline/get-docker-registry-details.sh -CONTAINER_TAG=${DOCKER_REGISTRY_URL}/xgb-ci.cpu:main +IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.cpu:main echo "--- Build CPU code" set -x @@ -24,7 +24,7 @@ export UBSAN_OPTIONS='print_stacktrace=1:log_path=ubsan_error.log' # Work around https://github.com/google/sanitizers/issues/1614 sudo sysctl vm.mmap_rnd_bits=28 python3 ops/docker_run.py \ - --container-tag ${CONTAINER_TAG} \ + --image-uri ${IMAGE_URI} \ --run-args '-e ASAN_SYMBOLIZER_PATH -e ASAN_OPTIONS -e UBSAN_OPTIONS --cap-add SYS_PTRACE' \ -- bash ops/pipeline/build-cpu-impl.sh cpu-sanitizer @@ -32,5 +32,5 @@ python3 ops/docker_run.py \ # Test without sanitizer rm -rf build/ python3 ops/docker_run.py \ - --container-tag ${CONTAINER_TAG} \ + --image-uri ${IMAGE_URI} \ -- bash ops/pipeline/build-cpu-impl.sh cpu diff --git a/ops/pipeline/build-cuda.sh b/ops/pipeline/build-cuda.sh index 182640e54e36..a34a0dcc7c28 100755 --- a/ops/pipeline/build-cuda.sh +++ b/ops/pipeline/build-cuda.sh @@ -11,10 +11,10 @@ fi if [[ "$#" -lt 2 ]] then - echo "Usage: $0 [container_id] {enable-rmm,disable-rmm}" + echo "Usage: $0 [image_repo] {enable-rmm,disable-rmm}" exit 2 fi -container_id="$1" +image_repo="$1" rmm_flag="$2" # Validate RMM flag @@ -35,8 +35,8 @@ source ops/pipeline/classify-git-branch.sh source ops/pipeline/get-docker-registry-details.sh WHEEL_TAG=manylinux_2_28_x86_64 -BUILD_CONTAINER_TAG="${DOCKER_REGISTRY_URL}/${container_id}:main" -MANYLINUX_CONTAINER_TAG="${DOCKER_REGISTRY_URL}/xgb-ci.${WHEEL_TAG}:main" +BUILD_IMAGE_URI="${DOCKER_REGISTRY_URL}/${image_repo}:main" +MANYLINUX_IMAGE_URI="${DOCKER_REGISTRY_URL}/xgb-ci.${WHEEL_TAG}:main" echo "--- Build with CUDA" @@ -57,13 +57,13 @@ fi set -x python3 ops/docker_run.py \ - --container-tag ${BUILD_CONTAINER_TAG} \ + --image-uri ${BUILD_IMAGE_URI} \ --run-args='-e BUILD_ONLY_SM75 -e USE_RMM' \ -- ops/pipeline/build-cuda-impl.sh echo "--- Audit binary wheel to ensure it's compliant with ${WHEEL_TAG} standard" python3 ops/docker_run.py \ - --container-tag ${MANYLINUX_CONTAINER_TAG} \ + --image-uri ${MANYLINUX_IMAGE_URI} \ -- auditwheel repair --only-plat \ --plat ${WHEEL_TAG} python-package/dist/*.whl python3 -m wheel tags --python-tag py3 --abi-tag none --platform ${WHEEL_TAG} --remove \ diff --git a/ops/pipeline/build-gpu-rpkg.sh b/ops/pipeline/build-gpu-rpkg.sh index 07a08ff15385..ced68245d4bf 100755 --- a/ops/pipeline/build-gpu-rpkg.sh +++ b/ops/pipeline/build-gpu-rpkg.sh @@ -11,12 +11,12 @@ fi source ops/pipeline/classify-git-branch.sh source ops/pipeline/get-docker-registry-details.sh -CONTAINER_TAG=${DOCKER_REGISTRY_URL}/xgb-ci.gpu_build_r_rockylinux8:main +IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.gpu_build_r_rockylinux8:main echo "--- Build XGBoost R package with CUDA" set -x python3 ops/docker_run.py \ - --container-tag ${CONTAINER_TAG} \ + --image-uri ${IMAGE_URI} \ -- ops/pipeline/build-gpu-rpkg-impl.sh \ ${GITHUB_SHA} diff --git a/ops/pipeline/build-jvm-doc.sh b/ops/pipeline/build-jvm-doc.sh index a61f903cb5b9..3f9fad3bc2d0 100755 --- a/ops/pipeline/build-jvm-doc.sh +++ b/ops/pipeline/build-jvm-doc.sh @@ -19,10 +19,10 @@ fi source ops/pipeline/get-docker-registry-details.sh -CONTAINER_TAG=${DOCKER_REGISTRY_URL}/xgb-ci.jvm_gpu_build:main +IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.jvm_gpu_build:main echo "--- Build JVM packages doc" set -x python3 ops/docker_run.py \ - --container-tag ${CONTAINER_TAG} \ + --image-uri ${IMAGE_URI} \ -- ops/pipeline/build-jvm-doc-impl.sh ${BRANCH_NAME} diff --git a/ops/pipeline/build-jvm-gpu.sh b/ops/pipeline/build-jvm-gpu.sh index 3d6f446eb462..2f71849efb3b 100755 --- a/ops/pipeline/build-jvm-gpu.sh +++ b/ops/pipeline/build-jvm-gpu.sh @@ -6,7 +6,7 @@ set -euo pipefail source ops/pipeline/classify-git-branch.sh source ops/pipeline/get-docker-registry-details.sh -CONTAINER_TAG=${DOCKER_REGISTRY_URL}/xgb-ci.jvm_gpu_build:main +IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.jvm_gpu_build:main echo "--- Build libxgboost4j.so with CUDA" @@ -32,5 +32,5 @@ mkdir -p build-gpu/ # TODO(hcho3): Remove this once new CUDA version ships with CCCL 2.6.0+ git clone https://github.com/NVIDIA/cccl.git -b v2.6.1 --quiet --depth 1 python3 ops/docker_run.py \ - --container-tag ${CONTAINER_TAG} \ + --image-uri ${IMAGE_URI} \ -- bash -c "${COMMAND}" diff --git a/ops/pipeline/build-jvm-manylinux2014.sh b/ops/pipeline/build-jvm-manylinux2014.sh index 068fb5fb0c44..5b3942fb67ef 100755 --- a/ops/pipeline/build-jvm-manylinux2014.sh +++ b/ops/pipeline/build-jvm-manylinux2014.sh @@ -10,19 +10,19 @@ then fi arch=$1 -container_id="xgb-ci.manylinux2014_${arch}" +image_repo="xgb-ci.manylinux2014_${arch}" source ops/pipeline/classify-git-branch.sh source ops/pipeline/get-docker-registry-details.sh -CONTAINER_TAG="${DOCKER_REGISTRY_URL}/${container_id}:main" +IMAGE_URI="${DOCKER_REGISTRY_URL}/${image_repo}:main" # Build XGBoost4J binary echo "--- Build libxgboost4j.so (targeting glibc 2.17)" set -x mkdir build python3 ops/docker_run.py \ - --container-tag "${CONTAINER_TAG}" \ + --image-uri "${IMAGE_URI}" \ -- bash -c \ "cd build && cmake .. -DJVM_BINDINGS=ON -DUSE_OPENMP=ON && make -j$(nproc)" ldd lib/libxgboost4j.so diff --git a/ops/pipeline/build-manylinux2014.sh b/ops/pipeline/build-manylinux2014.sh index fbc349568e72..cf8ca7a94da3 100755 --- a/ops/pipeline/build-manylinux2014.sh +++ b/ops/pipeline/build-manylinux2014.sh @@ -20,9 +20,9 @@ source ops/pipeline/classify-git-branch.sh source ops/pipeline/get-docker-registry-details.sh WHEEL_TAG="manylinux2014_${arch}" -container_id="xgb-ci.${WHEEL_TAG}" -python_bin="/opt/python/cp310-cp310/bin/python" -CONTAINER_TAG="${DOCKER_REGISTRY_URL}/${container_id}:main" +IMAGE_REPO="xgb-ci.${WHEEL_TAG}" +IMAGE_URI="${DOCKER_REGISTRY_URL}/${IMAGE_REPO}:main" +PYTHON_BIN="/opt/python/cp310-cp310/bin/python" echo "--- Build binary wheel for ${WHEEL_TAG}" set -x @@ -30,14 +30,14 @@ set -x patch -p0 < ops/patch/remove_nccl_dep.patch patch -p0 < ops/patch/manylinux2014_warning.patch python3 ops/docker_run.py \ - --container-tag "${CONTAINER_TAG}" \ + --image-uri "${IMAGE_URI}" \ -- bash -c \ - "cd python-package && ${python_bin} -m pip wheel --no-deps -v . --wheel-dir dist/" + "cd python-package && ${PYTHON_BIN} -m pip wheel --no-deps -v . --wheel-dir dist/" git checkout python-package/pyproject.toml python-package/xgboost/core.py # discard the patch python3 ops/docker_run.py \ - --container-tag "${CONTAINER_TAG}" \ + --image-uri "${IMAGE_URI}" \ -- auditwheel repair --only-plat \ --plat ${WHEEL_TAG} python-package/dist/*.whl python3 -m wheel tags --python-tag py3 --abi-tag none --platform ${WHEEL_TAG} --remove \ @@ -51,13 +51,13 @@ echo "--- Build binary wheel for ${WHEEL_TAG} (CPU only)" patch -p0 < ops/patch/remove_nccl_dep.patch patch -p0 < ops/patch/cpu_only_pypkg.patch python3 ops/docker_run.py \ - --container-tag "${CONTAINER_TAG}" \ + --image-uri "${IMAGE_URI}" \ -- bash -c \ - "cd python-package && ${python_bin} -m pip wheel --no-deps -v . --wheel-dir dist/" + "cd python-package && ${PYTHON_BIN} -m pip wheel --no-deps -v . --wheel-dir dist/" git checkout python-package/pyproject.toml # discard the patch python3 ops/docker_run.py \ - --container-tag "${CONTAINER_TAG}" \ + --image-uri "${IMAGE_URI}" \ -- auditwheel repair --only-plat \ --plat ${WHEEL_TAG} python-package/dist/xgboost_cpu-*.whl python3 -m wheel tags --python-tag py3 --abi-tag none --platform ${WHEEL_TAG} --remove \ diff --git a/ops/pipeline/build-test-jvm-packages.sh b/ops/pipeline/build-test-jvm-packages.sh index aea905e00294..5c1f5e8b0d4a 100755 --- a/ops/pipeline/build-test-jvm-packages.sh +++ b/ops/pipeline/build-test-jvm-packages.sh @@ -23,10 +23,10 @@ do fi done -CONTAINER_TAG=${DOCKER_REGISTRY_URL}/xgb-ci.jvm:main +IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.jvm:main set -x -python3 ops/docker_run.py --container-tag ${CONTAINER_TAG} \ +python3 ops/docker_run.py --image-uri ${IMAGE_URI} \ --run-args "-e SCALA_VERSION=${SCALA_VERSION}" \ -- ops/pipeline/build-test-jvm-packages-impl.sh diff --git a/ops/pipeline/deploy-jvm-packages.sh b/ops/pipeline/deploy-jvm-packages.sh index f76724a702cb..d73b289b165d 100755 --- a/ops/pipeline/deploy-jvm-packages.sh +++ b/ops/pipeline/deploy-jvm-packages.sh @@ -8,21 +8,21 @@ source ops/pipeline/get-docker-registry-details.sh if [[ "$#" -lt 3 ]] then - echo "Usage: $0 {cpu,gpu} [container_id] [scala_version]" + echo "Usage: $0 {cpu,gpu} [image_repo] [scala_version]" exit 1 fi variant="$1" -container_id="$2" +image_repo="$2" scala_version="$3" -CONTAINER_TAG="${DOCKER_REGISTRY_URL}/${container_id}:main" +IMAGE_URI="${DOCKER_REGISTRY_URL}/${image_repo}:main" set -x if [[ ($is_pull_request == 0) && ($is_release_branch == 1) ]] then echo "--- Deploy JVM packages to xgboost-maven-repo S3 repo" - python3 ops/docker_run.py --container-tag "${CONTAINER_TAG}" \ + python3 ops/docker_run.py --image-uri "${IMAGE_URI}" \ -- ops/pipeline/deploy-jvm-packages-impl.sh "${variant}" "${scala_version}" fi diff --git a/ops/pipeline/run-clang-tidy.sh b/ops/pipeline/run-clang-tidy.sh index 3f2019f3a330..c6166e99cb33 100755 --- a/ops/pipeline/run-clang-tidy.sh +++ b/ops/pipeline/run-clang-tidy.sh @@ -4,10 +4,10 @@ set -euo pipefail source ops/pipeline/get-docker-registry-details.sh -CONTAINER_TAG=${DOCKER_REGISTRY_URL}/xgb-ci.clang_tidy:main +IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.clang_tidy:main echo "--- Run clang-tidy" set -x python3 ops/docker_run.py \ - --container-tag ${CONTAINER_TAG} \ + --image-uri ${IMAGE_URI} \ -- python3 ops/script/run_clang_tidy.py --cuda-archs 75 diff --git a/ops/pipeline/test-cpp-gpu.sh b/ops/pipeline/test-cpp-gpu.sh index 9fdcd314264d..4262ebda7268 100755 --- a/ops/pipeline/test-cpp-gpu.sh +++ b/ops/pipeline/test-cpp-gpu.sh @@ -11,24 +11,24 @@ suite=$1 source ops/pipeline/get-docker-registry-details.sh -CONTAINER_TAG=${DOCKER_REGISTRY_URL}/xgb-ci.gpu:main +IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.gpu:main case "${suite}" in gpu) echo "--- Run Google Tests, using a single GPU" - python3 ops/docker_run.py --container-tag ${CONTAINER_TAG} --use-gpus \ + python3 ops/docker_run.py --image-uri ${IMAGE_URI} --use-gpus \ -- build/testxgboost ;; gpu-rmm) echo "--- Run Google Tests, using a single GPU, RMM enabled" - python3 ops/docker_run.py --container-tag ${CONTAINER_TAG} --use-gpus \ + python3 ops/docker_run.py --image-uri ${IMAGE_URI} --use-gpus \ -- build/testxgboost --use-rmm-pool ;; mgpu) echo "--- Run Google Tests, using multiple GPUs" - python3 ops/docker_run.py --container-tag ${CONTAINER_TAG} --use-gpus \ + python3 ops/docker_run.py --image-uri ${IMAGE_URI} --use-gpus \ --run-args='--shm-size=4g' \ -- build/testxgboost --gtest_filter=*MGPU* ;; diff --git a/ops/pipeline/test-cpp-i386.sh b/ops/pipeline/test-cpp-i386.sh index 19223041c3fb..dd7827ff2de9 100755 --- a/ops/pipeline/test-cpp-i386.sh +++ b/ops/pipeline/test-cpp-i386.sh @@ -5,9 +5,9 @@ set -euo pipefail source ops/pipeline/get-docker-registry-details.sh -CONTAINER_TAG="${DOCKER_REGISTRY_URL}/xgb-ci.i386:main" +IMAGE_URI="${DOCKER_REGISTRY_URL}/xgb-ci.i386:main" set -x python3 ops/docker_run.py \ - --container-tag ${CONTAINER_TAG} \ + --image-uri ${IMAGE_URI} \ -- bash ops/pipeline/test-cpp-i386-impl.sh diff --git a/ops/pipeline/test-jvm-gpu.sh b/ops/pipeline/test-jvm-gpu.sh index 0f517832113f..c7dfdd05b43f 100755 --- a/ops/pipeline/test-jvm-gpu.sh +++ b/ops/pipeline/test-jvm-gpu.sh @@ -25,10 +25,10 @@ done source ops/pipeline/get-docker-registry-details.sh -CONTAINER_TAG=${DOCKER_REGISTRY_URL}/xgb-ci.jvm_gpu_build:main +IMAGE_URI=${DOCKER_REGISTRY_URL}/xgb-ci.jvm_gpu_build:main set -x -python3 ops/docker_run.py --container-tag ${CONTAINER_TAG} --use-gpus \ +python3 ops/docker_run.py --image-uri ${IMAGE_URI} --use-gpus \ --run-args "-e SCALA_VERSION=${SCALA_VERSION} -e USE_CUDA=1 -e SKIP_NATIVE_BUILD=1 --shm-size=4g --privileged" \ -- ops/pipeline/build-test-jvm-packages-impl.sh diff --git a/ops/pipeline/test-python-wheel.sh b/ops/pipeline/test-python-wheel.sh index 56d54fd65d02..5551870e0b77 100755 --- a/ops/pipeline/test-python-wheel.sh +++ b/ops/pipeline/test-python-wheel.sh @@ -5,12 +5,12 @@ set -euo pipefail if [[ "$#" -lt 2 ]] then - echo "Usage: $0 {gpu|mgpu|cpu|cpu-arm64} [container_id]" + echo "Usage: $0 {gpu|mgpu|cpu|cpu-arm64} [image_repo]" exit 1 fi suite="$1" -container_id="$2" +image_repo="$2" if [[ "$suite" == "gpu" || "$suite" == "mgpu" ]] then @@ -20,9 +20,9 @@ else fi source ops/pipeline/get-docker-registry-details.sh -CONTAINER_TAG="${DOCKER_REGISTRY_URL}/${container_id}:main" +IMAGE_URI="${DOCKER_REGISTRY_URL}/${image_repo}:main" set -x -python3 ops/docker_run.py --container-tag "${CONTAINER_TAG}" ${gpu_option} \ +python3 ops/docker_run.py --image-uri "${IMAGE_URI}" ${gpu_option} \ --run-args='--shm-size=4g --privileged' \ -- bash ops/pipeline/test-python-wheel-impl.sh "${suite}"