From d765237e6584cfc10baf33d04dcbc1bc4aa447bf Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 14 Oct 2024 14:00:56 -0700 Subject: [PATCH 01/11] build wheels without build isolation --- .github/workflows/pr.yaml | 189 +------------------ ci/build_wheel.sh | 34 ++++ ci/build_wheel_cugraph.sh | 12 -- dependencies.yaml | 42 +++++ python/cugraph-dgl/pyproject.toml | 2 + python/cugraph-equivariant/pyproject.toml | 2 + python/cugraph-pyg/pyproject.toml | 2 + python/cugraph-service/client/pyproject.toml | 2 + python/cugraph-service/server/pyproject.toml | 2 + python/nx-cugraph/pyproject.toml | 2 + 10 files changed, 92 insertions(+), 197 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index b0a1308237e..67fb1aac7c9 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -12,132 +12,14 @@ concurrency: jobs: pr-builder: needs: - - changed-files - - checks - - conda-cpp-build - - conda-cpp-tests - - conda-cpp-checks - - conda-notebook-tests - - conda-python-build - - conda-python-tests - - docs-build - wheel-build-pylibcugraph - - wheel-tests-pylibcugraph - wheel-build-cugraph - - wheel-tests-cugraph - wheel-build-nx-cugraph - - wheel-tests-nx-cugraph - wheel-build-cugraph-dgl - - wheel-tests-cugraph-dgl - wheel-build-cugraph-pyg - - wheel-tests-cugraph-pyg - wheel-build-cugraph-equivariant - - wheel-tests-cugraph-equivariant - - devcontainer secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.12 - if: always() - with: - needs: ${{ toJSON(needs) }} - changed-files: - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@branch-24.12 - with: - files_yaml: | - test_cpp: - - '**' - - '!.devcontainers/**' - - '!CONTRIBUTING.md' - - '!README.md' - - '!docs/**' - - '!img/**' - - '!mg_utils/**' - - '!notebooks/**' - - '!python/**' - - '!readme_pages/**' - # TODO: Remove this before merging - - '!.github/**' - test_notebooks: - - '**' - - '!.devcontainers/**' - - '!CONTRIBUTING.md' - - '!README.md' - - '!docs/**' - # TODO: Remove this before merging - - '!.github/**' - test_python: - - '**' - - '!.devcontainers/**' - - '!CONTRIBUTING.md' - - '!README.md' - - '!docs/**' - - '!img/**' - - '!notebooks/**' - # TODO: Remove this before merging - - '!.github/**' - checks: - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.12 - with: - enable_check_generated_files: false - conda-cpp-build: - needs: checks - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.12 - with: - build_type: pull-request - node_type: cpu32 - conda-cpp-tests: - needs: [conda-cpp-build, changed-files] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.12 - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp - with: - build_type: pull-request - conda-cpp-checks: - needs: conda-cpp-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.12 - with: - build_type: pull-request - enable_check_symbols: true - symbol_exclusions: (cugraph::ops|hornet|void writeEdgeCountsKernel|void markUniqueOffsetsKernel) - conda-python-build: - needs: conda-cpp-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.12 - with: - build_type: pull-request - conda-python-tests: - needs: [conda-python-build, changed-files] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.12 - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python - with: - build_type: pull-request - conda-notebook-tests: - needs: [conda-python-build, changed-files] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.12 - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_notebooks - with: - build_type: pull-request - node_type: "gpu-v100-latest-1" - arch: "amd64" - container_image: "rapidsai/ci-conda:cuda11.8.0-ubuntu22.04-py3.10" - run_script: "ci/test_notebooks.sh" - docs-build: - needs: conda-python-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.12 - with: - build_type: pull-request - node_type: "gpu-v100-latest-1" - arch: "amd64" - container_image: "rapidsai/ci-conda:cuda11.8.0-ubuntu22.04-py3.10" - run_script: "ci/build_docs.sh" wheel-build-pylibcugraph: - needs: checks secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 with: @@ -147,16 +29,8 @@ jobs: extra-repo-sha: branch-24.12 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY node_type: cpu32 - wheel-tests-pylibcugraph: - needs: [wheel-build-pylibcugraph, changed-files] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python - with: - build_type: pull-request - script: ci/test_wheel_pylibcugraph.sh wheel-build-cugraph: - needs: wheel-tests-pylibcugraph + needs: wheel-build-pylibcugraph secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 with: @@ -165,85 +39,30 @@ jobs: extra-repo: rapidsai/cugraph-ops extra-repo-sha: branch-24.12 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY - wheel-tests-cugraph: - needs: [wheel-build-cugraph, changed-files] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python - with: - build_type: pull-request - script: ci/test_wheel_cugraph.sh wheel-build-nx-cugraph: - needs: wheel-tests-pylibcugraph + needs: wheel-build-cugraph secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 with: build_type: pull-request script: ci/build_wheel_nx-cugraph.sh - wheel-tests-nx-cugraph: - needs: [wheel-build-nx-cugraph, changed-files] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python - with: - build_type: pull-request - script: ci/test_wheel_nx-cugraph.sh wheel-build-cugraph-dgl: - needs: wheel-tests-cugraph + needs: wheel-build-cugraph secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 with: build_type: pull-request script: ci/build_wheel_cugraph-dgl.sh - wheel-tests-cugraph-dgl: - needs: [wheel-build-cugraph-dgl, changed-files] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python - with: - build_type: pull-request - script: ci/test_wheel_cugraph-dgl.sh - matrix_filter: map(select(.ARCH == "amd64")) wheel-build-cugraph-pyg: - needs: wheel-tests-cugraph + needs: wheel-build-cugraph secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 with: build_type: pull-request script: ci/build_wheel_cugraph-pyg.sh - wheel-tests-cugraph-pyg: - needs: [wheel-build-cugraph-pyg, changed-files] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python - with: - build_type: pull-request - script: ci/test_wheel_cugraph-pyg.sh - matrix_filter: map(select(.ARCH == "amd64")) wheel-build-cugraph-equivariant: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 with: build_type: pull-request script: ci/build_wheel_cugraph-equivariant.sh - wheel-tests-cugraph-equivariant: - needs: [wheel-build-cugraph-equivariant, changed-files] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python - with: - build_type: pull-request - script: ci/test_wheel_cugraph-equivariant.sh - matrix_filter: map(select(.ARCH == "amd64")) - devcontainer: - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.12 - with: - arch: '["amd64"]' - cuda: '["12.5"]' - node_type: cpu32 - extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY - build_command: | - sccache -z; - build-all --verbose -j$(nproc --ignore=1) -DBUILD_CUGRAPH_MG_TESTS=ON; - sccache -s; diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index f3979ab3049..c12325e3547 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -11,18 +11,52 @@ source rapids-date-string RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +touch /tmp/requirements-build.txt + +if [[ "${package_name}" == "cuspatial" ]]; then + # Download the pylibcugraph wheel built in the previous step and ensure + # it's installed in the build environment. + PYLIBCUGRAPH_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python /tmp/pylibcugraph_dist) + echo ${PYLIBCUGRAPH_WHEELHOUSE}/pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}*.whl >> /tmp/requirements-build.txt +fi + +rapids-logger "Generating build requirements" +declare -r matrix_selectors="cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};cuda_suffixed=true" + +rapids-dependency-file-generator \ + --output requirements \ + --file-key "py_build_${package_name/-/_}" \ + --matrix "${matrix_selectors}" \ +| tee -a /tmp/requirements-build.txt + +rapids-dependency-file-generator \ + --output requirements \ + --file-key "py_rapids_build_${package_name/-/_}" \ + --matrix "${matrix_selectors}" \ +| tee -a /tmp/requirements-build.txt + +rapids-logger "Installing build requirements" +python -m pip install \ + -v \ + --prefer-binary \ + -r /tmp/requirements-build.txt + rapids-generate-version > ./VERSION cd "${package_dir}" +rapids-logger "Building '${package_name}' wheel" python -m pip wheel \ -w dist \ -v \ + --no-build-isolation \ --no-deps \ --disable-pip-version-check \ --extra-index-url https://pypi.nvidia.com \ . +sccache --show-adv-stats + # pure-python packages should be marked as pure, and not have auditwheel run on them. if [[ ${package_name} == "nx-cugraph" ]] || \ [[ ${package_name} == "cugraph-dgl" ]] || \ diff --git a/ci/build_wheel_cugraph.sh b/ci/build_wheel_cugraph.sh index 20d9bf47e3e..f03a4f5780b 100755 --- a/ci/build_wheel_cugraph.sh +++ b/ci/build_wheel_cugraph.sh @@ -5,18 +5,6 @@ set -euo pipefail RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -# Download the pylibcugraph wheel built in the previous step and make it -# available for pip to find. -# -# ensure 'cugraph' wheel builds always use the 'pylibcugraph' just built in the same CI run -# -# using env variable PIP_CONSTRAINT is necessary to ensure the constraints -# are used when creating the isolated build environment -CPP_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 /tmp/pylibcugraph_dist) - -echo "pylibcugraph-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${CPP_WHEELHOUSE}/pylibcugraph_*.whl)" > ./constraints.txt -export PIP_CONSTRAINT="${PWD}/constraints.txt" - PARALLEL_LEVEL=$(python -c \ "from math import ceil; from multiprocessing import cpu_count; print(ceil(cpu_count()/4))") case "${RAPIDS_CUDA_VERSION}" in diff --git a/dependencies.yaml b/dependencies.yaml index a4143ff90c9..222e4597b15 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -158,6 +158,13 @@ files: includes: - python_build_rapids - python_build_wheel + py_rapids_build_nx_cugraph: + output: pyproject + pyproject_dir: python/nx-cugraph + extras: + table: tool.rapids-build-backend + key: requires + includes: [] py_run_nx_cugraph: output: pyproject pyproject_dir: python/nx-cugraph @@ -184,6 +191,13 @@ files: includes: - python_build_rapids - python_build_wheel + py_rapids_build_cugraph_dgl: + output: pyproject + pyproject_dir: python/cugraph-dgl + extras: + table: tool.rapids-build-backend + key: requires + includes: [] py_run_cugraph_dgl: output: pyproject pyproject_dir: python/cugraph-dgl @@ -211,6 +225,13 @@ files: includes: - python_build_rapids - python_build_wheel + py_rapids_build_cugraph_pyg: + output: pyproject + pyproject_dir: python/cugraph-pyg + extras: + table: tool.rapids-build-backend + key: requires + includes: [] py_run_cugraph_pyg: output: pyproject pyproject_dir: python/cugraph-pyg @@ -238,6 +259,13 @@ files: includes: - python_build_rapids - python_build_wheel + py_rapids_build_cugraph_equivariant: + output: pyproject + pyproject_dir: python/cugraph-equivariant + extras: + table: tool.rapids-build-backend + key: requires + includes: [] py_run_cugraph_equivariant: output: pyproject pyproject_dir: python/cugraph-equivariant @@ -262,6 +290,13 @@ files: includes: - python_build_rapids - python_build_wheel + py_rapids_build_cugraph_service_client: + output: pyproject + pyproject_dir: python/cugraph-service/client + extras: + table: tool.rapids-build-backend + key: requires + includes: [] py_run_cugraph_service_client: output: pyproject pyproject_dir: python/cugraph-service/client @@ -277,6 +312,13 @@ files: includes: - python_build_rapids - python_build_wheel + py_rapids_build_cugraph_service_server: + output: pyproject + pyproject_dir: python/cugraph-service/server + extras: + table: tool.rapids-build-backend + key: requires + includes: [] py_run_cugraph_service_server: output: pyproject pyproject_dir: python/cugraph-service/server diff --git a/python/cugraph-dgl/pyproject.toml b/python/cugraph-dgl/pyproject.toml index e3e12216ac7..0c96f01da5e 100644 --- a/python/cugraph-dgl/pyproject.toml +++ b/python/cugraph-dgl/pyproject.toml @@ -62,3 +62,5 @@ include = [ build-backend = "setuptools.build_meta" dependencies-file = "../../dependencies.yaml" matrix-entry = "cuda_suffixed=true" +requires = [ +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/cugraph-equivariant/pyproject.toml b/python/cugraph-equivariant/pyproject.toml index 7713e89ac20..f66bfc1fb1f 100644 --- a/python/cugraph-equivariant/pyproject.toml +++ b/python/cugraph-equivariant/pyproject.toml @@ -70,3 +70,5 @@ include = [ build-backend = "setuptools.build_meta" dependencies-file = "../../dependencies.yaml" matrix-entry = "cuda_suffixed=true" +requires = [ +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/cugraph-pyg/pyproject.toml b/python/cugraph-pyg/pyproject.toml index e157f36f8f6..65a62046e86 100644 --- a/python/cugraph-pyg/pyproject.toml +++ b/python/cugraph-pyg/pyproject.toml @@ -68,3 +68,5 @@ include = [ build-backend = "setuptools.build_meta" dependencies-file = "../../dependencies.yaml" matrix-entry = "cuda_suffixed=true" +requires = [ +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/cugraph-service/client/pyproject.toml b/python/cugraph-service/client/pyproject.toml index ac5e6bad0d5..92ce8d8fcfd 100644 --- a/python/cugraph-service/client/pyproject.toml +++ b/python/cugraph-service/client/pyproject.toml @@ -50,3 +50,5 @@ build-backend = "setuptools.build_meta" dependencies-file = "../../../dependencies.yaml" disable-cuda = true matrix-entry = "cuda_suffixed=true" +requires = [ +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/cugraph-service/server/pyproject.toml b/python/cugraph-service/server/pyproject.toml index f388fd4c126..e6c22c306cf 100644 --- a/python/cugraph-service/server/pyproject.toml +++ b/python/cugraph-service/server/pyproject.toml @@ -78,3 +78,5 @@ include = [ build-backend = "setuptools.build_meta" dependencies-file = "../../../dependencies.yaml" matrix-entry = "cuda_suffixed=true" +requires = [ +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/nx-cugraph/pyproject.toml b/python/nx-cugraph/pyproject.toml index d145aa549da..1dd4c90e40a 100644 --- a/python/nx-cugraph/pyproject.toml +++ b/python/nx-cugraph/pyproject.toml @@ -86,6 +86,8 @@ build-backend = "setuptools.build_meta" commit-files = ["_nx_cugraph/GIT_COMMIT"] dependencies-file = "../../dependencies.yaml" matrix-entry = "cuda_suffixed=true" +requires = [ +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [tool.black] line-length = 88 From 92da9432ff70c277b2d91b8ecb7498fafd28c237 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Mon, 14 Oct 2024 14:06:44 -0700 Subject: [PATCH 02/11] fix pr-builder --- .github/workflows/pr.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 67fb1aac7c9..8b12ea2a381 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -19,6 +19,10 @@ jobs: - wheel-build-cugraph-pyg - wheel-build-cugraph-equivariant secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.12 + if: always() + with: + needs: ${{ toJSON(needs) }} wheel-build-pylibcugraph: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 From f2d7290b1a7f2fa7e6e6e9191ad3bc5fb261d3ed Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 15 Oct 2024 08:29:46 -0500 Subject: [PATCH 03/11] changes from cuspatial review --- ci/build_wheel.sh | 11 +---------- ci/build_wheel_cugraph.sh | 12 ++++++++++++ 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index c12325e3547..95e3e8d9ef1 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -11,17 +11,8 @@ source rapids-date-string RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -touch /tmp/requirements-build.txt - -if [[ "${package_name}" == "cuspatial" ]]; then - # Download the pylibcugraph wheel built in the previous step and ensure - # it's installed in the build environment. - PYLIBCUGRAPH_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python /tmp/pylibcugraph_dist) - echo ${PYLIBCUGRAPH_WHEELHOUSE}/pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}*.whl >> /tmp/requirements-build.txt -fi - rapids-logger "Generating build requirements" -declare -r matrix_selectors="cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};cuda_suffixed=true" +matrix_selectors="cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};cuda_suffixed=true" rapids-dependency-file-generator \ --output requirements \ diff --git a/ci/build_wheel_cugraph.sh b/ci/build_wheel_cugraph.sh index f03a4f5780b..20d9bf47e3e 100755 --- a/ci/build_wheel_cugraph.sh +++ b/ci/build_wheel_cugraph.sh @@ -5,6 +5,18 @@ set -euo pipefail RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +# Download the pylibcugraph wheel built in the previous step and make it +# available for pip to find. +# +# ensure 'cugraph' wheel builds always use the 'pylibcugraph' just built in the same CI run +# +# using env variable PIP_CONSTRAINT is necessary to ensure the constraints +# are used when creating the isolated build environment +CPP_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 /tmp/pylibcugraph_dist) + +echo "pylibcugraph-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${CPP_WHEELHOUSE}/pylibcugraph_*.whl)" > ./constraints.txt +export PIP_CONSTRAINT="${PWD}/constraints.txt" + PARALLEL_LEVEL=$(python -c \ "from math import ceil; from multiprocessing import cpu_count; print(ceil(cpu_count()/4))") case "${RAPIDS_CUDA_VERSION}" in From 5e21e83311f8a9ca1f2b6af9611364448cfa6141 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 15 Oct 2024 12:40:21 -0500 Subject: [PATCH 04/11] enable all CI, make scripts look more like other RAPIDS repos --- .github/workflows/pr.yaml | 185 +++++++++++++++++++++++++++++++++++++- ci/build_wheel.sh | 7 +- 2 files changed, 185 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 8b12ea2a381..b0a1308237e 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -12,18 +12,132 @@ concurrency: jobs: pr-builder: needs: + - changed-files + - checks + - conda-cpp-build + - conda-cpp-tests + - conda-cpp-checks + - conda-notebook-tests + - conda-python-build + - conda-python-tests + - docs-build - wheel-build-pylibcugraph + - wheel-tests-pylibcugraph - wheel-build-cugraph + - wheel-tests-cugraph - wheel-build-nx-cugraph + - wheel-tests-nx-cugraph - wheel-build-cugraph-dgl + - wheel-tests-cugraph-dgl - wheel-build-cugraph-pyg + - wheel-tests-cugraph-pyg - wheel-build-cugraph-equivariant + - wheel-tests-cugraph-equivariant + - devcontainer secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.12 if: always() with: needs: ${{ toJSON(needs) }} + changed-files: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@branch-24.12 + with: + files_yaml: | + test_cpp: + - '**' + - '!.devcontainers/**' + - '!CONTRIBUTING.md' + - '!README.md' + - '!docs/**' + - '!img/**' + - '!mg_utils/**' + - '!notebooks/**' + - '!python/**' + - '!readme_pages/**' + # TODO: Remove this before merging + - '!.github/**' + test_notebooks: + - '**' + - '!.devcontainers/**' + - '!CONTRIBUTING.md' + - '!README.md' + - '!docs/**' + # TODO: Remove this before merging + - '!.github/**' + test_python: + - '**' + - '!.devcontainers/**' + - '!CONTRIBUTING.md' + - '!README.md' + - '!docs/**' + - '!img/**' + - '!notebooks/**' + # TODO: Remove this before merging + - '!.github/**' + checks: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.12 + with: + enable_check_generated_files: false + conda-cpp-build: + needs: checks + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.12 + with: + build_type: pull-request + node_type: cpu32 + conda-cpp-tests: + needs: [conda-cpp-build, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.12 + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp + with: + build_type: pull-request + conda-cpp-checks: + needs: conda-cpp-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.12 + with: + build_type: pull-request + enable_check_symbols: true + symbol_exclusions: (cugraph::ops|hornet|void writeEdgeCountsKernel|void markUniqueOffsetsKernel) + conda-python-build: + needs: conda-cpp-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.12 + with: + build_type: pull-request + conda-python-tests: + needs: [conda-python-build, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.12 + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python + with: + build_type: pull-request + conda-notebook-tests: + needs: [conda-python-build, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.12 + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_notebooks + with: + build_type: pull-request + node_type: "gpu-v100-latest-1" + arch: "amd64" + container_image: "rapidsai/ci-conda:cuda11.8.0-ubuntu22.04-py3.10" + run_script: "ci/test_notebooks.sh" + docs-build: + needs: conda-python-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.12 + with: + build_type: pull-request + node_type: "gpu-v100-latest-1" + arch: "amd64" + container_image: "rapidsai/ci-conda:cuda11.8.0-ubuntu22.04-py3.10" + run_script: "ci/build_docs.sh" wheel-build-pylibcugraph: + needs: checks secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 with: @@ -33,8 +147,16 @@ jobs: extra-repo-sha: branch-24.12 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY node_type: cpu32 + wheel-tests-pylibcugraph: + needs: [wheel-build-pylibcugraph, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python + with: + build_type: pull-request + script: ci/test_wheel_pylibcugraph.sh wheel-build-cugraph: - needs: wheel-build-pylibcugraph + needs: wheel-tests-pylibcugraph secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 with: @@ -43,30 +165,85 @@ jobs: extra-repo: rapidsai/cugraph-ops extra-repo-sha: branch-24.12 extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY + wheel-tests-cugraph: + needs: [wheel-build-cugraph, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python + with: + build_type: pull-request + script: ci/test_wheel_cugraph.sh wheel-build-nx-cugraph: - needs: wheel-build-cugraph + needs: wheel-tests-pylibcugraph secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 with: build_type: pull-request script: ci/build_wheel_nx-cugraph.sh + wheel-tests-nx-cugraph: + needs: [wheel-build-nx-cugraph, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python + with: + build_type: pull-request + script: ci/test_wheel_nx-cugraph.sh wheel-build-cugraph-dgl: - needs: wheel-build-cugraph + needs: wheel-tests-cugraph secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 with: build_type: pull-request script: ci/build_wheel_cugraph-dgl.sh + wheel-tests-cugraph-dgl: + needs: [wheel-build-cugraph-dgl, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python + with: + build_type: pull-request + script: ci/test_wheel_cugraph-dgl.sh + matrix_filter: map(select(.ARCH == "amd64")) wheel-build-cugraph-pyg: - needs: wheel-build-cugraph + needs: wheel-tests-cugraph secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 with: build_type: pull-request script: ci/build_wheel_cugraph-pyg.sh + wheel-tests-cugraph-pyg: + needs: [wheel-build-cugraph-pyg, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python + with: + build_type: pull-request + script: ci/test_wheel_cugraph-pyg.sh + matrix_filter: map(select(.ARCH == "amd64")) wheel-build-cugraph-equivariant: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 with: build_type: pull-request script: ci/build_wheel_cugraph-equivariant.sh + wheel-tests-cugraph-equivariant: + needs: [wheel-build-cugraph-equivariant, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python + with: + build_type: pull-request + script: ci/test_wheel_cugraph-equivariant.sh + matrix_filter: map(select(.ARCH == "amd64")) + devcontainer: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.12 + with: + arch: '["amd64"]' + cuda: '["12.5"]' + node_type: cpu32 + extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY + build_command: | + sccache -z; + build-all --verbose -j$(nproc --ignore=1) -DBUILD_CUGRAPH_MG_TESTS=ON; + sccache -s; diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 95e3e8d9ef1..31aa323e454 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -5,6 +5,7 @@ set -euo pipefail package_name=$1 package_dir=$2 +underscore_package_name=$(echo "${package_name}" | tr "-" "_") source rapids-configure-sccache source rapids-date-string @@ -16,13 +17,13 @@ matrix_selectors="cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VER rapids-dependency-file-generator \ --output requirements \ - --file-key "py_build_${package_name/-/_}" \ + --file-key "py_build_${underscore_package_name}" \ --matrix "${matrix_selectors}" \ -| tee -a /tmp/requirements-build.txt +| tee /tmp/requirements-build.txt rapids-dependency-file-generator \ --output requirements \ - --file-key "py_rapids_build_${package_name/-/_}" \ + --file-key "py_rapids_build_${underscore_package_name}" \ --matrix "${matrix_selectors}" \ | tee -a /tmp/requirements-build.txt From 6c7d930125cd84ab2655421cbf8847046142579d Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 23 Oct 2024 13:37:43 -0500 Subject: [PATCH 05/11] print sccache stats --- .pre-commit-config.yaml | 2 +- ci/build_cpp.sh | 2 ++ ci/build_python.sh | 5 +++++ ci/build_wheel.sh | 28 +++------------------------- ci/build_wheel_cugraph.sh | 33 ++++++++++++++++++++++++--------- ci/build_wheel_pylibcugraph.sh | 23 ++++++++++++++++++++++- 6 files changed, 57 insertions(+), 36 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3687562b48e..b5fbcf9ad42 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -55,7 +55,7 @@ repos: setup[.]cfg$ - id: verify-alpha-spec - repo: https://github.com/rapidsai/dependency-file-generator - rev: v1.13.11 + rev: v1.16.0 hooks: - id: rapids-dependency-file-generator args: ["--clean"] diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index 859977d76d5..93e5c680d07 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -17,4 +17,6 @@ rapids-logger "Begin cpp build" RAPIDS_PACKAGE_VERSION=$(rapids-generate-version) rapids-conda-retry mambabuild conda/recipes/libcugraph +sccache --show-adv-stats + rapids-upload-conda-to-s3 cpp diff --git a/ci/build_python.sh b/ci/build_python.sh index c94cc2a0fce..b49d5f534aa 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -27,12 +27,17 @@ rapids-conda-retry mambabuild \ --channel "${CPP_CHANNEL}" \ conda/recipes/pylibcugraph +sccache --show-adv-stats +sccache --zero-stats + rapids-conda-retry mambabuild \ --no-test \ --channel "${CPP_CHANNEL}" \ --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ conda/recipes/cugraph +sccache --show-adv-stats + # NOTE: nothing in nx-cugraph is CUDA-specific, but it is built on each CUDA # platform to ensure it is included in each set of artifacts, since test # scripts only install from one set of artifacts based on the CUDA version used diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 31aa323e454..958a56bbe77 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -5,43 +5,21 @@ set -euo pipefail package_name=$1 package_dir=$2 -underscore_package_name=$(echo "${package_name}" | tr "-" "_") source rapids-configure-sccache source rapids-date-string RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -rapids-logger "Generating build requirements" -matrix_selectors="cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};cuda_suffixed=true" - -rapids-dependency-file-generator \ - --output requirements \ - --file-key "py_build_${underscore_package_name}" \ - --matrix "${matrix_selectors}" \ -| tee /tmp/requirements-build.txt - -rapids-dependency-file-generator \ - --output requirements \ - --file-key "py_rapids_build_${underscore_package_name}" \ - --matrix "${matrix_selectors}" \ -| tee -a /tmp/requirements-build.txt - -rapids-logger "Installing build requirements" -python -m pip install \ - -v \ - --prefer-binary \ - -r /tmp/requirements-build.txt - rapids-generate-version > ./VERSION cd "${package_dir}" -rapids-logger "Building '${package_name}' wheel" +sccache --zero-stats + python -m pip wheel \ -w dist \ -v \ - --no-build-isolation \ --no-deps \ --disable-pip-version-check \ --extra-index-url https://pypi.nvidia.com \ @@ -54,7 +32,7 @@ if [[ ${package_name} == "nx-cugraph" ]] || \ [[ ${package_name} == "cugraph-dgl" ]] || \ [[ ${package_name} == "cugraph-pyg" ]] || \ [[ ${package_name} == "cugraph-equivariant" ]]; then - RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-upload-wheels-to-s3 dist + RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-upload-wheels-to-s3 python dist else case "${RAPIDS_CUDA_VERSION}" in 12.*) diff --git a/ci/build_wheel_cugraph.sh b/ci/build_wheel_cugraph.sh index 20d9bf47e3e..694d32c9e83 100755 --- a/ci/build_wheel_cugraph.sh +++ b/ci/build_wheel_cugraph.sh @@ -3,22 +3,37 @@ set -euo pipefail -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +package_name="pylibcugraph" -# Download the pylibcugraph wheel built in the previous step and make it -# available for pip to find. -# -# ensure 'cugraph' wheel builds always use the 'pylibcugraph' just built in the same CI run +rapids-logger "Generating build requirements" + +rapids-dependency-file-generator \ + --output requirements \ + --file-key "py_build_${package_name}" \ + --file-key "py_rapids_build_${package_name}" \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};cuda_suffixed=true" \ +| tee /tmp/requirements-build.txt + +# Download the pylibcugraph wheel built in the previous step and install it in the build environment. # -# using env variable PIP_CONSTRAINT is necessary to ensure the constraints -# are used when creating the isolated build environment +# Ensures 'cugraph' wheel builds always use the 'pylibcugraph' just built in the same CI run. +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" CPP_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 /tmp/pylibcugraph_dist) +echo "$(echo ${CPP_WHEELHOUSE}/pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" >> /tmp/requirements-build.txt + +rapids-logger "Installing build requirements" +python -m pip install \ + -v \ + --prefer-binary \ + -r /tmp/requirements-build.txt -echo "pylibcugraph-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${CPP_WHEELHOUSE}/pylibcugraph_*.whl)" > ./constraints.txt -export PIP_CONSTRAINT="${PWD}/constraints.txt" +# build with '--no-build-isolation', for better sccache hit rate +# 0 really means "add --no-build-isolation" (ref: https://github.com/pypa/pip/issues/5735) +export PIP_NO_BUILD_ISOLATION=0 PARALLEL_LEVEL=$(python -c \ "from math import ceil; from multiprocessing import cpu_count; print(ceil(cpu_count()/4))") + case "${RAPIDS_CUDA_VERSION}" in 12.*) EXTRA_CMAKE_ARGS=";-DUSE_CUDA_MATH_WHEELS=ON" diff --git a/ci/build_wheel_pylibcugraph.sh b/ci/build_wheel_pylibcugraph.sh index fa967b0be29..f7a9b72e21e 100755 --- a/ci/build_wheel_pylibcugraph.sh +++ b/ci/build_wheel_pylibcugraph.sh @@ -3,6 +3,27 @@ set -euo pipefail +package_name="pylibcugraph" + +rapids-logger "Generating build requirements" + +rapids-dependency-file-generator \ + --output requirements \ + --file-key "py_build_${package_name}" \ + --file-key "py_rapids_build_${package_name}" \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};cuda_suffixed=true" \ +| tee /tmp/requirements-build.txt + +rapids-logger "Installing build requirements" +python -m pip install \ + -v \ + --prefer-binary \ + -r /tmp/requirements-build.txt + +# build with '--no-build-isolation', for better sccache hit rate +# 0 really means "add --no-build-isolation" (ref: https://github.com/pypa/pip/issues/5735) +export PIP_NO_BUILD_ISOLATION=0 + PARALLEL_LEVEL=$(python -c \ "from math import ceil; from multiprocessing import cpu_count; print(ceil(cpu_count()/4))") @@ -18,4 +39,4 @@ esac export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DFIND_CUGRAPH_CPP=OFF;-DCPM_cugraph-ops_SOURCE=${GITHUB_WORKSPACE}/cugraph-ops/${EXTRA_CMAKE_ARGS}" export SKBUILD_BUILD_TOOL_ARGS="-j${PARALLEL_LEVEL};-l${PARALLEL_LEVEL}" -./ci/build_wheel.sh pylibcugraph python/pylibcugraph +./ci/build_wheel.sh "${package_name}" python/pylibcugraph From 7dbee7d95d7d5274cf902e4ea1877a44975a5071 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 23 Oct 2024 13:40:17 -0500 Subject: [PATCH 06/11] revert dependencies.yaml changes --- dependencies.yaml | 42 -------------------- python/cugraph-dgl/pyproject.toml | 2 - python/cugraph-equivariant/pyproject.toml | 2 - python/cugraph-pyg/pyproject.toml | 2 - python/cugraph-service/client/pyproject.toml | 2 - python/cugraph-service/server/pyproject.toml | 2 - python/nx-cugraph/pyproject.toml | 2 - 7 files changed, 54 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 222e4597b15..a4143ff90c9 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -158,13 +158,6 @@ files: includes: - python_build_rapids - python_build_wheel - py_rapids_build_nx_cugraph: - output: pyproject - pyproject_dir: python/nx-cugraph - extras: - table: tool.rapids-build-backend - key: requires - includes: [] py_run_nx_cugraph: output: pyproject pyproject_dir: python/nx-cugraph @@ -191,13 +184,6 @@ files: includes: - python_build_rapids - python_build_wheel - py_rapids_build_cugraph_dgl: - output: pyproject - pyproject_dir: python/cugraph-dgl - extras: - table: tool.rapids-build-backend - key: requires - includes: [] py_run_cugraph_dgl: output: pyproject pyproject_dir: python/cugraph-dgl @@ -225,13 +211,6 @@ files: includes: - python_build_rapids - python_build_wheel - py_rapids_build_cugraph_pyg: - output: pyproject - pyproject_dir: python/cugraph-pyg - extras: - table: tool.rapids-build-backend - key: requires - includes: [] py_run_cugraph_pyg: output: pyproject pyproject_dir: python/cugraph-pyg @@ -259,13 +238,6 @@ files: includes: - python_build_rapids - python_build_wheel - py_rapids_build_cugraph_equivariant: - output: pyproject - pyproject_dir: python/cugraph-equivariant - extras: - table: tool.rapids-build-backend - key: requires - includes: [] py_run_cugraph_equivariant: output: pyproject pyproject_dir: python/cugraph-equivariant @@ -290,13 +262,6 @@ files: includes: - python_build_rapids - python_build_wheel - py_rapids_build_cugraph_service_client: - output: pyproject - pyproject_dir: python/cugraph-service/client - extras: - table: tool.rapids-build-backend - key: requires - includes: [] py_run_cugraph_service_client: output: pyproject pyproject_dir: python/cugraph-service/client @@ -312,13 +277,6 @@ files: includes: - python_build_rapids - python_build_wheel - py_rapids_build_cugraph_service_server: - output: pyproject - pyproject_dir: python/cugraph-service/server - extras: - table: tool.rapids-build-backend - key: requires - includes: [] py_run_cugraph_service_server: output: pyproject pyproject_dir: python/cugraph-service/server diff --git a/python/cugraph-dgl/pyproject.toml b/python/cugraph-dgl/pyproject.toml index 0c96f01da5e..e3e12216ac7 100644 --- a/python/cugraph-dgl/pyproject.toml +++ b/python/cugraph-dgl/pyproject.toml @@ -62,5 +62,3 @@ include = [ build-backend = "setuptools.build_meta" dependencies-file = "../../dependencies.yaml" matrix-entry = "cuda_suffixed=true" -requires = [ -] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/cugraph-equivariant/pyproject.toml b/python/cugraph-equivariant/pyproject.toml index f66bfc1fb1f..7713e89ac20 100644 --- a/python/cugraph-equivariant/pyproject.toml +++ b/python/cugraph-equivariant/pyproject.toml @@ -70,5 +70,3 @@ include = [ build-backend = "setuptools.build_meta" dependencies-file = "../../dependencies.yaml" matrix-entry = "cuda_suffixed=true" -requires = [ -] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/cugraph-pyg/pyproject.toml b/python/cugraph-pyg/pyproject.toml index 65a62046e86..e157f36f8f6 100644 --- a/python/cugraph-pyg/pyproject.toml +++ b/python/cugraph-pyg/pyproject.toml @@ -68,5 +68,3 @@ include = [ build-backend = "setuptools.build_meta" dependencies-file = "../../dependencies.yaml" matrix-entry = "cuda_suffixed=true" -requires = [ -] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/cugraph-service/client/pyproject.toml b/python/cugraph-service/client/pyproject.toml index 92ce8d8fcfd..ac5e6bad0d5 100644 --- a/python/cugraph-service/client/pyproject.toml +++ b/python/cugraph-service/client/pyproject.toml @@ -50,5 +50,3 @@ build-backend = "setuptools.build_meta" dependencies-file = "../../../dependencies.yaml" disable-cuda = true matrix-entry = "cuda_suffixed=true" -requires = [ -] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/cugraph-service/server/pyproject.toml b/python/cugraph-service/server/pyproject.toml index e6c22c306cf..f388fd4c126 100644 --- a/python/cugraph-service/server/pyproject.toml +++ b/python/cugraph-service/server/pyproject.toml @@ -78,5 +78,3 @@ include = [ build-backend = "setuptools.build_meta" dependencies-file = "../../../dependencies.yaml" matrix-entry = "cuda_suffixed=true" -requires = [ -] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/nx-cugraph/pyproject.toml b/python/nx-cugraph/pyproject.toml index 1dd4c90e40a..d145aa549da 100644 --- a/python/nx-cugraph/pyproject.toml +++ b/python/nx-cugraph/pyproject.toml @@ -86,8 +86,6 @@ build-backend = "setuptools.build_meta" commit-files = ["_nx_cugraph/GIT_COMMIT"] dependencies-file = "../../dependencies.yaml" matrix-entry = "cuda_suffixed=true" -requires = [ -] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. [tool.black] line-length = 88 From 3aa957255c546a1bbeb7084c235545c2fecf21cd Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 23 Oct 2024 13:41:34 -0500 Subject: [PATCH 07/11] fix cugraph wheel name --- ci/build_wheel_cugraph.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/build_wheel_cugraph.sh b/ci/build_wheel_cugraph.sh index 694d32c9e83..e2b2e9a664f 100755 --- a/ci/build_wheel_cugraph.sh +++ b/ci/build_wheel_cugraph.sh @@ -3,7 +3,7 @@ set -euo pipefail -package_name="pylibcugraph" +package_name="cugraph" rapids-logger "Generating build requirements" @@ -46,4 +46,4 @@ esac export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DFIND_CUGRAPH_CPP=OFF;-DCPM_cugraph-ops_SOURCE=${GITHUB_WORKSPACE}/cugraph-ops/${EXTRA_CMAKE_ARGS}" export SKBUILD_BUILD_TOOL_ARGS="-j${PARALLEL_LEVEL};-l${PARALLEL_LEVEL}" -./ci/build_wheel.sh cugraph python/cugraph +./ci/build_wheel.sh "${package_name}" python/cugraph From 9537bbf86733cab1b362d94997f7eb0ca4cb39ff Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 23 Oct 2024 13:42:06 -0500 Subject: [PATCH 08/11] revert whitespace --- ci/build_wheel_cugraph.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/build_wheel_cugraph.sh b/ci/build_wheel_cugraph.sh index e2b2e9a664f..4849b9b9b70 100755 --- a/ci/build_wheel_cugraph.sh +++ b/ci/build_wheel_cugraph.sh @@ -33,7 +33,6 @@ export PIP_NO_BUILD_ISOLATION=0 PARALLEL_LEVEL=$(python -c \ "from math import ceil; from multiprocessing import cpu_count; print(ceil(cpu_count()/4))") - case "${RAPIDS_CUDA_VERSION}" in 12.*) EXTRA_CMAKE_ARGS=";-DUSE_CUDA_MATH_WHEELS=ON" From 8dcd93fd5ca18d188b29bfc0937853636b8de099 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 23 Oct 2024 13:45:10 -0500 Subject: [PATCH 09/11] one more package type for rapids-upload-wheels-to-s3 --- ci/build_wheel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 958a56bbe77..2bbd986cc47 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -52,5 +52,5 @@ else mkdir -p final_dist python -m auditwheel repair -w final_dist "${EXCLUDE_ARGS[@]}" dist/* - RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist + RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python final_dist fi From 8b1e174e58eed9be5f6588714cfd711b90bf2270 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 23 Oct 2024 14:40:36 -0500 Subject: [PATCH 10/11] make wheel scripts consistent with other RAPIDS repos --- ci/build_wheel.sh | 2 ++ ci/build_wheel_cugraph.sh | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 2bbd986cc47..944975c55ee 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -17,6 +17,8 @@ cd "${package_dir}" sccache --zero-stats +rapids-logger "Building '${package_name}' wheel" + python -m pip wheel \ -w dist \ -v \ diff --git a/ci/build_wheel_cugraph.sh b/ci/build_wheel_cugraph.sh index 4849b9b9b70..0666fb1154e 100755 --- a/ci/build_wheel_cugraph.sh +++ b/ci/build_wheel_cugraph.sh @@ -14,7 +14,7 @@ rapids-dependency-file-generator \ --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};cuda_suffixed=true" \ | tee /tmp/requirements-build.txt -# Download the pylibcugraph wheel built in the previous step and install it in the build environment. +# Download the pylibcugraph wheel built in the previous step and ensure it gets installed the build environment. # # Ensures 'cugraph' wheel builds always use the 'pylibcugraph' just built in the same CI run. RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" From a0c5139e993eea3aa00aa84e0e08fecaaf5db3de Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 29 Oct 2024 09:21:11 -0500 Subject: [PATCH 11/11] revert build-isolation stuff --- ci/build_python.sh | 2 ++ ci/build_wheel_cugraph.sh | 34 ++++++++++------------------------ ci/build_wheel_pylibcugraph.sh | 23 +---------------------- 3 files changed, 13 insertions(+), 46 deletions(-) diff --git a/ci/build_python.sh b/ci/build_python.sh index b49d5f534aa..af9c3c56403 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -20,6 +20,8 @@ export RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-logger "Begin py build" +sccache --zero-stats + # TODO: Remove `--no-test` flags once importing on a CPU # node works correctly rapids-conda-retry mambabuild \ diff --git a/ci/build_wheel_cugraph.sh b/ci/build_wheel_cugraph.sh index 0666fb1154e..20d9bf47e3e 100755 --- a/ci/build_wheel_cugraph.sh +++ b/ci/build_wheel_cugraph.sh @@ -3,33 +3,19 @@ set -euo pipefail -package_name="cugraph" - -rapids-logger "Generating build requirements" - -rapids-dependency-file-generator \ - --output requirements \ - --file-key "py_build_${package_name}" \ - --file-key "py_rapids_build_${package_name}" \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};cuda_suffixed=true" \ -| tee /tmp/requirements-build.txt +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -# Download the pylibcugraph wheel built in the previous step and ensure it gets installed the build environment. +# Download the pylibcugraph wheel built in the previous step and make it +# available for pip to find. # -# Ensures 'cugraph' wheel builds always use the 'pylibcugraph' just built in the same CI run. -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +# ensure 'cugraph' wheel builds always use the 'pylibcugraph' just built in the same CI run +# +# using env variable PIP_CONSTRAINT is necessary to ensure the constraints +# are used when creating the isolated build environment CPP_WHEELHOUSE=$(RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 /tmp/pylibcugraph_dist) -echo "$(echo ${CPP_WHEELHOUSE}/pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" >> /tmp/requirements-build.txt - -rapids-logger "Installing build requirements" -python -m pip install \ - -v \ - --prefer-binary \ - -r /tmp/requirements-build.txt -# build with '--no-build-isolation', for better sccache hit rate -# 0 really means "add --no-build-isolation" (ref: https://github.com/pypa/pip/issues/5735) -export PIP_NO_BUILD_ISOLATION=0 +echo "pylibcugraph-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo ${CPP_WHEELHOUSE}/pylibcugraph_*.whl)" > ./constraints.txt +export PIP_CONSTRAINT="${PWD}/constraints.txt" PARALLEL_LEVEL=$(python -c \ "from math import ceil; from multiprocessing import cpu_count; print(ceil(cpu_count()/4))") @@ -45,4 +31,4 @@ esac export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DFIND_CUGRAPH_CPP=OFF;-DCPM_cugraph-ops_SOURCE=${GITHUB_WORKSPACE}/cugraph-ops/${EXTRA_CMAKE_ARGS}" export SKBUILD_BUILD_TOOL_ARGS="-j${PARALLEL_LEVEL};-l${PARALLEL_LEVEL}" -./ci/build_wheel.sh "${package_name}" python/cugraph +./ci/build_wheel.sh cugraph python/cugraph diff --git a/ci/build_wheel_pylibcugraph.sh b/ci/build_wheel_pylibcugraph.sh index f7a9b72e21e..fa967b0be29 100755 --- a/ci/build_wheel_pylibcugraph.sh +++ b/ci/build_wheel_pylibcugraph.sh @@ -3,27 +3,6 @@ set -euo pipefail -package_name="pylibcugraph" - -rapids-logger "Generating build requirements" - -rapids-dependency-file-generator \ - --output requirements \ - --file-key "py_build_${package_name}" \ - --file-key "py_rapids_build_${package_name}" \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};cuda_suffixed=true" \ -| tee /tmp/requirements-build.txt - -rapids-logger "Installing build requirements" -python -m pip install \ - -v \ - --prefer-binary \ - -r /tmp/requirements-build.txt - -# build with '--no-build-isolation', for better sccache hit rate -# 0 really means "add --no-build-isolation" (ref: https://github.com/pypa/pip/issues/5735) -export PIP_NO_BUILD_ISOLATION=0 - PARALLEL_LEVEL=$(python -c \ "from math import ceil; from multiprocessing import cpu_count; print(ceil(cpu_count()/4))") @@ -39,4 +18,4 @@ esac export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DFIND_CUGRAPH_CPP=OFF;-DCPM_cugraph-ops_SOURCE=${GITHUB_WORKSPACE}/cugraph-ops/${EXTRA_CMAKE_ARGS}" export SKBUILD_BUILD_TOOL_ARGS="-j${PARALLEL_LEVEL};-l${PARALLEL_LEVEL}" -./ci/build_wheel.sh "${package_name}" python/pylibcugraph +./ci/build_wheel.sh pylibcugraph python/pylibcugraph