Improve CI usage by increasing parallelism

.github/workflows/build.yaml

@@ -34,6 +34,7 @@ jobs:
       branch: ${{ inputs.branch }}
       date: ${{ inputs.date }}
       sha: ${{ inputs.sha }}
+      script: "ci/cpp/build/build_cpp.sh"
   python-build:
     needs: [cpp-build]
     secrets: inherit
@@ -65,7 +66,7 @@ jobs:
       container_image: "rapidsai/ci-conda:latest"
       date: ${{ inputs.date }}
       node_type: "gpu-l4-latest-1"
-      run_script: "ci/build_docs.sh"
+      run_script: "ci/docs/build_docs.sh"
       sha: ${{ inputs.sha }}
   wheel-build-libcudf:
     secrets: inherit
@@ -77,7 +78,7 @@ jobs:
       branch: ${{ inputs.branch }}
       sha: ${{ inputs.sha }}
       date: ${{ inputs.date }}
-      script: ci/build_wheel_libcudf.sh
+      script: ci/python/build/build_wheel_libcudf.sh
   wheel-publish-libcudf:
     needs: wheel-build-libcudf
     secrets: inherit
@@ -98,7 +99,7 @@ jobs:
       branch: ${{ inputs.branch }}
       sha: ${{ inputs.sha }}
       date: ${{ inputs.date }}
-      script: ci/build_wheel_pylibcudf.sh
+      script: ci/python/build/build_wheel_pylibcudf.sh
   wheel-publish-pylibcudf:
     needs: wheel-build-pylibcudf
     secrets: inherit
@@ -119,7 +120,7 @@ jobs:
       branch: ${{ inputs.branch }}
       sha: ${{ inputs.sha }}
       date: ${{ inputs.date }}
-      script: ci/build_wheel_cudf.sh
+      script: ci/python/build/build_wheel_cudf.sh
   wheel-publish-cudf:
     needs: wheel-build-cudf
     secrets: inherit
@@ -142,7 +143,7 @@ jobs:
       branch: ${{ inputs.branch }}
       sha: ${{ inputs.sha }}
       date: ${{ inputs.date }}
-      script: ci/build_wheel_dask_cudf.sh
+      script: ci/python/build/build_wheel_dask_cudf.sh
   wheel-publish-dask-cudf:
     needs: wheel-build-dask-cudf
     secrets: inherit
@@ -165,7 +166,7 @@ jobs:
       branch: ${{ inputs.branch }}
       sha: ${{ inputs.sha }}
       date: ${{ inputs.date }}
-      script: ci/build_wheel_cudf_polars.sh
+      script: ci/python/build/build_wheel_cudf_polars.sh
   wheel-publish-cudf-polars:
     needs: wheel-build-cudf-polars
     secrets: inherit

.github/workflows/pr.yaml

@@ -86,6 +86,9 @@ jobs:
           - '!java/**'
           - '!notebooks/**'
           - '!python/**'
+          - '!ci/python/**'
+          - '!ci/java/**'
+          - '!ci/docs/**'
         test_cudf_pandas:
           - '**'
           - '!.devcontainer/**'
@@ -105,6 +108,8 @@ jobs:
           - '!img/**'
           - '!notebooks/**'
           - '!python/**'
+          - '!ci/python/**'
+          - '!ci/docs/**'
         test_notebooks:
           - '**'
           - '!.devcontainer/**'
@@ -122,6 +127,8 @@ jobs:
           - '!img/**'
           - '!java/**'
           - '!notebooks/**'
+          - '!ci/java/**'
+          - '!ci/docs/**'
   checks:
     secrets: inherit
     needs: telemetry-setup
@@ -136,13 +143,14 @@ jobs:
     with:
       build_type: pull-request
       node_type: "cpu16"
+      script: "ci/cpp/build/build_cpp.sh"
   cpp-linters:
     secrets: inherit
     needs: checks
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     with:
       build_type: pull-request
-      run_script: "ci/cpp_linters.sh"
+      run_script: "ci/cpp/test/cpp_linters.sh"
       node_type: "cpu16"
   conda-cpp-checks:
     needs: conda-cpp-build
@@ -158,20 +166,22 @@ jobs:
     if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp
     with:
       build_type: pull-request
+      script: "ci/cpp/test/test_cpp.sh"
   conda-python-build:
     needs: conda-cpp-build
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     with:
       build_type: pull-request
+      script: "ci/python/build/build_python.sh"
   conda-python-cudf-tests:
     needs: [conda-python-build, changed-files]
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python
     with:
       build_type: pull-request
-      script: "ci/test_python_cudf.sh"
+      script: "ci/python/test/test_python_cudf.sh"
   conda-python-other-tests:
     # Tests for dask_cudf, custreamz, cudf_kafka are separated for CI parallelism
     needs: [conda-python-build, changed-files]
@@ -180,7 +190,7 @@ jobs:
     if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python
     with:
       build_type: pull-request
-      script: "ci/test_python_other.sh"
+      script: "ci/python/test/test_python_other.sh"
   conda-java-tests:
     needs: [conda-cpp-build, changed-files]
     secrets: inherit
@@ -191,7 +201,7 @@ jobs:
       node_type: "gpu-l4-latest-1"
       arch: "amd64"
       container_image: "rapidsai/ci-conda:latest"
-      run_script: "ci/test_java.sh"
+      run_script: "ci/java/test/test_java.sh"
   static-configure:
     needs: checks
     secrets: inherit
@@ -201,7 +211,7 @@ jobs:
       # Use the wheel container so we can skip conda solves and since our
       # primary static consumers (Spark) are not in conda anyway.
       container_image: "rapidsai/ci-wheel:latest"
-      run_script: "ci/configure_cpp_static.sh"
+      run_script: "ci/cpp/build/configure_cpp_static.sh"
   conda-notebook-tests:
     needs: [conda-python-build, changed-files]
     secrets: inherit
@@ -212,7 +222,7 @@ jobs:
       node_type: "gpu-l4-latest-1"
       arch: "amd64"
       container_image: "rapidsai/ci-conda:latest"
-      run_script: "ci/test_notebooks.sh"
+      run_script: "ci/python/test/test_notebooks.sh"
   docs-build:
     needs: conda-python-build
     secrets: inherit
@@ -222,7 +232,7 @@ jobs:
       node_type: "gpu-l4-latest-1"
       arch: "amd64"
       container_image: "rapidsai/ci-conda:latest"
-      run_script: "ci/build_docs.sh"
+      run_script: "ci/docs/build_docs.sh"
   wheel-build-libcudf:
     needs: checks
     secrets: inherit
@@ -232,29 +242,29 @@ jobs:
       matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber)))
       build_type: pull-request
       node_type: "cpu16"
-      script: "ci/build_wheel_libcudf.sh"
+      script: "ci/python/build/build_wheel_libcudf.sh"
   wheel-build-pylibcudf:
     needs: [checks, wheel-build-libcudf]
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     with:
       build_type: pull-request
-      script: "ci/build_wheel_pylibcudf.sh"
+      script: "ci/python/build/build_wheel_pylibcudf.sh"
   wheel-build-cudf:
     needs: wheel-build-pylibcudf
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     with:
       build_type: pull-request
-      script: "ci/build_wheel_cudf.sh"
+      script: "ci/python/build/build_wheel_cudf.sh"
   wheel-tests-cudf:
     needs: [wheel-build-cudf, changed-files]
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
     if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python
     with:
       build_type: pull-request
-      script: ci/test_wheel_cudf.sh
+      script: ci/python/test/test_wheel_cudf.sh
   wheel-build-cudf-polars:
     needs: wheel-build-pylibcudf
     secrets: inherit
@@ -263,7 +273,7 @@ jobs:
       # This selects "ARCH=amd64 + the latest supported Python + CUDA".
       matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))]))
       build_type: pull-request
-      script: "ci/build_wheel_cudf_polars.sh"
+      script: "ci/python/build/build_wheel_cudf_polars.sh"
   wheel-tests-cudf-polars:
     needs: [wheel-build-cudf-polars, changed-files]
     secrets: inherit
@@ -273,7 +283,7 @@ jobs:
       # This selects "ARCH=amd64 + the latest supported Python + CUDA".
       matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))]))
       build_type: pull-request
-      script: "ci/test_wheel_cudf_polars.sh"
+      script: "ci/python/test/test_wheel_cudf_polars.sh"
   cudf-polars-polars-tests:
     needs: wheel-build-cudf-polars
     secrets: inherit
@@ -282,7 +292,7 @@ jobs:
       # This selects "ARCH=amd64 + the latest supported Python + CUDA".
       matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))]))
       build_type: pull-request
-      script: "ci/test_cudf_polars_polars_tests.sh"
+      script: "ci/python/test/test_cudf_polars_polars_tests.sh"
   wheel-build-dask-cudf:
     needs: wheel-build-cudf
     secrets: inherit
@@ -291,7 +301,7 @@ jobs:
       # This selects "ARCH=amd64 + the latest supported Python + CUDA".
       matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))]))
       build_type: pull-request
-      script: "ci/build_wheel_dask_cudf.sh"
+      script: "ci/python/build/build_wheel_dask_cudf.sh"
   wheel-tests-dask-cudf:
     needs: [wheel-build-dask-cudf, changed-files]
     secrets: inherit
@@ -301,7 +311,7 @@ jobs:
       # This selects "ARCH=amd64 + the latest supported Python + CUDA".
       matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))]))
       build_type: pull-request
-      script: ci/test_wheel_dask_cudf.sh
+      script: ci/python/test/test_wheel_dask_cudf.sh
   devcontainer:
     secrets: inherit
     needs: telemetry-setup

.github/workflows/test.yaml

@@ -34,6 +34,7 @@ jobs:
       branch: ${{ inputs.branch }}
       date: ${{ inputs.date }}
       sha: ${{ inputs.sha }}
+      script: "ci/cpp/test/test_cpp.sh"
   conda-cpp-memcheck-tests:
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
@@ -45,7 +46,7 @@ jobs:
       node_type: "gpu-l4-latest-1"
       arch: "amd64"
       container_image: "rapidsai/ci-conda:latest"
-      run_script: "ci/test_cpp_memcheck.sh"
+      run_script: "ci/cpp/test/test_cpp_memcheck.sh"
   static-configure:
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
@@ -57,7 +58,7 @@ jobs:
       # Use the wheel container so we can skip conda solves and since our
       # primary static consumers (Spark) are not in conda anyway.
       container_image: "rapidsai/ci-wheel:latest"
-      run_script: "ci/configure_cpp_static.sh"
+      run_script: "ci/cpp/build/configure_cpp_static.sh"
   cpp-linters:
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
@@ -66,7 +67,7 @@ jobs:
       branch: ${{ inputs.branch }}
       date: ${{ inputs.date }}
       sha: ${{ inputs.sha }}
-      run_script: "ci/cpp_linters.sh"
+      run_script: "ci/cpp/test/cpp_linters.sh"
       file_to_upload: iwyu_results.txt
   conda-python-cudf-tests:
     secrets: inherit
@@ -76,7 +77,7 @@ jobs:
       branch: ${{ inputs.branch }}
       date: ${{ inputs.date }}
       sha: ${{ inputs.sha }}
-      script: "ci/test_python_cudf.sh"
+      script: "ci/python/test/test_python_cudf.sh"
   conda-python-other-tests:
     # Tests for dask_cudf, custreamz, cudf_kafka are separated for CI parallelism
     secrets: inherit
@@ -86,7 +87,7 @@ jobs:
       branch: ${{ inputs.branch }}
       date: ${{ inputs.date }}
       sha: ${{ inputs.sha }}
-      script: "ci/test_python_other.sh"
+      script: "ci/python/test/test_python_other.sh"
   conda-java-tests:
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
@@ -98,7 +99,7 @@ jobs:
       node_type: "gpu-l4-latest-1"
       arch: "amd64"
       container_image: "rapidsai/ci-conda:latest"
-      run_script: "ci/test_java.sh"
+      run_script: "ci/java/test/test_java.sh"
   conda-notebook-tests:
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
@@ -110,7 +111,7 @@ jobs:
       node_type: "gpu-l4-latest-1"
       arch: "amd64"
       container_image: "rapidsai/ci-conda:latest"
-      run_script: "ci/test_notebooks.sh"
+      run_script: "ci/python/test/test_notebooks.sh"
   wheel-tests-cudf:
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
@@ -119,7 +120,7 @@ jobs:
       branch: ${{ inputs.branch }}
       date: ${{ inputs.date }}
       sha: ${{ inputs.sha }}
-      script: ci/test_wheel_cudf.sh
+      script: ci/python/test/test_wheel_cudf.sh
   wheel-tests-dask-cudf:
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
@@ -128,7 +129,7 @@ jobs:
       branch: ${{ inputs.branch }}
       date: ${{ inputs.date }}
       sha: ${{ inputs.sha }}
-      script: ci/test_wheel_dask_cudf.sh
+      script: ci/python/test/test_wheel_dask_cudf.sh
   unit-tests-cudf-pandas:
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
@@ -158,7 +159,7 @@ jobs:
       branch: ${{ inputs.branch }}
       date: ${{ inputs.date }}
       sha: ${{ inputs.sha }}
-      script: "ci/test_wheel_cudf_polars.sh"
+      script: "ci/python/test/test_wheel_cudf_polars.sh"
   cudf-polars-polars-tests:
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/[email protected]
@@ -167,4 +168,4 @@ jobs:
       branch: ${{ inputs.branch }}
       date: ${{ inputs.date }}
       sha: ${{ inputs.sha }}
-      script: "ci/test_cudf_polars_polars_tests.sh"
+      script: "ci/python/test/test_cudf_polars_polars_tests.sh"

.gitignore

@@ -22,6 +22,8 @@ compile_commands.json
 dask-worker-space/
 dist/
 cudf.egg-info/
+!ci/python/build
+!ci/python/*/build
 python/build
 python/*/build
 python/cudf/cudf-coverage.xml
@@ -77,6 +79,7 @@ test-results
 CMakeFiles/
 Debug
 build/
+!ci/cpp/build/
 cpp/build/
 cpp/examples/*/install/
 cpp/examples/*/build/