update build-test

.github/workflows/build-test.yml

@@ -3,34 +3,65 @@ on:
   # makes workflow reusable
   workflow_call:
     inputs:
-      build_label:
-        description: "requested runner label (specifies instance)"
+      wf_category:
+        description: "categories: REMOTE, NIGHTLY, RELEASE"
         type: string
-        required: true
-      timeout:
-        description: "time limit for run in minutes "
+        default: "REMOTE"
+      python:
+        description: "python version, e.g. 3.10.12"
         type: string
         required: true
-      gitref:
-        description: "git commit hash or branch name"
+      # build related parameters
+      build_label:
+        description: "requested runner label (specifies instance)"
         type: string
-        required: true
+        default: "gcp-build-static"
+      build_timeout:
+        description: "time limit for build in minutes "
+        type: string
+        default: "60"
       Gi_per_thread:
         description: 'requested GiB to reserve per thread'
         type: string
-        required: true
+        default: "1"
       nvcc_threads:
         description: "number of threads nvcc build threads"
         type: string
+        default: "4"
+      # test related parameters
+      test_label_solo:
+        description: "requested runner label (specifies instance)"
+        type: string
         required: true
-      python:
-        description: "python version, e.g. 3.10.12"
+      test_label_multi:
+        description: "requested runner label (specifies instance)"
+        type: string
+        required: true
+      test_timeout:
+        description: "time limit for test run in minutes "
+        type: string
+        required: true
+      gitref:
+        description: "git commit hash or branch name"
         type: string
         required: true
       test_skip_list:
         description: 'file containing tests to skip'
         type: string
         required: true
+      # benchmark related parameters
+      benchmark_label:
+        description: "requested benchmark label (specifies instance)"
+        type: string
+        default: ""
+      benchmark_config_list_file:
+        description: "benchmark configs file, e.g. 'nm_benchmark_nightly_configs_list.txt'"
+        type: string
+        required: true
+      benchmark_timeout:
+        description: "time limit for benchmarking"
+        type: string
+        default: "720"
 
   # makes workflow manually callable
   workflow_dispatch:
@@ -39,8 +70,20 @@ on:
         description: "requested runner label (specifies instance)"
         type: string
         required: true
-      timeout:
-        description: "time limit for run in minutes "
+      build_timeout:
+        description: "time limit for build in minutes "
+        type: string
+        required: true
+      test_label_solo:
+        description: "requested runner label (specifies instance)"
+        type: string
+        required: true
+      test_label_multi:
+        description: "requested runner label (specifies instance)"
+        type: string
+        required: true
+      test_timeout:
+        description: "time limit for test run in minutes "
         type: string
         required: true
       gitref:
@@ -70,25 +113,65 @@ jobs:
         uses: ./.github/workflows/build.yml
         with:
             build_label: ${{ inputs.build_label }}
-            timeout: ${{ inputs.timeout }}
-            gitref: ${{ inputs.gitref }}
+            timeout: ${{ inputs.build_timeout }}
+            gitref: ${{ github.ref }}
             Gi_per_thread: ${{ inputs.Gi_per_thread }}
             nvcc_threads: ${{ inputs.nvcc_threads }}
             python: ${{ inputs.python }}
         secrets: inherit
 
-    TEST:
+    TEST-SOLO:
+        needs: [BUILD]
+        if: success() && ${{ inputs.wf_category == 'NIGHTLY' || inputs.wf_category == 'RELEASE' }}
+        uses: ./.github/workflows/test.yml
+        with:
+            test_label: ${{ inputs.test_label_solo }}
+            timeout: ${{ inputs.test_timeout }}
+            gitref: ${{ github.ref }}
+            python: ${{ inputs.python }}
+            whl: ${{ needs.BUILD.outputs.whl }}
+            test_skip_list: ${{ inputs.test_skip_list }}
+        secrets: inherit
+
+    # TODO: this should only run if doing a NIGHTLY or RELEASE
+    TEST-MULTI:
         needs: [BUILD]
         if: success()
-        strategy:
-            matrix:
-                test_label: [aws-avx2-192G-4-a10g-96G]
         uses: ./.github/workflows/test.yml
         with:
-            test_label: ${{ matrix.test_label }}
-            timeout: ${{ inputs.timeout }}
-            gitref: ${{ inputs.gitref }}
+            test_label: ${{ inputs.test_label_multi }}
+            timeout: ${{ inputs.test_timeout }}
+            gitref: ${{ github.ref }}
             python: ${{ inputs.python }}
             whl: ${{ needs.BUILD.outputs.whl }}
             test_skip_list: ${{ inputs.test_skip_list }}
         secrets: inherit
+
+    BENCHMARK:
+        needs: [BUILD]
+        if: success()
+        uses: ./.github/workflows/nm-benchmark.yml
+        with:
+            label: ${{ inputs.test_label_solo }}
+            benchmark_config_list_file:  ./.github/data/nm_benchmark_nightly_configs_list.txt
+            timeout: ${{ inputs.benchmark_timeout }}
+            gitref: ${{ github.ref }}
+            python: ${{ inputs.python }}
+            # Always push if it is a scheduled job
+            push_benchmark_results_to_gh_pages: "${{ github.event_name == 'schedule' || inputs.push_benchmark_results_to_gh_pages }}"
+        secrets: inherit
+
+    # TODO: decide if this should build or use the whl
+    # single gpu
+    # TODO: this should only run if doing a NIGHTLY or RELEASE
+    Accuracy-Smoke-AWS-AVX2-32G-A10G-24G:
+        if: ${{ inputs.wf_category == 'NIGHTLY' || inputs.wf_category == 'RELEASE' }}
+        uses: ./.github/workflows/nm-lm-eval-smoke.yml
+        with:
+            label: ${{ inputs.test_label_solo }}
+            timeout: ${{ inputs.benchmark_timeout }}
+            gitref: ${{ github.ref }}
+            Gi_per_thread: ${{ inputs.Gi_per_thread }}
+            nvcc_threads: ${{ inputs.nvcc_threads }}
+            python: ${{ inputs.python }}
+        secrets: inherit