[CI] UT logs post op and artifact

.github/scripts/ut_result_check.sh

@@ -0,0 +1,74 @@
+ut_suite=${1:-op_regression}   # op_regression / op_extended / op_ut / torch_xpu
+
+if [[ ${ut_suite} == 'op_regression' || ${ut_suite} == 'op_regression_dev1' || ${ut_suite} == 'op_extended' ]]; then
+    grep "^FAILED" ${ut_suite}_test.log | awk '{print $2}' > ./${ut_suite}_failed.log
+    grep "PASSED" ${ut_suite}_test.log | awk '{print $1}' > ./${ut_suite}_passed.log
+    num_failed=$(cat ./${ut_suite}_failed.log | wc -l)
+    num_passed=$(cat ./${ut_suite}_passed.log | wc -l)
+    echo -e "========================================================================="
+    echo -e "Show Failed cases in ${ut_suite}"
+    echo -e "========================================================================="
+    cat ./${ut_suite}_failed.log
+    if [[ $num_failed -gt 0 ]] && [[ $num_passed -lt 0 ]]; then
+      echo -e "[ERROR] UT ${ut_suite} test Fail"
+      exit 1
+    else
+      echo -e "[PASS] UT ${ut_suite} test Pass"
+    fi
+fi
+if [[ ${ut_suite} == 'op_ut' ]]; then
+    grep "^FAILED" op_ut_with_skip_test.log | awk '{print $2}' > ./${ut_suite}_with_skip_test_failed.log
+    grep "^FAILED" op_ut_with_only_test.log | awk '{print $2}' > ./${ut_suite}_with_only_test_failed.log
+    num_failed_with_skip=$(cat ./${ut_suite}_with_skip_test_failed.log | wc -l)
+    num_failed_with_only=$(cat ./${ut_suite}_with_only_test_failed.log | wc -l)
+    echo -e "========================================================================="
+    echo -e "Show Failed cases in ${ut_suite} with skip"
+    echo -e "========================================================================="
+    cat ./${ut_suite}_with_skip_test_failed.log
+    echo -e "========================================================================="
+    echo -e "Show Failed cases in ${ut_suite} with only"
+    echo -e "========================================================================="
+    cat ./${ut_suite}_with_only_test_failed.log
+    let num_failed=num_failed_with_skip+num_failed_with_only
+    grep "PASSED" op_ut_with_skip_test.log | awk '{print $1}' > ./${ut_suite}_with_skip_test_passed.log
+    grep "PASSED" op_ut_with_only_test.log | awk '{print $1}' > ./${ut_suite}_with_only_test_passed.log
+    num_passed_with_skip=$(cat ./${ut_suite}_with_skip_test_passed.log | wc -l)
+    num_passed_with_only=$(cat ./${ut_suite}_with_only_test_passed.log | wc -l)
+    let num_passed=num_passed_with_skip+num_passed_with_only
+    if [[ $num_failed -gt 0 ]] && [[ $num_passed -lt 0 ]]; then
+      echo -e "[ERROR] UT ${ut_suite} test Fail"
+      exit 1
+    else
+      echo -e "[PASS] UT ${ut_suite} test Pass"
+    fi
+fi
+if [[ ${ut_suite} == 'torch_xpu' ]]; then
+    echo "Pytorch XPU binary UT checking"
+    cd ../../pytorch
+    TEST_REPORTS_DIR=$(pwd)/test/test-reports
+    for xpu_case in build/bin/*{xpu,sycl}*; do
+      if [[ "$xpu_case" != *"*"* && "$xpu_case" != *.so && "$xpu_case" != *.a ]]; then
+        case_name=$(basename "$xpu_case")
+        cd ../ut_log/torch_xpu
+        grep -E "FAILED" binary_ut_${ut_suite}_${case_name}_test.log | awk '{print $2}' > ./binary_ut_${ut_suite}_${case_name}_failed.log
+        echo $(cat ./binary_ut_${ut_suite}_${case_name}_failed.log | wc -l) | tee -a ./binary_ut_${ut_suite}_failed_summary.log
+        grep -E "PASSED|Pass" binary_ut_${ut_suite}_${case_name}_test.log | awk '{print $2}' > ./binary_ut_${ut_suite}_${case_name}_passed.log
+        echo $(cat ./binary_ut_${ut_suite}_${case_name}_passed.log | wc -l) | tee -a ./binary_ut_${ut_suite}_passed_summary.log
+        cd -
+      fi
+    done
+    echo -e "========================================================================="
+    echo -e "Show Failed cases in ${ut_suite}"
+    echo -e "========================================================================="
+    cd ../ut_log/torch_xpu
+    cat ./binary_ut_${ut_suite}_${case_name}_failed.log
+    num_failed_binary_ut=$(awk '{sum += $1};END {print sum}' binary_ut_${ut_suite}_failed_summary.log)
+    num_passed_binary_ut=$(awk '{sum += $1};END {print sum}' binary_ut_${ut_suite}_passed_summary.log)
+    let num_failed=num_failed_binary_ut
+    if [[ $num_failed -gt 0 ]] && [[ $num_passed_binary_ut -lt 0 ]]; then
+      echo -e "[ERROR] UT ${ut_suite} test Fail"
+      exit 1
+    else
+      echo -e "[PASS] UT ${ut_suite} test Pass"
+    fi
+fi

.github/workflows/_linux_ut.yml

@@ -142,17 +142,19 @@ jobs:
         if: contains(inputs.ut, 'op_regression') || github.event_name == 'schedule'
         run: |
           cd ${{ github.workspace }}
+          mkdir -p ut_log/op_regression
           xpu-smi discovery
           source .github/scripts/env.sh ${{ inputs.pytorch }}
           source activate xpu_op_${ZE_AFFINITY_MASK}
           cd ${{ github.workspace }}
           cd test/regressions
           pip install pytest
-          timeout 8000 pytest -v
+          timeout 8000 pytest -v 2>&1 | tee ${{ github.workspace }}/ut_log/op_regression/op_regression_test.log
       - name: Run XPU OP Regressions test on device 1
         if: contains(inputs.ut, 'op_regression_dev1') || github.event_name == 'schedule'
         run: |
           cd ${{ github.workspace }}
+          mkdir -p ut_log/op_regression_dev1
           xpu-smi discovery
           source .github/scripts/env.sh ${{ inputs.pytorch }}
           source activate xpu_op_${ZE_AFFINITY_MASK}
@@ -161,35 +163,41 @@ jobs:
           cd ${{ github.workspace }}
           cd test/regressions
           pip install pytest
-          timeout 8000 pytest -v test_operation_on_device_1.py
+          timeout 8000 pytest -v test_operation_on_device_1.py 2>&1 | tee ${{ github.workspace }}/ut_log/op_regression_dev1/op_regression_dev1_test.log
           export ZE_AFFINITY_MASK=${ZE_AFFINITY_MASK_OLD}
       - name: Run XPU OP Extended UT
         if: contains(inputs.ut, 'op_extended') || github.event_name == 'schedule'
         run: |
           source .github/scripts/env.sh ${{ inputs.pytorch }}
           source activate xpu_op_${ZE_AFFINITY_MASK}
           export PYTORCH_TEST_WITH_SLOW=1
+          cd ${{ github.workspace }}
+          mkdir -p ut_log/op_extended
           cd ../pytorch/third_party/torch-xpu-ops/test/xpu/extended/
-          timeout 10000 python run_test_with_skip.py
+          timeout 10000 python run_test_with_skip.py 2>&1 | tee ${{ github.workspace }}/ut_log/op_extended/op_extended_test.log
       - name: Run XPU OP UT
         if: contains(inputs.ut, 'op_ut') || github.event_name == 'schedule'
         run: |
           source .github/scripts/env.sh ${{ inputs.pytorch }}
           source activate xpu_op_${ZE_AFFINITY_MASK}
           export PYTORCH_ENABLE_XPU_FALLBACK=1
           export PYTORCH_TEST_WITH_SLOW=1
+          cd ${{ github.workspace }}
+          mkdir -p ut_log/op_ut
           cd ../pytorch/third_party/torch-xpu-ops/test/xpu
-          timeout 10000 python run_test_with_skip.py 
+          timeout 10000 python run_test_with_skip.py 2>&1 | tee ${{ github.workspace }}/ut_log/op_ut/op_ut_with_skip_test.log
           # Cases run with a on-demand white list, since some suites are too
           # slow to go through all operators on CPU. So add cases on-demand
           # when XPU implementatoin is done.
           # test_foreach, test_decomp
-          timeout 10000 python run_test_with_only.py 
+          timeout 10000 python run_test_with_only.py 2>&1 | tee ${{ github.workspace }}/ut_log/op_ut/op_ut_with_only_test.log
       - name: Run Torch XPU UT
         if: contains(inputs.ut, 'torch_xpu') || github.event_name == 'schedule'
         run: |
           source .github/scripts/env.sh ${{ inputs.pytorch }}
           source activate xpu_op_${ZE_AFFINITY_MASK}
+          cd ${{ github.workspace }}
+          mkdir -p ut_log/torch_xpu
           cd ../pytorch
           TEST_REPORTS_DIR=$(pwd)/test/test-reports
           rm -rf "$TEST_REPORTS_DIR" && mkdir -p "$TEST_REPORTS_DIR"
@@ -198,7 +206,7 @@ jobs:
             if [[ "$xpu_case" != *"*"* && "$xpu_case" != *.so && "$xpu_case" != *.a ]]; then
               case_name=$(basename "$xpu_case")
               echo "Testing ${case_name} ..."
-              "$xpu_case" --gtest_output=xml:"$TEST_REPORTS_DIR"/"$case_name".xml 
+              "$xpu_case" --gtest_output=xml:"$TEST_REPORTS_DIR"/"$case_name".xml 2>&1 | tee ${{ github.workspace }}/ut_log/torch_xpu/binary_ut_torch_xpu_${case_name}_test.log
             fi
           done
           # Run Pytorch XPU python UT
@@ -219,4 +227,31 @@ jobs:
           if [ -f "test/test_xpu.py" ]; then
             test_cmd="${test_cmd} test_xpu.py"
           fi
-          eval $test_cmd
+          eval $test_cmd 2>&1 | tee ${{ github.workspace }}/ut_log/torch_xpu/Inductor_ut_torch_xpu_test.log
+      - name: UT Test Results Check
+        shell: bash
+        run: |
+           function contains() {
+              contains_status="echo 'Start $2 ...'"
+              {
+                [[ $1 =~ (^|,)$2($|,) ]]
+              } || {
+                echo "[Warning] $2 is not suppotted type! Skipped!"
+                contains_status="continue"
+              }
+            }
+          set -xe
+          for ut_suite in $(echo ${{ inputs.ut }} |sed 's/,/ /g')
+          do
+            contains "op_regression,op_regression_dev1,op_extended,op_ut,torch_xpu" $ut_suite
+            $contains_status
+            cd ${{ github.workspace }}/ut_log/${ut_suite}
+            cp ${{ github.workspace }}/.github/scripts/ut_result_check.sh ./
+            bash ut_result_check.sh ${ut_suite}
+          done
+      - name: Upload Inductor XPU UT Log
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: Inductor-XPU-UT-Data-${{ github.event.pull_request.number || github.sha }}-${{ inputs.abi }} 
+          path: ${{ github.workspace }}/ut_log