diff --git a/.github/scripts/ut_result_check.sh b/.github/scripts/ut_result_check.sh new file mode 100644 index 000000000..b1dd7d4a4 --- /dev/null +++ b/.github/scripts/ut_result_check.sh @@ -0,0 +1,74 @@ +ut_suite=${1:-op_regression} # op_regression / op_extended / op_ut / torch_xpu + +if [[ ${ut_suite} == 'op_regression' || ${ut_suite} == 'op_regression_dev1' || ${ut_suite} == 'op_extended' ]]; then + grep "^FAILED" ${ut_suite}_test.log | awk '{print $2}' > ./${ut_suite}_failed.log + grep "PASSED" ${ut_suite}_test.log | awk '{print $1}' > ./${ut_suite}_passed.log + num_failed=$(cat ./${ut_suite}_failed.log | wc -l) + num_passed=$(cat ./${ut_suite}_passed.log | wc -l) + echo -e "=========================================================================" + echo -e "Show Failed cases in ${ut_suite}" + echo -e "=========================================================================" + cat ./${ut_suite}_failed.log + if [[ $num_failed -gt 0 ]] && [[ $num_passed -lt 0 ]]; then + echo -e "[ERROR] UT ${ut_suite} test Fail" + exit 1 + else + echo -e "[PASS] UT ${ut_suite} test Pass" + fi +fi +if [[ ${ut_suite} == 'op_ut' ]]; then + grep "^FAILED" op_ut_with_skip_test.log | awk '{print $2}' > ./${ut_suite}_with_skip_test_failed.log + grep "^FAILED" op_ut_with_only_test.log | awk '{print $2}' > ./${ut_suite}_with_only_test_failed.log + num_failed_with_skip=$(cat ./${ut_suite}_with_skip_test_failed.log | wc -l) + num_failed_with_only=$(cat ./${ut_suite}_with_only_test_failed.log | wc -l) + echo -e "=========================================================================" + echo -e "Show Failed cases in ${ut_suite} with skip" + echo -e "=========================================================================" + cat ./${ut_suite}_with_skip_test_failed.log + echo -e "=========================================================================" + echo -e "Show Failed cases in ${ut_suite} with only" + echo -e "=========================================================================" + cat ./${ut_suite}_with_only_test_failed.log + let num_failed=num_failed_with_skip+num_failed_with_only + grep "PASSED" op_ut_with_skip_test.log | awk '{print $1}' > ./${ut_suite}_with_skip_test_passed.log + grep "PASSED" op_ut_with_only_test.log | awk '{print $1}' > ./${ut_suite}_with_only_test_passed.log + num_passed_with_skip=$(cat ./${ut_suite}_with_skip_test_passed.log | wc -l) + num_passed_with_only=$(cat ./${ut_suite}_with_only_test_passed.log | wc -l) + let num_passed=num_passed_with_skip+num_passed_with_only + if [[ $num_failed -gt 0 ]] && [[ $num_passed -lt 0 ]]; then + echo -e "[ERROR] UT ${ut_suite} test Fail" + exit 1 + else + echo -e "[PASS] UT ${ut_suite} test Pass" + fi +fi +if [[ ${ut_suite} == 'torch_xpu' ]]; then + echo "Pytorch XPU binary UT checking" + cd ../../pytorch + TEST_REPORTS_DIR=$(pwd)/test/test-reports + for xpu_case in build/bin/*{xpu,sycl}*; do + if [[ "$xpu_case" != *"*"* && "$xpu_case" != *.so && "$xpu_case" != *.a ]]; then + case_name=$(basename "$xpu_case") + cd ../ut_log/torch_xpu + grep -E "FAILED" binary_ut_${ut_suite}_${case_name}_test.log | awk '{print $2}' > ./binary_ut_${ut_suite}_${case_name}_failed.log + echo $(cat ./binary_ut_${ut_suite}_${case_name}_failed.log | wc -l) | tee -a ./binary_ut_${ut_suite}_failed_summary.log + grep -E "PASSED|Pass" binary_ut_${ut_suite}_${case_name}_test.log | awk '{print $2}' > ./binary_ut_${ut_suite}_${case_name}_passed.log + echo $(cat ./binary_ut_${ut_suite}_${case_name}_passed.log | wc -l) | tee -a ./binary_ut_${ut_suite}_passed_summary.log + cd - + fi + done + echo -e "=========================================================================" + echo -e "Show Failed cases in ${ut_suite}" + echo -e "=========================================================================" + cd ../ut_log/torch_xpu + cat ./binary_ut_${ut_suite}_${case_name}_failed.log + num_failed_binary_ut=$(awk '{sum += $1};END {print sum}' binary_ut_${ut_suite}_failed_summary.log) + num_passed_binary_ut=$(awk '{sum += $1};END {print sum}' binary_ut_${ut_suite}_passed_summary.log) + let num_failed=num_failed_binary_ut + if [[ $num_failed -gt 0 ]] && [[ $num_passed_binary_ut -lt 0 ]]; then + echo -e "[ERROR] UT ${ut_suite} test Fail" + exit 1 + else + echo -e "[PASS] UT ${ut_suite} test Pass" + fi +fi diff --git a/.github/workflows/_linux_ut.yml b/.github/workflows/_linux_ut.yml index b724d4259..7f958535e 100644 --- a/.github/workflows/_linux_ut.yml +++ b/.github/workflows/_linux_ut.yml @@ -142,17 +142,19 @@ jobs: if: contains(inputs.ut, 'op_regression') || github.event_name == 'schedule' run: | cd ${{ github.workspace }} + mkdir -p ut_log/op_regression xpu-smi discovery source .github/scripts/env.sh ${{ inputs.pytorch }} source activate xpu_op_${ZE_AFFINITY_MASK} cd ${{ github.workspace }} cd test/regressions pip install pytest - timeout 8000 pytest -v + timeout 8000 pytest -v 2>&1 | tee ${{ github.workspace }}/ut_log/op_regression/op_regression_test.log - name: Run XPU OP Regressions test on device 1 if: contains(inputs.ut, 'op_regression_dev1') || github.event_name == 'schedule' run: | cd ${{ github.workspace }} + mkdir -p ut_log/op_regression_dev1 xpu-smi discovery source .github/scripts/env.sh ${{ inputs.pytorch }} source activate xpu_op_${ZE_AFFINITY_MASK} @@ -161,7 +163,7 @@ jobs: cd ${{ github.workspace }} cd test/regressions pip install pytest - timeout 8000 pytest -v test_operation_on_device_1.py + timeout 8000 pytest -v test_operation_on_device_1.py 2>&1 | tee ${{ github.workspace }}/ut_log/op_regression_dev1/op_regression_dev1_test.log export ZE_AFFINITY_MASK=${ZE_AFFINITY_MASK_OLD} - name: Run XPU OP Extended UT if: contains(inputs.ut, 'op_extended') || github.event_name == 'schedule' @@ -169,8 +171,10 @@ jobs: source .github/scripts/env.sh ${{ inputs.pytorch }} source activate xpu_op_${ZE_AFFINITY_MASK} export PYTORCH_TEST_WITH_SLOW=1 + cd ${{ github.workspace }} + mkdir -p ut_log/op_extended cd ../pytorch/third_party/torch-xpu-ops/test/xpu/extended/ - timeout 10000 python run_test_with_skip.py + timeout 10000 python run_test_with_skip.py 2>&1 | tee ${{ github.workspace }}/ut_log/op_extended/op_extended_test.log - name: Run XPU OP UT if: contains(inputs.ut, 'op_ut') || github.event_name == 'schedule' run: | @@ -178,18 +182,22 @@ jobs: source activate xpu_op_${ZE_AFFINITY_MASK} export PYTORCH_ENABLE_XPU_FALLBACK=1 export PYTORCH_TEST_WITH_SLOW=1 + cd ${{ github.workspace }} + mkdir -p ut_log/op_ut cd ../pytorch/third_party/torch-xpu-ops/test/xpu - timeout 10000 python run_test_with_skip.py + timeout 10000 python run_test_with_skip.py 2>&1 | tee ${{ github.workspace }}/ut_log/op_ut/op_ut_with_skip_test.log # Cases run with a on-demand white list, since some suites are too # slow to go through all operators on CPU. So add cases on-demand # when XPU implementatoin is done. # test_foreach, test_decomp - timeout 10000 python run_test_with_only.py + timeout 10000 python run_test_with_only.py 2>&1 | tee ${{ github.workspace }}/ut_log/op_ut/op_ut_with_only_test.log - name: Run Torch XPU UT if: contains(inputs.ut, 'torch_xpu') || github.event_name == 'schedule' run: | source .github/scripts/env.sh ${{ inputs.pytorch }} source activate xpu_op_${ZE_AFFINITY_MASK} + cd ${{ github.workspace }} + mkdir -p ut_log/torch_xpu cd ../pytorch TEST_REPORTS_DIR=$(pwd)/test/test-reports rm -rf "$TEST_REPORTS_DIR" && mkdir -p "$TEST_REPORTS_DIR" @@ -198,7 +206,7 @@ jobs: if [[ "$xpu_case" != *"*"* && "$xpu_case" != *.so && "$xpu_case" != *.a ]]; then case_name=$(basename "$xpu_case") echo "Testing ${case_name} ..." - "$xpu_case" --gtest_output=xml:"$TEST_REPORTS_DIR"/"$case_name".xml + "$xpu_case" --gtest_output=xml:"$TEST_REPORTS_DIR"/"$case_name".xml 2>&1 | tee ${{ github.workspace }}/ut_log/torch_xpu/binary_ut_torch_xpu_${case_name}_test.log fi done # Run Pytorch XPU python UT @@ -219,4 +227,31 @@ jobs: if [ -f "test/test_xpu.py" ]; then test_cmd="${test_cmd} test_xpu.py" fi - eval $test_cmd + eval $test_cmd 2>&1 | tee ${{ github.workspace }}/ut_log/torch_xpu/Inductor_ut_torch_xpu_test.log + - name: UT Test Results Check + shell: bash + run: | + function contains() { + contains_status="echo 'Start $2 ...'" + { + [[ $1 =~ (^|,)$2($|,) ]] + } || { + echo "[Warning] $2 is not suppotted type! Skipped!" + contains_status="continue" + } + } + set -xe + for ut_suite in $(echo ${{ inputs.ut }} |sed 's/,/ /g') + do + contains "op_regression,op_regression_dev1,op_extended,op_ut,torch_xpu" $ut_suite + $contains_status + cd ${{ github.workspace }}/ut_log/${ut_suite} + cp ${{ github.workspace }}/.github/scripts/ut_result_check.sh ./ + bash ut_result_check.sh ${ut_suite} + done + - name: Upload Inductor XPU UT Log + if: always() + uses: actions/upload-artifact@v4 + with: + name: Inductor-XPU-UT-Data-${{ github.event.pull_request.number || github.sha }}-${{ inputs.abi }} + path: ${{ github.workspace }}/ut_log