Merge branch 'main' into chunk-cat

.github/actions/inductor-xpu-e2e-test/action.yml

@@ -51,7 +51,7 @@ runs:
       shell: bash
       run: |
         source activate e2e_ci
-        source .github/scripts/env.sh
+        source .github/scripts/env.sh ${{ inputs.pytorch }}
         if [[ ${{ inputs.suite }} == *"torchbench"* ]]; then
           if [ "${{ inputs.pytorch }}" != "nightly_wheel" ]; then
             cd ../ && rm -rf audio && git clone --single-branch -b main https://github.com/pytorch/audio.git
@@ -94,7 +94,7 @@ runs:
       shell: bash
       run: |
         source activate e2e_ci
-        source .github/scripts/env.sh
+        source .github/scripts/env.sh ${{ inputs.pytorch }}
         cp .github/scripts/inductor_xpu_test.sh ../pytorch
         cd ../pytorch
 

.github/ci_expected_accuracy/check_expected.py

@@ -16,14 +16,14 @@
 
 
 # load csv files
-test_data= pd.read_csv(args.csv_file)
+test_data= pd.read_csv(args.csv_file, comment='#')
 # test_data = test_data.reset_index()  # make sure indexes pair with number of rows
 # test_data = test_data.sort_values(by=["name"], ascending=True)
 test_names = [row["name"] for index, row in test_data.iterrows()]
 
 current_path = pathlib.Path(__file__).parent.resolve()
 refer_file = str(current_path) + "/" + args.category + "_" + args.suite + "_" + args.mode + ".csv"
-refer_data= pd.read_csv(refer_file)
+refer_data= pd.read_csv(refer_file, comment='#')
 # refer_data = refer_data.reset_index()  # make sure indexes pair with number of rows
 # refer_data = refer_data.sort_values(by=["name"], ascending=True)
 refer_names = [row["name"] for index, row in refer_data.iterrows()]

.github/ci_expected_accuracy/inductor_huggingface_training.csv

@@ -13,7 +13,8 @@ CamemBert,pass,pass,pass,pass,pass
 DebertaForMaskedLM,pass,pass,pass,pass,pass
 DebertaForQuestionAnswering,pass,pass,pass,pass,pass
 DebertaV2ForMaskedLM,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip
-DebertaV2ForQuestionAnswering,pass,pass,pass,pass,pass
+# Skip DebertaV2ForQuestionAnswering issue: https://github.com/intel/torch-xpu-ops/issues/1216
+DebertaV2ForQuestionAnswering,fail_accuracy,fail_accuracy,fail_accuracy,pass,pass
 DistilBertForMaskedLM,pass,pass,pass,pass,pass
 DistilBertForQuestionAnswering,pass,pass,pass,pass,pass
 DistillGPT2,pass,pass,pass,pass,pass

.github/ci_expected_accuracy/inductor_torchbench_inference.csv

@@ -102,5 +102,6 @@ torch_multimodal_clip,pass,pass,pass,eager_fail_to_run,eager_fail_to_run
 tts_angular,pass,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run
 vgg16,pass,pass,pass,pass,pass
 vision_maskrcnn,pass,pass,pass,eager_fail_to_run,eager_fail_to_run
-yolov3,pass,pass,pass,pass,pass
+# Skip yolov3 for known torchbench issue: https://github.com/intel/torch-xpu-ops/issues/1229
+yolov3,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run
 hf_Roberta_base,pass,pass,pass,pass,pass

.github/ci_expected_accuracy/inductor_torchbench_training.csv

@@ -102,5 +102,6 @@ torch_multimodal_clip,pass,pass,pass,eager_fail_to_run,eager_fail_to_run
 tts_angular,pass,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run
 vgg16,pass,pass,pass,pass,pass
 vision_maskrcnn,pass,pass,pass,eager_fail_to_run,eager_fail_to_run
-yolov3,pass,pass,pass,pass,pass
+# Skip yolov3 for known torchbench issue: https://github.com/intel/torch-xpu-ops/issues/1229
+yolov3,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run
 hf_Roberta_base,pass,pass,pass,pass,pass

.github/scripts/apply_torch_pr.py

@@ -12,9 +12,7 @@
         # Fallback to CPU for XPU FP64
         "https://github.com/pytorch/pytorch/pull/126516",
         # Modify the tolerance level in TIMM benchmark
-        "https://github.com/pytorch/pytorch/pull/129735",
-        # [XPU] Update XPU C Shim Header
-        "https://github.com/pytorch/pytorch/pull/141086",
+        "https://github.com/pytorch/pytorch/pull/143739",
     ]
 )
 parser.add_argument('--extra-pr-list', '-e', nargs='+',default=[])
@@ -59,7 +57,7 @@ def appyly_pr(pr_info, re_apply_msg):
     pr_file = pr_info["diff_url"].split("/")[-1]
     urllib.request.urlretrieve(pr_info["diff_url"], pr_file)
     # apply diff
-    apply_cmd = "git apply --3way " + pr_file + " && rm -f " + pr_file
+    apply_cmd = "git apply --3way " + pr_file
     apply_info = subprocess.Popen(apply_cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=True)
     apply_message = apply_info.communicate()[0].decode("utf-8")
     apply_status = apply_info.returncode

.github/scripts/env.sh

@@ -1,4 +1,11 @@
 #!/bin/bash
-source /opt/intel/oneapi/compiler/latest/env/vars.sh
-source /opt/intel/oneapi/umf/latest/env/vars.sh
-source /opt/intel/oneapi/pti/latest/env/vars.sh
+
+if [ "$1" != "nightly_wheel" ];then
+    source /opt/intel/oneapi/compiler/latest/env/vars.sh
+    source /opt/intel/oneapi/umf/latest/env/vars.sh
+    source /opt/intel/oneapi/pti/latest/env/vars.sh
+    source /opt/intel/oneapi/ccl/latest/env/vars.sh
+    source /opt/intel/oneapi/mpi/latest/env/vars.sh
+else
+    echo "Don't need to source DL-Essential for nightly wheel"
+fi

.github/scripts/inductor_summary.py

@@ -3,6 +3,8 @@
 import pandas as pd
 from scipy.stats import gmean
 from styleframe import StyleFrame, Styler, utils
+import numpy as np
+from openpyxl import Workbook
 
 parser = argparse.ArgumentParser(description="Generate report")
 parser.add_argument('-s', '--suite', default=["huggingface"], nargs='*', type=str, help='model suite name')
@@ -665,6 +667,73 @@ def update_summary(excel, scenario, suite):
             sf.set_row_height(j, 30)
         sf.to_excel(sheet_name=suite + '_'  + scenario + '_Summary', excel_writer=excel)
 
+def summary_conclusion(scenario, excel):
+    excel.book.save(excel)
+    df = pd.read_excel(excel, sheet_name = None, header = None)
+    #df = pd.DataFrame(excel)
+    if scenario == 'performance':
+        sheet_names = list(df.keys())
+        sheet_names = [s for s in sheet_names if 'Summary' in s and 'performance' in s]
+        sheet_names.sort()
+        print(f"Merge excel as below:\n{sheet_names}")
+        print("\n")
+        features = [[]] * 21
+        for sheet_name in sheet_names:
+            df_sheet = df[sheet_name]
+            df_sheet = df_sheet.values
+            features = np.hstack((features, df_sheet))
+
+        if len(sheet_names) == 1:
+            print("sheet not merge")
+        elif len(sheet_names) == 2:
+            print("2 sheets merge")
+            if 'huggingface' in sheet_names[0]:
+                features[:, 4:5] = features[:, 14:15]
+                features[:, 6:7] = features[:, 16:17]
+            else:
+                features[:, 4:5] = features[:, 14:15]
+        else:
+            print("3 sheets merge")
+            features[:, 4:5] = features[:, 24:25]
+            features[:, 6:7] = features[:, 16:17]
+
+        df_concat = StyleFrame(pd.DataFrame(features).iloc[:,:10])
+        for i in range(10):
+            df_concat.set_column_width(i, 22)
+        for j in range(1, 23):
+            df_concat.set_row_height(j, 30)
+        df_concat.to_excel(sheet_name='Perf_Summary', excel_writer=excel, index=False)
+    else:
+        sheet_names = list(df.keys())
+        sheet_names = [s for s in sheet_names if 'Summary' in s and 'accuracy' in s]
+        sheet_names.sort()
+        print(f"Merge excel as below:\n{sheet_names}")
+        print("\n")
+        features = [[]] * 11
+        for sheet_name in sheet_names:
+            df_sheet = df[sheet_name]
+            df_sheet = df_sheet.values
+            features = np.hstack((features, df_sheet))
+        if len(sheet_names) == 1:
+            print("sheet not merge")
+        elif len(sheet_names) == 2:
+            print("2 sheets merge")
+            if 'huggingface' in sheet_names[0]:
+                features[:, 3:4] = features[:, 12:13]
+                features[:, 5:6] = features[:, 14:15]
+            else:
+                features[:, 3:4] = features[:, 12:13]
+        else:
+            print("3 sheets merge")
+            features[:, 3:4] = features[:, 21:22]
+            features[:, 5:6] = features[:, 14:15]
+
+        df_concat = StyleFrame(pd.DataFrame(features).iloc[:,:9])
+        for i in range(10):
+            df_concat.set_column_width(i, 22)
+        for j in range(1, 13):
+            df_concat.set_row_height(j, 30)
+        df_concat.to_excel(sheet_name='Acc_Summary', excel_writer=excel, index=False)
 
 def generate_report(excel, scenario_list, precision_list, mode_list, suite_list):
     for sc in scenario_list:
@@ -693,8 +762,19 @@ def excel_postprocess(file, scenario, precison, mode, suite):
                     wdt.merge_cells(start_row=1, end_row=1, start_column=13, end_column=16)
             wb.save(file)
 
+        if len(scenario) == 2:
+            wb.move_sheet("Perf_Summary", -(len(wb.worksheets)-1))
+            wb.move_sheet("Acc_Summary", -(len(wb.worksheets)-1))
+        elif len(scenario) == 1 and sc == 'accuracy':
+            wb.move_sheet("Acc_Summary", -(len(wb.worksheets)-1))
+        else:
+            wb.move_sheet("Perf_Summary", -(len(wb.worksheets)-1))
+
 
 if __name__ == '__main__':
     excel = StyleFrame.ExcelWriter('inductor_log/Inductor_E2E_Test_Report.xlsx')
     generate_report(excel, args.scenario, args.precision, args.mode, args.suite)
+    for sc in args.scenario:
+        summary_conclusion(sc, excel)
     excel_postprocess(excel, args.scenario, args.precision, args.mode, args.suite)
+    excel.close()

.github/scripts/spec.py

@@ -0,0 +1,7 @@
+import torch
+
+DEVICE_NAME = 'xpu'
+
+MANUAL_SEED_FN = torch.xpu.manual_seed
+EMPTY_CACHE_FN = torch.xpu.empty_cache
+DEVICE_COUNT_FN = torch.xpu.device_count