diff --git a/.github/actions/inductor-xpu-e2e-test/action.yml b/.github/actions/inductor-xpu-e2e-test/action.yml index f9084fe9b..ccbabf407 100644 --- a/.github/actions/inductor-xpu-e2e-test/action.yml +++ b/.github/actions/inductor-xpu-e2e-test/action.yml @@ -81,6 +81,7 @@ runs: source .github/scripts/env.sh cp .github/scripts/inductor_xpu_test.sh ../pytorch cd ../pytorch + # check param function contains() { contains_status="echo 'Start $2 ...'" @@ -136,8 +137,7 @@ runs: done done - - name: Summary E2E Accuracy Test (${{ inputs.suite }} ${{ inputs.dt }} ${{ inputs.mode }} ${{ inputs.scenario }}) - if: contains(inputs.scenario, 'accuracy') + - name: Summary E2E Test (${{ inputs.suite }} ${{ inputs.dt }} ${{ inputs.mode }} ${{ inputs.scenario }}) shell: bash env: HUGGING_FACE_HUB_TOKEN: ${{ inputs.hf_token }} @@ -150,60 +150,15 @@ runs: cat $var >> inductor_log/summary_accuracy.csv done - # check param - function contains() { - contains_status="echo 'Start $2 ...'" - { - [[ $1 =~ (^|,)$2($|,) ]] - } || { - echo "[Warning] $2 is not suppotted type! Skipped!" - contains_status="continue" - } - } - #Accuracy summary source activate e2e_ci cd ${{ github.workspace }} - cp .github/scripts/inductor_accuracy_summary.py ../pytorch + cp .github/scripts/inductor_summary.py ../pytorch cd ../pytorch pip install styleFrame scipy pandas set -xe - for suite in $(echo ${{ inputs.suite }} |sed 's/,/ /g') - do - contains "huggingface,timm_models,torchbench" $suite - $contains_status - dt=$(echo ${{ inputs.dt }} |sed 's/,/ /g') - mode=$(echo ${{ inputs.mode }} |sed 's/,/ /g') - python inductor_accuracy_summary.py -p ${dt} -s ${suite} -m ${mode} - done - - - name: Summary E2E Performance Test (${{ inputs.suite }} ${{ inputs.dt }} ${{ inputs.mode }} ${{ inputs.scenario }}) - if: contains(inputs.scenario, 'performance') - shell: bash - env: - HUGGING_FACE_HUB_TOKEN: ${{ inputs.hf_token }} - run: | - # check param - function contains() { - contains_status="echo 'Start $2 ...'" - { - [[ $1 =~ (^|,)$2($|,) ]] - } || { - echo "[Warning] $2 is not suppotted type! Skipped!" - contains_status="continue" - } - } - - source activate e2e_ci - cp .github/scripts/inductor_perf_summary.py ../pytorch - cd ../pytorch - source /opt/intel/oneapi/pytorch-gpu-dev-0.5/oneapi-vars.sh - pip install styleFrame scipy pandas - set -xe - for suite in $(echo ${{ inputs.suite }} |sed 's/,/ /g') - do - contains "huggingface,timm_models,torchbench" $suite - $contains_status - dt=$(echo ${{ inputs.dt }} |sed 's/,/ /g') - mode=$(echo ${{ inputs.mode }} |sed 's/,/ /g') - python inductor_perf_summary.py -p ${dt} -s ${suite} -m ${mode} - done + dt=$(echo ${{ inputs.dt }} |sed 's/,/ /g') + mode=$(echo ${{ inputs.mode }} |sed 's/,/ /g') + suite=$(echo ${{ inputs.suite }} |sed 's/,/ /g') + scenario=$(echo ${{ inputs.scenario }} |sed 's/,/ /g') + python inductor_summary.py -p ${dt} -s ${suite} -m ${mode} -sc ${scenario} + diff --git a/.github/scripts/inductor_accuracy_summary.py b/.github/scripts/inductor_accuracy_summary.py deleted file mode 100644 index 131df9831..000000000 --- a/.github/scripts/inductor_accuracy_summary.py +++ /dev/null @@ -1,350 +0,0 @@ -import argparse -import pandas as pd -from scipy.stats import gmean -from styleframe import StyleFrame, Styler, utils - -parser = argparse.ArgumentParser(description="Generate report") -parser.add_argument('-s', '--suite', default='huggingface', choices=["torchbench", "huggingface", "timm_models"], type=str, help='model suite name') -parser.add_argument('-p', '--precision', default=["amp_fp16", "float32"], nargs='*', type=str, help='precision') -parser.add_argument('-r', '--reference', type=str, help='reference log files') -parser.add_argument('-m', '--mode', default=["inference", "training"], nargs='*', type=str, help='mode name') -args = parser.parse_args() - -passrate_values = {} - -failure_style = Styler(bg_color='#FF0000', font_color=utils.colors.black) -regression_style = Styler(bg_color='#F0E68C', font_color=utils.colors.red) -improve_style = Styler(bg_color='#00FF00', font_color=utils.colors.black) - - -def percentage(part, whole, decimals=2): - if whole == 0: - return 0 - return round(100 * float(part) / float(whole), decimals) - -def get_passing_entries(df, column_name): - return df[df[column_name].notnull()] - -def caculate_passrate(df, key_word): - # some models may be new for reference - df = get_passing_entries(df, key_word) - total = len(df.index) - passing = df[df[key_word].fillna('').str.contains('pass')][key_word].count() - perc = int(percentage(passing, total, decimals=0)) - return f"{perc}%, {passing}/{total}" - -def get_acc_csv(precision, mode): - target_path = 'inductor_log/' + args.suite + '/' + precision + '/inductor_' + args.suite + '_' + precision + '_' + mode + '_xpu_accuracy.csv' - target_ori_data = pd.read_csv(target_path) - target_data = target_ori_data.copy() - target_data.sort_values(by=['name']) - - if args.reference is not None: - reference_file_path = args.reference + '/inductor_log/' + args.suite + '/' + precision + '/inductor_' + args.suite + '_' + precision + '_' + mode + '_xpu_accuracy.csv' - reference_ori_data = pd.read_csv(reference_file_path) - reference_data = reference_ori_data.copy() - reference_data.sort_values(by=['name']) - data = pd.merge(target_data,reference_data,on=['name'],how= 'outer') - return data - else: - return target_data - -def process(input, precision, mode): - global passrate_values - if input is not None: - if args.reference is None: - data_new = input[['name', 'batch_size', 'accuracy']].rename(columns={'name': 'name', 'batch_size': 'batch_size', 'accuracy': 'accuracy'}) - passrate_values['target_' + str(precision) + '_' + str(mode)] = caculate_passrate(data_new, 'accuracy') - data = StyleFrame({'name': list(data_new['name']), - 'batch_size': list(data_new['batch_size']), - 'accuracy': list(data_new['accuracy'])}) - data.set_column_width(1, 10) - data.set_column_width(2, 18) - data.set_column_width(3, 18) - data.set_row_height(rows=data.row_indexes, height=15) - else: - data_new=input[['name','batch_size_x','accuracy_x']].rename(columns={'name':'name','batch_size_x':'batch_size_new','accuracy_x':'accuracy_new'}) - passrate_values['target_' + str(precision) + '_' + str(mode)] = caculate_passrate(data_new, 'accuracy_new') - data_old=input[['batch_size_y','accuracy_y']].rename(columns={'batch_size_y':'batch_size_old','accuracy_y':'accuracy_old'}) - passrate_values['reference_' + str(precision) + '_' + str(mode)] = caculate_passrate(data_old, 'accuracy_old') - data_comp = pd.DataFrame((data_new['accuracy_new'] != 'pass') & (data_old['accuracy_old'] == 'pass'),columns=['Accuracy regression']) - combined_data = pd.DataFrame({ - 'name': list(data_new['name']), - 'batch_size_new': list(data_new['batch_size_new']), - 'accuracy_new': list(data_new['accuracy_new']), - 'batch_size_old': list(data_old['batch_size_old']), - 'accuracy_old': list(data_old['accuracy_old']), - 'Accuracy regression': list(data_comp['Accuracy regression']) - }) - data = StyleFrame(combined_data) - data.set_column_width(1, 10) - data.set_column_width(2, 18) - data.set_column_width(3, 18) - data.set_column_width(4, 18) - data.set_column_width(5, 15) - data.set_column_width(6, 20) - data.apply_style_by_indexes(indexes_to_style=data[(data['Accuracy regression'] == 'regression')],styler_obj=regression_style) - data.set_row_height(rows=data.row_indexes, height=15) - return data - else: - return pd.DataFrame() - -def update_details(precision, mode, excel): - h = {"A": 'Model suite', "B": '', "C": "target", "D": '', "E": args.reference, "F": '', "G": 'Result Comp'} - if args.reference is None: - h = {"A": 'Model suite', "B": '', "C": "target", "D": ''} - head = StyleFrame(pd.DataFrame(h, index=[0])) - head.set_column_width(1, 15) - head.set_row_height(rows=[1], height=15) - - head.to_excel(excel_writer=excel, sheet_name=precision + '_' + mode, index=False, startrow=0, header=False) - target_raw_data = get_acc_csv(precision, mode) - target_data = process(target_raw_data, precision, mode) - target_data.to_excel(excel_writer=excel, sheet_name=precision + '_' + mode, index=False, startrow=1, startcol=1) - -def update_summary(excel): - data = { - 'Test Secnario': ['AMP_BF16 Inference', 'AMP_BF16 Training', 'AMP_FP16 Inference', 'AMP_FP16 Training', 'BF16 Inference', 'BF16 Training', 'FP16 Inference', 'FP16 Training', 'FP32 Inference', 'FP32 Training'], - 'Comp Item': ['Pass Rate', 'Pass Rate', 'Pass Rate', 'Pass Rate', 'Pass Rate', 'Pass Rate', 'Pass Rate', 'Pass Rate', 'Pass Rate', 'Pass Rate'], - 'compiler': ['inductor', 'inductor', 'inductor', 'inductor', 'inductor', 'inductor', 'inductor', 'inductor', 'inductor', 'inductor'], - 'torchbench': [' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '], - 'huggingface': [' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '], - 'timm_models ': [' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '], - 'refer_torchbench ': [' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '], - 'refer_huggingface ': [' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '], - 'refer_timm_models ': [' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '] - } - summary = pd.DataFrame(data) - - if 'huggingface' in args.suite: - if 'amp_bf16' in args.precision: - if 'inference' in args.mode: - # passrate - summary.iloc[0:1, 4:5] = passrate_values['target_amp_bf16_inference'] - if 'training' in args.mode: - summary.iloc[1:2, 4:5] = passrate_values['target_amp_bf16_training'] - - if 'amp_fp16' in args.precision: - if 'inference' in args.mode: - summary.iloc[2:3, 4:5] = passrate_values['target_amp_fp16_inference'] - if 'training' in args.mode: - summary.iloc[3:4, 4:5] = passrate_values['target_amp_fp16_training'] - - if 'bfloat16' in args.precision: - if 'inference' in args.mode: - summary.iloc[4:5, 4:5] = passrate_values['target_bfloat16_inference'] - if 'training' in args.mode: - summary.iloc[5:6, 4:5] = passrate_values['target_bfloat16_training'] - if 'float16' in args.precision: - if 'inference' in args.mode: - summary.iloc[6:7, 4:5] = passrate_values['target_float16_inference'] - if 'training' in args.mode: - summary.iloc[7:8, 4:5] = passrate_values['target_float16_training'] - if 'float32' in args.precision: - if 'inference' in args.mode: - summary.iloc[8:9, 4:5] = passrate_values['target_float32_inference'] - if 'training' in args.mode: - summary.iloc[9:10, 4:5] = passrate_values['target_float32_training'] - - if args.reference is not None: - if 'amp_bf16' in args.precision: - if 'inference' in args.mode: - # passrate - summary.iloc[0:1, 7:8] = passrate_values['reference_amp_bf16_inference'] - if 'training' in args.mode: - summary.iloc[1:2, 7:8] = passrate_values['reference_amp_bf16_training'] - if 'amp_fp16' in args.precision: - if 'inference' in args.mode: - summary.iloc[2:3, 7:8] = passrate_values['reference_amp_fp16_inference'] - if 'training' in args.mode: - summary.iloc[3:4, 7:8] = passrate_values['reference_amp_fp16_training'] - - if 'bfloat16' in args.precision: - if 'inference' in args.mode: - summary.iloc[4:5, 7:8] = passrate_values['reference_bfloat16_inference'] - if 'training' in args.mode: - summary.iloc[5:6, 7:8] = passrate_values['reference_bfloat16_training'] - if 'float16' in args.precision: - if 'inference' in args.mode: - summary.iloc[6:7, 7:8] = passrate_values['reference_float16_inference'] - if 'training' in args.mode: - summary.iloc[7:8, 7:8] = passrate_values['reference_float16_training'] - if 'float32' in args.precision: - if 'inference' in args.mode: - summary.iloc[8:9, 7:8] = passrate_values['reference_float32_inference'] - if 'training' in args.mode: - summary.iloc[9:10, 7:8] = passrate_values['reference_float32_training'] - - print("====================Summary=============================") - print(summary) - - sf = StyleFrame(summary) - for i in range(1, 10): - sf.set_column_width(i, 22) - for j in range(1, 12): - sf.set_row_height(j, 30) - sf.to_excel(sheet_name='Summary', excel_writer=excel) - - if 'timm_models' in args.suite: - if 'amp_bf16' in args.precision: - if 'inference' in args.mode: - # passrate - summary.iloc[0:1, 5:6] = passrate_values['target_amp_bf16_inference'] - if 'training' in args.mode: - summary.iloc[1:2, 5:6] = passrate_values['target_amp_bf16_training'] - if 'amp_fp16' in args.precision: - if 'inference' in args.mode: - summary.iloc[2:3, 5:6] = passrate_values['target_amp_fp16_inference'] - if 'training' in args.mode: - summary.iloc[3:4, 5:6] = passrate_values['target_amp_fp16_training'] - - if 'bfloat16' in args.precision: - if 'inference' in args.mode: - summary.iloc[4:5, 5:6] = passrate_values['target_bfloat16_inference'] - if 'training' in args.mode: - summary.iloc[5:6, 5:6] = passrate_values['target_bfloat16_training'] - if 'float16' in args.precision: - if 'inference' in args.mode: - summary.iloc[6:7, 5:6] = passrate_values['target_float16_inference'] - if 'training' in args.mode: - summary.iloc[7:8, 5:6] = passrate_values['target_float16_training'] - if 'float32' in args.precision: - if 'inference' in args.mode: - summary.iloc[8:9, 5:6] = passrate_values['target_float32_inference'] - if 'training' in args.mode: - summary.iloc[9:10, 5:6] = passrate_values['target_float32_training'] - - if args.reference is not None: - if 'amp_bf16' in args.precision: - if 'inference' in args.mode: - # passrate - summary.iloc[0:1, 8:9] = passrate_values['reference_amp_bf16_inference'] - if 'training' in args.mode: - summary.iloc[1:2, 8:9] = passrate_values['reference_amp_bf16_training'] - if 'amp_fp16' in args.precision: - if 'inference' in args.mode: - summary.iloc[2:3, 8:9] = passrate_values['reference_amp_fp16_inference'] - if 'training' in args.mode: - summary.iloc[3:4, 8:9] = passrate_values['reference_amp_fp16_training'] - - if 'bfloat16' in args.precision: - if 'inference' in args.mode: - summary.iloc[4:5, 8:9] = passrate_values['reference_bfloat16_inference'] - if 'training' in args.mode: - summary.iloc[5:6, 8:9] = passrate_values['reference_bfloat16_training'] - if 'float16' in args.precision: - if 'inference' in args.mode: - summary.iloc[6:7, 8:9] = passrate_values['reference_float16_inference'] - if 'training' in args.mode: - summary.iloc[7:8, 8:9] = passrate_values['reference_float16_training'] - if 'float32' in args.precision: - if 'inference' in args.mode: - summary.iloc[8:9, 8:9] = passrate_values['reference_float32_inference'] - if 'training' in args.mode: - summary.iloc[9:10, 8:9] = passrate_values['reference_float32_training'] - - print("====================Summary=============================") - print(summary) - - sf = StyleFrame(summary) - for i in range(1, 10): - sf.set_column_width(i, 22) - for j in range(1, 12): - sf.set_row_height(j, 30) - sf.to_excel(sheet_name='Summary', excel_writer=excel) - - if 'torchbench' in args.suite: - if 'amp_bf16' in args.precision: - if 'inference' in args.mode: - # passrate - summary.iloc[0:1, 3:4] = passrate_values['target_amp_bf16_inference'] - if 'training' in args.mode: - summary.iloc[1:2, 3:4] = passrate_values['target_amp_bf16_training'] - if 'amp_fp16' in args.precision: - if 'inference' in args.mode: - summary.iloc[2:3, 3:4] = passrate_values['target_amp_fp16_inference'] - if 'training' in args.mode: - summary.iloc[3:4, 3:4] = passrate_values['target_amp_fp16_training'] - - if 'bfloat16' in args.precision: - if 'inference' in args.mode: - summary.iloc[4:5, 3:4] = passrate_values['target_bfloat16_inference'] - if 'training' in args.mode: - summary.iloc[5:6, 3:4] = passrate_values['target_bfloat16_training'] - if 'float16' in args.precision: - if 'inference' in args.mode: - summary.iloc[6:7, 3:4] = passrate_values['target_float16_inference'] - if 'training' in args.mode: - summary.iloc[7:8, 3:4] = passrate_values['target_float16_training'] - if 'float32' in args.precision: - if 'inference' in args.mode: - summary.iloc[8:9, 3:4] = passrate_values['target_float32_inference'] - if 'training' in args.mode: - summary.iloc[9:10, 3:4] = passrate_values['target_float32_training'] - - if args.reference is not None: - if 'amp_bf16' in args.precision: - if 'inference' in args.mode: - # passrate - summary.iloc[0:1, 6:7] = passrate_values['reference_amp_bf16_inference'] - if 'training' in args.mode: - summary.iloc[1:2, 6:7] = passrate_values['reference_amp_bf16_training'] - - if 'amp_fp16' in args.precision: - if 'inference' in args.mode: - summary.iloc[2:3, 6:7] = passrate_values['reference_amp_fp16_inference'] - if 'training' in args.mode: - summary.iloc[3:4, 6:7] = passrate_values['reference_amp_fp16_training'] - - if 'bfloat16' in args.precision: - if 'inference' in args.mode: - summary.iloc[4:5, 6:7] = passrate_values['reference_bfloat16_inference'] - if 'training' in args.mode: - summary.iloc[5:6, 6:7] = passrate_values['reference_bfloat16_training'] - if 'float16' in args.precision: - if 'inference' in args.mode: - summary.iloc[6:7, 6:7] = passrate_values['reference_float16_inference'] - if 'training' in args.mode: - summary.iloc[7:8, 6:7] = passrate_values['reference_float16_training'] - if 'float32' in args.precision: - if 'inference' in args.mode: - summary.iloc[8:9, 6:7] = passrate_values['reference_float32_inference'] - if 'training' in args.mode: - summary.iloc[9:10, 6:7] = passrate_values['reference_float32_training'] - - print("====================Summary=============================") - print(summary) - - sf = StyleFrame(summary) - for i in range(1, 10): - sf.set_column_width(i, 22) - for j in range(1, 12): - sf.set_row_height(j, 30) - sf.to_excel(sheet_name='Summary', excel_writer=excel) - -def generate_report(excel, precision_list, mode_list): - for p in precision_list: - for m in mode_list: - update_details(p, m, excel) - update_summary(excel) - - -def excel_postprocess(file, precison, mode): - wb = file.book - # details - for p in precison: - for m in mode: - wdt = wb[p + '_' + m] - wdt.merge_cells(start_row=1, end_row=2, start_column=1, end_column=1) - wdt.merge_cells(start_row=1, end_row=1, start_column=3, end_column=4) - wdt.merge_cells(start_row=1, end_row=1, start_column=5, end_column=6) - wdt.merge_cells(start_row=1, end_row=1, start_column=7, end_column=7) - wb.save(file) - - -if __name__ == '__main__': - summary_path = 'inductor_log/' + str(args.suite) + '/Inductor_' + args.suite + '_E2E_Test_Acc_Report.xlsx' - excel = StyleFrame.ExcelWriter(summary_path) - print(f"=========Acc Check Summary file located in {summary_path}==================") - generate_report(excel, args.precision, args.mode) - excel_postprocess(excel, args.precision, args.mode) diff --git a/.github/scripts/inductor_perf_summary.py b/.github/scripts/inductor_perf_summary.py deleted file mode 100644 index 02a68eea7..000000000 --- a/.github/scripts/inductor_perf_summary.py +++ /dev/null @@ -1,498 +0,0 @@ -import argparse - -import pandas as pd -from scipy.stats import gmean -from styleframe import StyleFrame, Styler, utils - -parser = argparse.ArgumentParser(description="Generate report") -parser.add_argument('-s', '--suite', default='huggingface', choices=["torchbench", "huggingface", "timm_models"], type=str, help='model suite name') -parser.add_argument('-p', '--precision', default=["amp_fp16", "float32"], nargs='*', type=str, help='precision') -parser.add_argument('-r', '--reference', type=str, help='reference log files') -parser.add_argument('-m', '--mode', default=["inference", "training"], nargs='*', type=str, help='mode name') -parser.add_argument('--html_off', action='store_true', help='turn off html file generate') -args = parser.parse_args() - -passrate_values = {} -geomean_values = {} - -new_performance_regression=pd.DataFrame() - -failure_style = Styler(bg_color='#FF0000', font_color=utils.colors.black) -regression_style = Styler(bg_color='#F0E68C', font_color=utils.colors.red) -improve_style = Styler(bg_color='#00FF00', font_color=utils.colors.black) - -# refer https://github.com/pytorch/pytorch/blob/main/benchmarks/dynamo/runner.py#L757-L778 - - -def percentage(part, whole, decimals=2): - if whole == 0: - return 0 - return round(100 * float(part) / float(whole), decimals) - - -def get_passing_entries(df, column_name): - return df[column_name][df[column_name] > 0] - - -def caculate_geomean(df, column_name): - cleaned_df = get_passing_entries(df, column_name).clip(1) - if cleaned_df.empty: - return "0.0x" - return f"{gmean(cleaned_df):.2f}x" - - -def caculate_passrate(df, compiler): - total = len(df.index) - passing = df[df[compiler] > 0.0][compiler].count() - perc = int(percentage(passing, total, decimals=0)) - return f"{perc}%, {passing}/{total}" - -def get_perf_csv(precision, mode): - target_path = 'inductor_log/' + args.suite + '/' + precision + '/inductor_' + args.suite + '_' + precision + '_' + mode + '_xpu_performance.csv' - target_ori_data = pd.read_csv(target_path, header=0, encoding='utf-8') - target_data = target_ori_data.copy() - target_data.sort_values(by=['name']) - - if args.reference is not None: - reference_file_path = args.reference + '/inductor_log/' + args.suite + '/' + precision + '/inductor_' + args.suite + '_' + precision + '_' + mode + '_xpu_performance.csv' - reference_ori_data = pd.read_csv(reference_file_path, header=0, encoding='utf-8') - reference_data = reference_ori_data.copy() - reference_data.sort_values(by=['name']) - data = pd.merge(target_data,reference_data,on=['name'],how= 'outer') - return data - else: - return target_data - -def process(input, precision, mode): - global geomean_values, passrate_values - if input is not None: - if args.reference is None: - data_new = input[['name', 'batch_size', 'speedup', 'abs_latency','compilation_latency']].rename(columns={'name': 'name', 'batch_size': 'batch_size', 'speedup': 'speedup', "abs_latency": 'inductor', "compilation_latency": 'compilation_latency'}) - data_new['inductor'] = data_new['inductor'].apply(pd.to_numeric, errors='coerce').div(1000) - data_new['speedup'] = data_new['speedup'].apply(pd.to_numeric, errors='coerce').fillna(0.0) - data_new['eager'] = data_new['speedup'] * data_new['inductor'] - geomean_values['target_' + str(precision) + '_' + str(mode)] = caculate_geomean(data_new, 'speedup') - passrate_values['target_' + str(precision) + '_' + str(mode)] = caculate_passrate(data_new, 'inductor') - data = StyleFrame({'name': list(data_new['name']), - 'batch_size': list(data_new['batch_size']), - 'speedup': list(data_new['speedup']), - 'inductor': list(data_new['inductor']), - 'eager': list(data_new['eager']), - 'compilation_latency': list(data_new['compilation_latency'])}) - data.set_column_width(1, 10) - data.set_column_width(2, 18) - data.set_column_width(3, 18) - data.set_column_width(4, 18) - data.set_column_width(5, 15) - data.set_column_width(6, 20) - data.set_row_height(rows=data.row_indexes, height=15) - else: - data_new=input[['name','batch_size_x','speedup_x','abs_latency_x','compilation_latency_x']].rename(columns={'name':'name','batch_size_x':'batch_size_new','speedup_x':'speed_up_new',"abs_latency_x":'inductor_new',"compilation_latency_x":'compilation_latency_new'}) - data_new['inductor_new']=data_new['inductor_new'].astype(float).div(1000) - data_new['speed_up_new']=data_new['speed_up_new'].apply(pd.to_numeric, errors='coerce').fillna(0.0) - data_new['eager_new'] = data_new['speed_up_new'] * data_new['inductor_new'] - geomean_values['target_' + str(precision) + '_' + str(mode)] = caculate_geomean(data_new, 'speed_up_new') - passrate_values['target_' + str(precision) + '_' + str(mode)] = caculate_passrate(data_new, 'inductor_new') - data_old=input[['batch_size_y','speedup_y','abs_latency_y','compilation_latency_y']].rename(columns={'batch_size_y':'batch_size_old','speedup_y':'speed_up_old',"abs_latency_y":'inductor_old',"compilation_latency_y":'compilation_latency_old'}) - data_old['inductor_old']=data_old['inductor_old'].astype(float).div(1000) - data_old['speed_up_old']=data_old['speed_up_old'].apply(pd.to_numeric, errors='coerce').fillna(0.0) - data_old['eager_old'] = data_old['speed_up_old'] * data_old['inductor_old'] - input['speedup_x']=input['speedup_x'].apply(pd.to_numeric, errors='coerce').fillna(0.0) - input['speedup_y']=input['speedup_y'].apply(pd.to_numeric, errors='coerce').fillna(0.0) - geomean_values['reference_' + str(precision) + '_' + str(mode)] = caculate_geomean(data_old, 'speed_up_old') - passrate_values['reference_' + str(precision) + '_' + str(mode)] = caculate_passrate(data_old, 'inductor_old') - data_ratio= pd.DataFrame(round(input['speedup_x'] / input['speedup_y'],2),columns=['Ratio Speedup(New/old)']) - data_ratio['Eager Ratio(old/new)'] = pd.DataFrame(round(data_old['eager_old'] / data_new['eager_new'],2)) - data_ratio['Inductor Ratio(old/new)'] = pd.DataFrame(round(data_old['inductor_old'] / data_new['inductor_new'],2)) - data_ratio['Compilation_latency_Ratio(old/new)'] = pd.DataFrame(round(data_old['compilation_latency_old'] / data_new['compilation_latency_new'],2)) - - combined_data = pd.DataFrame({ - 'name': list(data_new['name']), - 'batch_size_new': list(data_new['batch_size_new']), - 'speed_up_new': list(data_new['speed_up_new']), - 'inductor_new': list(data_new['inductor_new']), - 'eager_new': list(data_new['eager_new']), - 'compilation_latency_new': list(data_new['compilation_latency_new']), - 'batch_size_old': list(data_old['batch_size_old']), - 'speed_up_old': list(data_old['speed_up_old']), - 'inductor_old': list(data_old['inductor_old']), - 'eager_old': list(data_old['eager_old']), - 'compilation_latency_old': list(data_old['compilation_latency_old']), - 'Ratio Speedup(New/old)': list(data_ratio['Ratio Speedup(New/old)']), - 'Eager Ratio(old/new)': list(data_ratio['Eager Ratio(old/new)']), - 'Inductor Ratio(old/new)': list(data_ratio['Inductor Ratio(old/new)']), - 'Compilation_latency_Ratio(old/new)': list(data_ratio['Compilation_latency_Ratio(old/new)']) - }) - data = StyleFrame(combined_data) - data.set_column_width(1, 10) - data.set_column_width(2, 18) - data.set_column_width(3, 18) - data.set_column_width(4, 18) - data.set_column_width(5, 15) - data.set_column_width(6, 20) - data.set_column_width(7, 18) - data.set_column_width(8, 18) - data.set_column_width(9, 18) - data.set_column_width(10, 15) - data.set_column_width(11, 20) - data.set_column_width(12, 28) - data.set_column_width(13, 28) - data.set_column_width(14, 28) - data.set_column_width(15, 32) - data.apply_style_by_indexes(indexes_to_style=data[data['batch_size_new'] == 0], styler_obj=failure_style) - data.apply_style_by_indexes(indexes_to_style=data[(data['Inductor Ratio(old/new)'] > 0) & (data['Inductor Ratio(old/new)'] < 0.9)],styler_obj=regression_style) - global new_performance_regression - regression = data.loc[(data['Inductor Ratio(old/new)'] > 0) & (data['Inductor Ratio(old/new)'] < 0.9)] - regression = regression.copy() - regression.loc[0] = list(regression.shape[1]*'*') - new_performance_regression = pd.concat([new_performance_regression,regression]) - data.apply_style_by_indexes(indexes_to_style=data[data['Inductor Ratio(old/new)'] > 1.1],styler_obj=improve_style) - data.set_row_height(rows=data.row_indexes, height=15) - return data - else: - return pd.DataFrame() - -def update_details(precision, mode, excel): - h = {"A": 'Model suite', "B": '', "C": "target", "D": '', "E": '', "F": '', "G": '',"H": args.reference, "I": '', "J": '',"K": '',"L":'',"M": 'Result Comp',"N": '',"O": '',"P":''} - if args.reference is None: - h = {"A": 'Model suite', "B": '', "C": "target", "D": '', "E": '', "F": '', "G": ''} - head = StyleFrame(pd.DataFrame(h, index=[0])) - head.set_column_width(1, 15) - head.set_row_height(rows=[1], height=15) - - head.to_excel(excel_writer=excel, sheet_name=precision + '_' + mode, index=False, startrow=0, header=False) - target_raw_data = get_perf_csv(precision, mode) - #print("target_raw_data", target_raw_data) - target_data = process(target_raw_data, precision, mode) - target_data.to_excel(excel_writer=excel, sheet_name=precision + '_' + mode, index=False, startrow=1, startcol=1) - -def update_summary(excel): - data = { - 'Test Secnario': ['AMP_BF16 Inference', ' ', 'AMP_BF16 Training', ' ', 'AMP_FP16 Inference', ' ', 'AMP_FP16 Training', ' ', 'BF16 Inference', ' ', 'BF16 Training', ' ', 'FP16 Inference', ' ', 'FP16 Training', ' ', 'FP32 Inference', ' ', 'FP32 Training', ' '], - 'Comp Item': ['Pass Rate', 'Geomean Speedup', 'Pass Rate', 'Geomean Speedup', 'Pass Rate', 'Geomean Speedup', 'Pass Rate', 'Geomean Speedup', 'Pass Rate', 'Geomean Speedup', 'Pass Rate', 'Geomean Speedup', 'Pass Rate', 'Geomean Speedup', 'Pass Rate', 'Geomean Speedup', 'Pass Rate', 'Geomean Speedup', 'Pass Rate', 'Geomean Speedup'], - 'Date': [' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '], - 'Compiler': ['inductor', 'inductor', 'inductor', 'inductor', 'inductor', 'inductor', 'inductor', 'inductor', 'inductor', 'inductor', 'inductor', 'inductor', 'inductor', 'inductor', 'inductor', 'inductor', 'inductor', 'inductor', 'inductor', 'inductor'], - 'torchbench': [' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '], - 'huggingface': [' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '], - 'timm_models ': [' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '], - 'ref_torchbench ': [' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '], - 'refer_huggingface ': [' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '], - 'refer_timm_models ': [' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '] - } - summary = pd.DataFrame(data) - - if 'huggingface' in args.suite: - if 'amp_bf16' in args.precision: - if 'inference' in args.mode: - summary.iloc[0:1, 5:6] = passrate_values['target_amp_bf16_inference'] - summary.iloc[1:2, 5:6] = geomean_values['target_amp_bf16_inference'] - if 'training' in args.mode: - summary.iloc[2:3, 5:6] = passrate_values['target_amp_bf16_training'] - summary.iloc[3:4, 5:6] = geomean_values['target_amp_bf16_training'] - if 'amp_fp16' in args.precision: - if 'inference' in args.mode: - summary.iloc[4:5, 5:6] = passrate_values['target_amp_fp16_inference'] - summary.iloc[5:6, 5:6] = geomean_values['target_amp_fp16_inference'] - if 'training' in args.mode: - summary.iloc[6:7, 5:6] = passrate_values['target_amp_fp16_training'] - summary.iloc[7:8, 5:6] = geomean_values['target_amp_fp16_training'] - - if 'bfloat16' in args.precision: - if 'inference' in args.mode: - summary.iloc[8:9, 5:6] = passrate_values['target_bfloat16_inference'] - summary.iloc[9:10, 5:6] = geomean_values['target_bfloat16_inference'] - if 'training' in args.mode: - summary.iloc[10:11, 5:6] = passrate_values['target_bfloat16_training'] - summary.iloc[11:12, 5:6] = geomean_values['target_bfloat16_training'] - if 'float16' in args.precision: - if 'inference' in args.mode: - summary.iloc[12:13, 5:6] = passrate_values['target_float16_inference'] - summary.iloc[13:14, 5:6] = geomean_values['target_float16_inference'] - if 'training' in args.mode: - summary.iloc[14:15, 5:6] = passrate_values['target_float16_training'] - summary.iloc[15:16, 5:6] = geomean_values['target_float16_training'] - if 'float32' in args.precision: - if 'inference' in args.mode: - summary.iloc[16:17, 5:6] = passrate_values['target_float32_inference'] - summary.iloc[17:18, 5:6] = geomean_values['target_float32_inference'] - if 'training' in args.mode: - summary.iloc[18:19, 5:6] = passrate_values['target_float32_training'] - summary.iloc[19:20, 5:6] = geomean_values['target_float32_training'] - - if args.reference is not None: - if 'amp_bf16' in args.precision: - if 'inference' in args.mode: - summary.iloc[0:1, 8:9] = passrate_values['reference_amp_bf16_inference'] - summary.iloc[1:2, 8:9] = geomean_values['reference_amp_bf16_inference'] - if 'training' in args.mode: - summary.iloc[2:3, 8:9] = passrate_values['reference_amp_bf16_training'] - summary.iloc[3:4, 8:9] = geomean_values['reference_amp_bf16_training'] - if 'amp_fp16' in args.precision: - if 'inference' in args.mode: - summary.iloc[4:5, 8:9] = passrate_values['reference_amp_fp16_inference'] - summary.iloc[5:6, 8:9] = geomean_values['reference_amp_fp16_inference'] - if 'training' in args.mode: - summary.iloc[6:7, 8:9] = passrate_values['reference_amp_fp16_training'] - summary.iloc[7:8, 8:9] = geomean_values['reference_amp_fp16_training'] - - if 'bfloat16' in args.precision: - if 'inference' in args.mode: - summary.iloc[8:9, 8:9] = passrate_values['reference_bfloat16_inference'] - summary.iloc[9:10, 8:9] = geomean_values['reference_bfloat16_inference'] - if 'training' in args.mode: - summary.iloc[10:11, 8:9] = passrate_values['reference_bfloat16_training'] - summary.iloc[11:12, 8:9] = geomean_values['reference_bfloat16_training'] - if 'float16' in args.precision: - if 'inference' in args.mode: - summary.iloc[12:13, 8:9] = passrate_values['reference_float16_inference'] - summary.iloc[13:14, 8:9] = geomean_values['reference_float16_inference'] - if 'training' in args.mode: - summary.iloc[14:15, 8:9] = passrate_values['reference_float16_training'] - summary.iloc[15:16, 8:9] = geomean_values['reference_float16_training'] - if 'float32' in args.precision: - if 'inference' in args.mode: - summary.iloc[16:17, 8:9] = passrate_values['reference_float32_inference'] - summary.iloc[17:18, 8:9] = geomean_values['reference_float32_inference'] - if 'training' in args.mode: - summary.iloc[18:19, 8:9] = passrate_values['reference_float32_training'] - summary.iloc[19:20, 8:9] = geomean_values['reference_float32_training'] - - print("====================Summary=============================") - print(summary) - - sf = StyleFrame(summary) - for i in range(1, 11): - sf.set_column_width(i, 22) - for j in range(1, 22): - sf.set_row_height(j, 30) - sf.to_excel(sheet_name='Summary', excel_writer=excel) - - if 'timm_models' in args.suite: - if 'amp_bf16' in args.precision: - if 'inference' in args.mode: - summary.iloc[0:1, 6:7] = passrate_values['target_amp_bf16_inference'] - summary.iloc[1:2, 6:7] = geomean_values['target_amp_bf16_inference'] - if 'training' in args.mode: - summary.iloc[2:3, 6:7] = passrate_values['target_amp_bf16_training'] - summary.iloc[3:4, 6:7] = geomean_values['target_amp_bf16_training'] - if 'amp_fp16' in args.precision: - if 'inference' in args.mode: - summary.iloc[4:5, 6:7] = passrate_values['target_amp_fp16_inference'] - summary.iloc[5:6, 6:7] = geomean_values['target_amp_fp16_inference'] - if 'training' in args.mode: - summary.iloc[6:7, 6:7] = passrate_values['target_amp_fp16_training'] - summary.iloc[7:8, 6:7] = geomean_values['target_amp_fp16_training'] - - if 'bfloat16' in args.precision: - if 'inference' in args.mode: - summary.iloc[8:9, 6:7] = passrate_values['target_bfloat16_inference'] - summary.iloc[9:10, 6:7] = geomean_values['target_bfloat16_inference'] - if 'training' in args.mode: - summary.iloc[10:11, 6:7] = passrate_values['target_bfloat16_training'] - summary.iloc[11:12, 6:7] = geomean_values['target_bfloat16_training'] - if 'float16' in args.precision: - if 'inference' in args.mode: - summary.iloc[12:13, 6:7] = passrate_values['target_float16_inference'] - summary.iloc[13:14, 6:7] = geomean_values['target_float16_inference'] - if 'training' in args.mode: - summary.iloc[14:15, 6:7] = passrate_values['target_float16_training'] - summary.iloc[15:16, 6:7] = geomean_values['target_float16_training'] - if 'float32' in args.precision: - if 'inference' in args.mode: - summary.iloc[16:17, 6:7] = passrate_values['target_float32_inference'] - summary.iloc[17:18, 6:7] = geomean_values['target_float32_inference'] - if 'training' in args.mode: - summary.iloc[18:19, 6:7] = passrate_values['target_float32_training'] - summary.iloc[19:20, 6:7] = geomean_values['target_float32_training'] - - if args.reference is not None: - if 'amp_bf16' in args.precision: - if 'inference' in args.mode: - summary.iloc[0:1, 9:10] = passrate_values['reference_amp_bf16_inference'] - summary.iloc[1:2, 9:10] = geomean_values['reference_amp_bf16_inference'] - if 'training' in args.mode: - summary.iloc[2:3, 9:10] = passrate_values['reference_amp_bf16_training'] - summary.iloc[3:4, 9:10] = geomean_values['reference_amp_bf16_training'] - - if 'amp_fp16' in args.precision: - if 'inference' in args.mode: - summary.iloc[4:5, 9:10] = passrate_values['reference_amp_fp16_inference'] - summary.iloc[5:6, 9:10] = geomean_values['reference_amp_fp16_inference'] - if 'training' in args.mode: - summary.iloc[6:7, 9:10] = passrate_values['reference_amp_fp16_training'] - summary.iloc[7:8, 9:10] = geomean_values['reference_amp_fp16_training'] - - if 'bfloat16' in args.precision: - if 'inference' in args.mode: - summary.iloc[8:9, 9:10] = passrate_values['reference_bfloat16_inference'] - summary.iloc[9:10, 9:10] = geomean_values['reference_bfloat16_inference'] - if 'training' in args.mode: - summary.iloc[10:11, 9:10] = passrate_values['reference_bfloat16_training'] - summary.iloc[11:12, 9:10] = geomean_values['reference_bfloat16_training'] - if 'float16' in args.precision: - if 'inference' in args.mode: - summary.iloc[12:13, 9:10] = passrate_values['reference_float16_inference'] - summary.iloc[13:14, 9:10] = geomean_values['reference_float16_inference'] - if 'training' in args.mode: - summary.iloc[14:15, 9:10] = passrate_values['reference_float16_training'] - summary.iloc[15:16, 9:10] = geomean_values['reference_float16_training'] - if 'float32' in args.precision: - if 'inference' in args.mode: - summary.iloc[16:17, 9:10] = passrate_values['reference_float32_inference'] - summary.iloc[17:18, 9:10] = geomean_values['reference_float32_inference'] - if 'training' in args.mode: - summary.iloc[18:19, 9:10] = passrate_values['reference_float32_training'] - summary.iloc[19:20, 9:10] = geomean_values['reference_float32_training'] - - print("====================Summary=============================") - print(summary) - - sf = StyleFrame(summary) - for i in range(1, 11): - sf.set_column_width(i, 22) - for j in range(1, 22): - sf.set_row_height(j, 30) - sf.to_excel(sheet_name='Summary', excel_writer=excel) - - if 'torchbench' in args.suite: - if 'amp_bf16' in args.precision: - if 'inference' in args.mode: - summary.iloc[0:1, 4:5] = passrate_values['target_amp_bf16_inference'] - summary.iloc[1:2, 4:5] = geomean_values['target_amp_bf16_inference'] - if 'training' in args.mode: - summary.iloc[2:3, 4:5] = passrate_values['target_amp_bf16_training'] - summary.iloc[3:4, 4:5] = geomean_values['target_amp_bf16_training'] - if 'amp_fp16' in args.precision: - if 'inference' in args.mode: - summary.iloc[4:5, 4:5] = passrate_values['target_amp_fp16_inference'] - summary.iloc[5:6, 4:5] = geomean_values['target_amp_fp16_inference'] - if 'training' in args.mode: - summary.iloc[6:7, 4:5] = passrate_values['target_amp_fp16_training'] - summary.iloc[7:8, 4:5] = geomean_values['target_amp_fp16_training'] - - if 'bfloat16' in args.precision: - if 'inference' in args.mode: - summary.iloc[8:9, 4:5] = passrate_values['target_bfloat16_inference'] - summary.iloc[9:10, 4:5] = geomean_values['target_bfloat16_inference'] - if 'training' in args.mode: - summary.iloc[10:11, 4:5] = passrate_values['target_bfloat16_training'] - summary.iloc[11:12, 4:5] = geomean_values['target_bfloat16_training'] - if 'float16' in args.precision: - if 'inference' in args.mode: - summary.iloc[12:13, 4:5] = passrate_values['target_float16_inference'] - summary.iloc[13:14, 4:5] = geomean_values['target_float16_inference'] - if 'training' in args.mode: - summary.iloc[14:15, 4:5] = passrate_values['target_float16_training'] - summary.iloc[15:16, 4:5] = geomean_values['target_float16_training'] - if 'float32' in args.precision: - if 'inference' in args.mode: - summary.iloc[16:17, 4:5] = passrate_values['target_float32_inference'] - summary.iloc[17:18, 4:5] = geomean_values['target_float32_inference'] - if 'training' in args.mode: - summary.iloc[18:19, 4:5] = passrate_values['target_float32_training'] - summary.iloc[19:20, 4:5] = geomean_values['target_float32_training'] - - if args.reference is not None: - if 'amp_bf16' in args.precision: - if 'inference' in args.mode: - summary.iloc[0:1, 7:8] = passrate_values['reference_amp_bf16_inference'] - summary.iloc[1:2, 7:8] = geomean_values['reference_amp_bf16_inference'] - if 'training' in args.mode: - summary.iloc[2:3, 7:8] = passrate_values['reference_amp_bf16_training'] - summary.iloc[3:4, 7:8] = geomean_values['reference_amp_bf16_training'] - if 'amp_fp16' in args.precision: - if 'inference' in args.mode: - summary.iloc[4:5, 7:8] = passrate_values['reference_amp_fp16_inference'] - summary.iloc[5:6, 7:8] = geomean_values['reference_amp_fp16_inference'] - if 'training' in args.mode: - summary.iloc[6:7, 7:8] = passrate_values['reference_amp_fp16_training'] - summary.iloc[7:8, 7:8] = geomean_values['reference_amp_fp16_training'] - - if 'bfloat16' in args.precision: - if 'inference' in args.mode: - summary.iloc[8:9, 7:8] = passrate_values['reference_bfloat16_inference'] - summary.iloc[9:10, 7:8] = geomean_values['reference_bfloat16_inference'] - if 'training' in args.mode: - summary.iloc[10:11, 7:8] = passrate_values['reference_bfloat16_training'] - summary.iloc[11:12, 7:8] = geomean_values['reference_bfloat16_training'] - if 'float16' in args.precision: - if 'inference' in args.mode: - summary.iloc[12:13, 7:8] = passrate_values['reference_float16_inference'] - summary.iloc[13:14, 7:8] = geomean_values['reference_float16_inference'] - if 'training' in args.mode: - summary.iloc[14:15, 7:8] = passrate_values['reference_float16_training'] - summary.iloc[15:16, 7:8] = geomean_values['reference_float16_training'] - if 'float32' in args.precision: - if 'inference' in args.mode: - summary.iloc[16:17, 7:8] = passrate_values['reference_float32_inference'] - summary.iloc[17:18, 7:8] = geomean_values['reference_float32_inference'] - if 'training' in args.mode: - summary.iloc[18:19, 7:8] = passrate_values['reference_float32_training'] - summary.iloc[19:20, 7:8] = geomean_values['reference_float32_training'] - - print("====================Summary=============================") - print(summary) - - sf = StyleFrame(summary) - for i in range(1, 11): - sf.set_column_width(i, 22) - for j in range(1, 22): - sf.set_row_height(j, 30) - sf.to_excel(sheet_name='Summary', excel_writer=excel) - - -def generate_report(excel, precision_list, mode_list): - for p in precision_list: - for m in mode_list: - update_details(p, m, excel) - update_summary(excel) - - -def excel_postprocess(file, precison, mode): - wb = file.book - # Summary - ws = wb['Summary'] - for i in range(2, 21, 2): - ws.merge_cells(start_row=i, end_row=i + 1, start_column=1, end_column=1) - # details - for p in precison: - for m in mode: - wdt = wb[p + '_' + m] - wdt.merge_cells(start_row=1, end_row=2, start_column=1, end_column=1) - wdt.merge_cells(start_row=1, end_row=1, start_column=3, end_column=7) - wdt.merge_cells(start_row=1, end_row=1, start_column=8, end_column=12) - wdt.merge_cells(start_row=1, end_row=1, start_column=13, end_column=16) - wb.save(file) - -def html_generate(html_off): - if not html_off: - try: - content = pd.read_excel('inductor_log/' + args.suite + '/Inductor_' + args.suite + '_' + args.precision + '_' + args.mode + '_E2E_Test_Report' + '.xlsx') - #summary= pd.DataFrame(content[-1]).to_html(classes="table",index = False) - #swinfo= pd.DataFrame(content[1]).to_html(classes="table",index = False) - #refer_swinfo_html = '' - #if args.reference is not None: - # refer_swinfo = pd.read_table(args.reference+'/inductor_log/version.txt', sep = '\:', header = None,names=['item', 'commit'],engine='python') - # refer_swinfo_html = refer_swinfo.to_html(classes="table",index = False) - #mt_failures= pd.DataFrame(content[2]).to_html(classes="table",index = False) - #st_failures= pd.DataFrame(content[3]).to_html(classes="table",index = False) - perf_regression= new_performance_regression.to_html(classes="table",index = False) - #failures_regression= new_failures.to_html(classes="table",index = False) - #with open('inductor_log/inductor_model_bench.html',mode = "a") as f,open('inductor_log/inductor_perf_regression.html',mode = "a") as perf_f,open('inductor_log/inductor_failures.html',mode = "a") as failure_f: - with open('inductor_log/' + args.suite + '/inductor_perf_regression.html',mode = "a") as perf_f: - #f.write(html_head()+"

Summary

"+summary+"

SW info

"+swinfo+"

Multi-threads Failures

"+mt_failures+"

Single-thread Failures

"+st_failures+"

new_perf_regression

"+perf_regression+"

new_failures

"+failures_regression+f"

image: docker pull ccr-registry.caas.intel.com/pytorch/pt_inductor:{args.image_tag}

"+html_tail()) - perf_f.write("

new_perf_regression

"+perf_regression) - #failure_f.write(f"

new_failures in {str((datetime.now() - timedelta(days=2)).date())}

"+failures_regression) - #f.close() - perf_f.close() - #failure_f.close() - except: - print("html_generate_failed") - pass - - -if __name__ == '__main__': - excel = StyleFrame.ExcelWriter('inductor_log/' + str(args.suite) + '/Inductor_' + args.suite + '_E2E_Test_Perf_Report.xlsx') - generate_report(excel, args.precision, args.mode) - excel_postprocess(excel, args.precision, args.mode) - html_generate(args.html_off) diff --git a/.github/scripts/inductor_summary.py b/.github/scripts/inductor_summary.py new file mode 100644 index 000000000..e11147664 --- /dev/null +++ b/.github/scripts/inductor_summary.py @@ -0,0 +1,700 @@ +import argparse + +import pandas as pd +from scipy.stats import gmean +from styleframe import StyleFrame, Styler, utils + +parser = argparse.ArgumentParser(description="Generate report") +parser.add_argument('-s', '--suite', default=["huggingface"], nargs='*', type=str, help='model suite name') +parser.add_argument('-p', '--precision', default=["amp_fp16", "float32"], nargs='*', type=str, help='precision') +parser.add_argument('-r', '--reference', type=str, help='reference log files') +parser.add_argument('-m', '--mode', default=["inference", "training"], nargs='*', type=str, help='mode name') +parser.add_argument('-sc', '--scenario', default=["performance"], nargs='*', type=str, help='Test scenario set') +args = parser.parse_args() + +passrate_values = {} +geomean_values = {} + +new_performance_regression=pd.DataFrame() + +failure_style = Styler(bg_color='#FF0000', font_color=utils.colors.black) +regression_style = Styler(bg_color='#F0E68C', font_color=utils.colors.red) +improve_style = Styler(bg_color='#00FF00', font_color=utils.colors.black) + +# refer https://github.com/pytorch/pytorch/blob/main/benchmarks/dynamo/runner.py#L757-L778 + + +def percentage(part, whole, decimals=2): + if whole == 0: + return 0 + return round(100 * float(part) / float(whole), decimals) + + +def get_passing_entries(df, column_name, scenario): + if scenario == 'performance': + return df[column_name][df[column_name] > 0] + else: + return df[df[column_name].notnull()] + + +def caculate_geomean(df, column_name, scenario): + cleaned_df = get_passing_entries(df, column_name, scenario).clip(1) + if cleaned_df.empty: + return "0.0x" + return f"{gmean(cleaned_df):.2f}x" + + +def caculate_passrate(df, key_word, scenario): + total = len(df.index) + if scenario == 'performance': + passing = df[df[key_word] > 0.0][key_word].count() + else: + df = get_passing_entries(df, key_word, scenario) + passing = df[df[key_word].fillna('').str.contains('pass')][key_word].count() + perc = int(percentage(passing, total, decimals=0)) + return f"{perc}%, {passing}/{total}" + +def get_perf_csv(scenario, precision, mode, suite): + target_path = 'inductor_log/' + suite + '/' + precision + '/inductor_' + suite + '_' + precision + '_' + mode + '_xpu_' + scenario + '.csv' + target_ori_data = pd.read_csv(target_path, header=0, encoding='utf-8') + target_data = target_ori_data.copy() + target_data.sort_values(by=['name']) + + if args.reference is not None: + reference_file_path = args.reference + '/inductor_log/' + suite + '/' + precision + '/inductor_' + suite + '_' + precision + '_' + mode + '_xpu_' + scenario + '.csv' + reference_ori_data = pd.read_csv(reference_file_path, header=0, encoding='utf-8') + reference_data = reference_ori_data.copy() + reference_data.sort_values(by=['name']) + data = pd.merge(target_data,reference_data,on=['name'],how= 'outer') + return data + else: + return target_data + +def process(input, scenario, precision, mode): + global geomean_values, passrate_values + processed_data = pd.DataFrame() + if input is not None and scenario == 'performance': + if args.reference is None: + data_new = input[['name', 'batch_size', 'speedup', 'abs_latency','compilation_latency']].rename(columns={'name': 'name', 'batch_size': 'batch_size', 'speedup': 'speedup', "abs_latency": 'inductor', "compilation_latency": 'compilation_latency'}) + data_new['inductor'] = data_new['inductor'].apply(pd.to_numeric, errors='coerce').div(1000) + data_new['speedup'] = data_new['speedup'].apply(pd.to_numeric, errors='coerce').fillna(0.0) + data_new['eager'] = data_new['speedup'] * data_new['inductor'] + geomean_values['target_' + str(precision) + '_' + str(mode)] = caculate_geomean(data_new, 'speedup', scenario) + passrate_values['target_' + str(precision) + '_' + str(mode)] = caculate_passrate(data_new, 'inductor', scenario) + processed_data = StyleFrame({'name': list(data_new['name']), + 'batch_size': list(data_new['batch_size']), + 'speedup': list(data_new['speedup']), + 'inductor': list(data_new['inductor']), + 'eager': list(data_new['eager']), + 'compilation_latency': list(data_new['compilation_latency'])}) + processed_data.set_column_width(1, 10) + processed_data.set_column_width(2, 18) + processed_data.set_column_width(3, 18) + processed_data.set_column_width(4, 18) + processed_data.set_column_width(5, 15) + processed_data.set_column_width(6, 20) + processed_data.set_row_height(rows=processed_data.row_indexes, height=15) + else: + data_new=input[['name','batch_size_x','speedup_x','abs_latency_x','compilation_latency_x']].rename(columns={'name':'name','batch_size_x':'batch_size_new','speedup_x':'speed_up_new',"abs_latency_x":'inductor_new',"compilation_latency_x":'compilation_latency_new'}) + data_new['inductor_new']=data_new['inductor_new'].astype(float).div(1000) + data_new['speed_up_new']=data_new['speed_up_new'].apply(pd.to_numeric, errors='coerce').fillna(0.0) + data_new['eager_new'] = data_new['speed_up_new'] * data_new['inductor_new'] + geomean_values['target_' + str(precision) + '_' + str(mode)] = caculate_geomean(data_new, 'speed_up_new', scenario) + passrate_values['target_' + str(precision) + '_' + str(mode)] = caculate_passrate(data_new, 'inductor_new', scenario) + data_old=input[['batch_size_y','speedup_y','abs_latency_y','compilation_latency_y']].rename(columns={'batch_size_y':'batch_size_old','speedup_y':'speed_up_old',"abs_latency_y":'inductor_old',"compilation_latency_y":'compilation_latency_old'}) + data_old['inductor_old']=data_old['inductor_old'].astype(float).div(1000) + data_old['speed_up_old']=data_old['speed_up_old'].apply(pd.to_numeric, errors='coerce').fillna(0.0) + data_old['eager_old'] = data_old['speed_up_old'] * data_old['inductor_old'] + input['speedup_x']=input['speedup_x'].apply(pd.to_numeric, errors='coerce').fillna(0.0) + input['speedup_y']=input['speedup_y'].apply(pd.to_numeric, errors='coerce').fillna(0.0) + geomean_values['reference_' + str(precision) + '_' + str(mode)] = caculate_geomean(data_old, 'speed_up_old', scenario) + passrate_values['reference_' + str(precision) + '_' + str(mode)] = caculate_passrate(data_old, 'inductor_old', scenario) + data_ratio= pd.DataFrame(round(input['speedup_x'] / input['speedup_y'],2),columns=['Ratio Speedup(New/old)']) + data_ratio['Eager Ratio(old/new)'] = pd.DataFrame(round(data_old['eager_old'] / data_new['eager_new'],2)) + data_ratio['Inductor Ratio(old/new)'] = pd.DataFrame(round(data_old['inductor_old'] / data_new['inductor_new'],2)) + data_ratio['Compilation_latency_Ratio(old/new)'] = pd.DataFrame(round(data_old['compilation_latency_old'] / data_new['compilation_latency_new'],2)) + + combined_data = pd.DataFrame({ + 'name': list(data_new['name']), + 'batch_size_new': list(data_new['batch_size_new']), + 'speed_up_new': list(data_new['speed_up_new']), + 'inductor_new': list(data_new['inductor_new']), + 'eager_new': list(data_new['eager_new']), + 'compilation_latency_new': list(data_new['compilation_latency_new']), + 'batch_size_old': list(data_old['batch_size_old']), + 'speed_up_old': list(data_old['speed_up_old']), + 'inductor_old': list(data_old['inductor_old']), + 'eager_old': list(data_old['eager_old']), + 'compilation_latency_old': list(data_old['compilation_latency_old']), + 'Ratio Speedup(New/old)': list(data_ratio['Ratio Speedup(New/old)']), + 'Eager Ratio(old/new)': list(data_ratio['Eager Ratio(old/new)']), + 'Inductor Ratio(old/new)': list(data_ratio['Inductor Ratio(old/new)']), + 'Compilation_latency_Ratio(old/new)': list(data_ratio['Compilation_latency_Ratio(old/new)']) + }) + processed_data = StyleFrame(combined_data) + processed_data.set_column_width(1, 10) + processed_data.set_column_width(2, 18) + processed_data.set_column_width(3, 18) + processed_data.set_column_width(4, 18) + processed_data.set_column_width(5, 15) + processed_data.set_column_width(6, 20) + processed_data.set_column_width(7, 18) + processed_data.set_column_width(8, 18) + processed_data.set_column_width(9, 18) + processed_data.set_column_width(10, 15) + processed_data.set_column_width(11, 20) + processed_data.set_column_width(12, 28) + processed_data.set_column_width(13, 28) + processed_data.set_column_width(14, 28) + processed_data.set_column_width(15, 32) + processed_data.apply_style_by_indexes(indexes_to_style=processed_data[processed_data['batch_size_new'] == 0], styler_obj=failure_style) + processed_data.apply_style_by_indexes(indexes_to_style=processed_data[(processed_data['Inductor Ratio(old/new)'] > 0) & (processed_data['Inductor Ratio(old/new)'] < 0.9)],styler_obj=regression_style) + global new_performance_regression + regression = processed_data.loc[(processed_data['Inductor Ratio(old/new)'] > 0) & (processed_data['Inductor Ratio(old/new)'] < 0.9)] + regression = regression.copy() + regression.loc[0] = list(regression.shape[1]*'*') + new_performance_regression = pd.concat([new_performance_regression,regression]) + processed_data.apply_style_by_indexes(indexes_to_style=processed_data[processed_data['Inductor Ratio(old/new)'] > 1.1],styler_obj=improve_style) + processed_data.set_row_height(rows=processed_data.row_indexes, height=15) + if input is not None and scenario == 'accuracy': + if args.reference is None: + data_new = input[['name', 'batch_size', 'accuracy']].rename(columns={'name': 'name', 'batch_size': 'batch_size', 'accuracy': 'accuracy'}) + passrate_values['target_' + str(precision) + '_' + str(mode)] = caculate_passrate(data_new, 'accuracy', scenario) + processed_data = data_new + processed_data = StyleFrame({'name': list(data_new['name']), + 'batch_size': list(data_new['batch_size']), + 'accuracy': list(data_new['accuracy'])}) + processed_data.set_column_width(1, 10) + processed_data.set_column_width(2, 18) + processed_data.set_column_width(3, 18) + processed_data.set_row_height(rows=processed_data.row_indexes, height=15) + else: + data_new=input[['name','batch_size_x','accuracy_x']].rename(columns={'name':'name','batch_size_x':'batch_size_new','accuracy_x':'accuracy_new'}) + passrate_values['target_' + str(precision) + '_' + str(mode)] = caculate_passrate(data_new, 'accuracy_new', scenario) + data_old=input[['batch_size_y','accuracy_y']].rename(columns={'batch_size_y':'batch_size_old','accuracy_y':'accuracy_old'}) + passrate_values['reference_' + str(precision) + '_' + str(mode)] = caculate_passrate(data_old, 'accuracy_old', scenario) + data_comp = pd.DataFrame((data_new['accuracy_new'] != 'pass') & (data_old['accuracy_old'] == 'pass'),columns=['Accuracy regression']) + combined_data = pd.DataFrame({ + 'name': list(data_new['name']), + 'batch_size_new': list(data_new['batch_size_new']), + 'accuracy_new': list(data_new['accuracy_new']), + 'batch_size_old': list(data_old['batch_size_old']), + 'accuracy_old': list(data_old['accuracy_old']), + 'Accuracy regression': list(data_comp['Accuracy regression']) + }) + processed_data = StyleFrame(combined_data) + processed_data.set_column_width(1, 10) + processed_data.set_column_width(2, 18) + processed_data.set_column_width(3, 18) + processed_data.set_column_width(4, 18) + processed_data.set_column_width(5, 15) + processed_data.set_column_width(6, 20) + processed_data.apply_style_by_indexes(indexes_to_style=processed_data[(processed_data['Accuracy regression'] == 'regression')],styler_obj=regression_style) + processed_data.set_row_height(rows=processed_data.row_indexes, height=15) + return processed_data + +def update_details(scenario, precision, mode, suite, excel): + if scenario == 'performance': + h = {"A": 'Model suite', "B": '', "C": "target", "D": '', "E": '', "F": '', "G": '',"H": args.reference, "I": '', "J": '',"K": '',"L":'',"M": 'Result Comp',"N": '',"O": '',"P":''} + if args.reference is None: + h = {"A": 'Model suite', "B": '', "C": "target", "D": '', "E": '', "F": '', "G": ''} + else: + h = {"A": 'Model suite', "B": '', "C": "target", "D": '', "E": args.reference, "F": '', "G": 'Result Comp'} + if args.reference is None: + h = {"A": 'Model suite', "B": '', "C": "target", "D": ''} + head = StyleFrame(pd.DataFrame(h, index=[0])) + head.set_column_width(1, 15) + head.set_row_height(rows=[1], height=15) + head.to_excel(excel_writer=excel, sheet_name=suite + '_' + precision + '_' + mode[0:3] + '_' + scenario[0:3], index=False, startrow=0, header=False) + target_raw_data = get_perf_csv(scenario, precision, mode, suite) + target_data = process(target_raw_data, scenario, precision, mode) + target_data.to_excel(excel_writer=excel, sheet_name=suite + '_' + precision + '_' + mode[0:3] + '_' + scenario[0:3], index=False, startrow=1, startcol=1) + +def update_summary(excel, scenario, suite): + if scenario == 'performance': + data = { + 'Test Secnario': ['AMP_BF16 Inference', ' ', 'AMP_BF16 Training', ' ', 'AMP_FP16 Inference', ' ', 'AMP_FP16 Training', ' ', 'BF16 Inference', ' ', 'BF16 Training', ' ', 'FP16 Inference', ' ', 'FP16 Training', ' ', 'FP32 Inference', ' ', 'FP32 Training', ' '], + 'Comp Item': ['Pass Rate', 'Geomean Speedup', 'Pass Rate', 'Geomean Speedup', 'Pass Rate', 'Geomean Speedup', 'Pass Rate', 'Geomean Speedup', 'Pass Rate', 'Geomean Speedup', 'Pass Rate', 'Geomean Speedup', 'Pass Rate', 'Geomean Speedup', 'Pass Rate', 'Geomean Speedup', 'Pass Rate', 'Geomean Speedup', 'Pass Rate', 'Geomean Speedup'], + 'Date': [' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '], + 'Compiler': ['inductor', 'inductor', 'inductor', 'inductor', 'inductor', 'inductor', 'inductor', 'inductor', 'inductor', 'inductor', 'inductor', 'inductor', 'inductor', 'inductor', 'inductor', 'inductor', 'inductor', 'inductor', 'inductor', 'inductor'], + 'torchbench': [' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '], + 'huggingface': [' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '], + 'timm_models ': [' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '], + 'ref_torchbench ': [' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '], + 'refer_huggingface ': [' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '], + 'refer_timm_models ': [' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '] + } + summary = pd.DataFrame(data) + + if suite == 'huggingface': + if 'amp_bf16' in args.precision: + if 'inference' in args.mode: + summary.iloc[0:1, 5:6] = passrate_values['target_amp_bf16_inference'] + summary.iloc[1:2, 5:6] = geomean_values['target_amp_bf16_inference'] + if 'training' in args.mode: + summary.iloc[2:3, 5:6] = passrate_values['target_amp_bf16_training'] + summary.iloc[3:4, 5:6] = geomean_values['target_amp_bf16_training'] + if 'amp_fp16' in args.precision: + if 'inference' in args.mode: + summary.iloc[4:5, 5:6] = passrate_values['target_amp_fp16_inference'] + summary.iloc[5:6, 5:6] = geomean_values['target_amp_fp16_inference'] + if 'training' in args.mode: + summary.iloc[6:7, 5:6] = passrate_values['target_amp_fp16_training'] + summary.iloc[7:8, 5:6] = geomean_values['target_amp_fp16_training'] + + if 'bfloat16' in args.precision: + if 'inference' in args.mode: + summary.iloc[8:9, 5:6] = passrate_values['target_bfloat16_inference'] + summary.iloc[9:10, 5:6] = geomean_values['target_bfloat16_inference'] + if 'training' in args.mode: + summary.iloc[10:11, 5:6] = passrate_values['target_bfloat16_training'] + summary.iloc[11:12, 5:6] = geomean_values['target_bfloat16_training'] + if 'float16' in args.precision: + if 'inference' in args.mode: + summary.iloc[12:13, 5:6] = passrate_values['target_float16_inference'] + summary.iloc[13:14, 5:6] = geomean_values['target_float16_inference'] + if 'training' in args.mode: + summary.iloc[14:15, 5:6] = passrate_values['target_float16_training'] + summary.iloc[15:16, 5:6] = geomean_values['target_float16_training'] + if 'float32' in args.precision: + if 'inference' in args.mode: + summary.iloc[16:17, 5:6] = passrate_values['target_float32_inference'] + summary.iloc[17:18, 5:6] = geomean_values['target_float32_inference'] + if 'training' in args.mode: + summary.iloc[18:19, 5:6] = passrate_values['target_float32_training'] + summary.iloc[19:20, 5:6] = geomean_values['target_float32_training'] + + if args.reference is not None: + if 'amp_bf16' in args.precision: + if 'inference' in args.mode: + summary.iloc[0:1, 8:9] = passrate_values['reference_amp_bf16_inference'] + summary.iloc[1:2, 8:9] = geomean_values['reference_amp_bf16_inference'] + if 'training' in args.mode: + summary.iloc[2:3, 8:9] = passrate_values['reference_amp_bf16_training'] + summary.iloc[3:4, 8:9] = geomean_values['reference_amp_bf16_training'] + if 'amp_fp16' in args.precision: + if 'inference' in args.mode: + summary.iloc[4:5, 8:9] = passrate_values['reference_amp_fp16_inference'] + summary.iloc[5:6, 8:9] = geomean_values['reference_amp_fp16_inference'] + if 'training' in args.mode: + summary.iloc[6:7, 8:9] = passrate_values['reference_amp_fp16_training'] + summary.iloc[7:8, 8:9] = geomean_values['reference_amp_fp16_training'] + + if 'bfloat16' in args.precision: + if 'inference' in args.mode: + summary.iloc[8:9, 8:9] = passrate_values['reference_bfloat16_inference'] + summary.iloc[9:10, 8:9] = geomean_values['reference_bfloat16_inference'] + if 'training' in args.mode: + summary.iloc[10:11, 8:9] = passrate_values['reference_bfloat16_training'] + summary.iloc[11:12, 8:9] = geomean_values['reference_bfloat16_training'] + if 'float16' in args.precision: + if 'inference' in args.mode: + summary.iloc[12:13, 8:9] = passrate_values['reference_float16_inference'] + summary.iloc[13:14, 8:9] = geomean_values['reference_float16_inference'] + if 'training' in args.mode: + summary.iloc[14:15, 8:9] = passrate_values['reference_float16_training'] + summary.iloc[15:16, 8:9] = geomean_values['reference_float16_training'] + if 'float32' in args.precision: + if 'inference' in args.mode: + summary.iloc[16:17, 8:9] = passrate_values['reference_float32_inference'] + summary.iloc[17:18, 8:9] = geomean_values['reference_float32_inference'] + if 'training' in args.mode: + summary.iloc[18:19, 8:9] = passrate_values['reference_float32_training'] + summary.iloc[19:20, 8:9] = geomean_values['reference_float32_training'] + + if suite == 'timm_models': + if 'amp_bf16' in args.precision: + if 'inference' in args.mode: + summary.iloc[0:1, 6:7] = passrate_values['target_amp_bf16_inference'] + summary.iloc[1:2, 6:7] = geomean_values['target_amp_bf16_inference'] + if 'training' in args.mode: + summary.iloc[2:3, 6:7] = passrate_values['target_amp_bf16_training'] + summary.iloc[3:4, 6:7] = geomean_values['target_amp_bf16_training'] + if 'amp_fp16' in args.precision: + if 'inference' in args.mode: + summary.iloc[4:5, 6:7] = passrate_values['target_amp_fp16_inference'] + summary.iloc[5:6, 6:7] = geomean_values['target_amp_fp16_inference'] + if 'training' in args.mode: + summary.iloc[6:7, 6:7] = passrate_values['target_amp_fp16_training'] + summary.iloc[7:8, 6:7] = geomean_values['target_amp_fp16_training'] + + if 'bfloat16' in args.precision: + if 'inference' in args.mode: + summary.iloc[8:9, 6:7] = passrate_values['target_bfloat16_inference'] + summary.iloc[9:10, 6:7] = geomean_values['target_bfloat16_inference'] + if 'training' in args.mode: + summary.iloc[10:11, 6:7] = passrate_values['target_bfloat16_training'] + summary.iloc[11:12, 6:7] = geomean_values['target_bfloat16_training'] + if 'float16' in args.precision: + if 'inference' in args.mode: + summary.iloc[12:13, 6:7] = passrate_values['target_float16_inference'] + summary.iloc[13:14, 6:7] = geomean_values['target_float16_inference'] + if 'training' in args.mode: + summary.iloc[14:15, 6:7] = passrate_values['target_float16_training'] + summary.iloc[15:16, 6:7] = geomean_values['target_float16_training'] + if 'float32' in args.precision: + if 'inference' in args.mode: + summary.iloc[16:17, 6:7] = passrate_values['target_float32_inference'] + summary.iloc[17:18, 6:7] = geomean_values['target_float32_inference'] + if 'training' in args.mode: + summary.iloc[18:19, 6:7] = passrate_values['target_float32_training'] + summary.iloc[19:20, 6:7] = geomean_values['target_float32_training'] + + if args.reference is not None: + if 'amp_bf16' in args.precision: + if 'inference' in args.mode: + summary.iloc[0:1, 9:10] = passrate_values['reference_amp_bf16_inference'] + summary.iloc[1:2, 9:10] = geomean_values['reference_amp_bf16_inference'] + if 'training' in args.mode: + summary.iloc[2:3, 9:10] = passrate_values['reference_amp_bf16_training'] + summary.iloc[3:4, 9:10] = geomean_values['reference_amp_bf16_training'] + + if 'amp_fp16' in args.precision: + if 'inference' in args.mode: + summary.iloc[4:5, 9:10] = passrate_values['reference_amp_fp16_inference'] + summary.iloc[5:6, 9:10] = geomean_values['reference_amp_fp16_inference'] + if 'training' in args.mode: + summary.iloc[6:7, 9:10] = passrate_values['reference_amp_fp16_training'] + summary.iloc[7:8, 9:10] = geomean_values['reference_amp_fp16_training'] + + if 'bfloat16' in args.precision: + if 'inference' in args.mode: + summary.iloc[8:9, 9:10] = passrate_values['reference_bfloat16_inference'] + summary.iloc[9:10, 9:10] = geomean_values['reference_bfloat16_inference'] + if 'training' in args.mode: + summary.iloc[10:11, 9:10] = passrate_values['reference_bfloat16_training'] + summary.iloc[11:12, 9:10] = geomean_values['reference_bfloat16_training'] + if 'float16' in args.precision: + if 'inference' in args.mode: + summary.iloc[12:13, 9:10] = passrate_values['reference_float16_inference'] + summary.iloc[13:14, 9:10] = geomean_values['reference_float16_inference'] + if 'training' in args.mode: + summary.iloc[14:15, 9:10] = passrate_values['reference_float16_training'] + summary.iloc[15:16, 9:10] = geomean_values['reference_float16_training'] + if 'float32' in args.precision: + if 'inference' in args.mode: + summary.iloc[16:17, 9:10] = passrate_values['reference_float32_inference'] + summary.iloc[17:18, 9:10] = geomean_values['reference_float32_inference'] + if 'training' in args.mode: + summary.iloc[18:19, 9:10] = passrate_values['reference_float32_training'] + summary.iloc[19:20, 9:10] = geomean_values['reference_float32_training'] + + if suite == 'torchbench': + if 'amp_bf16' in args.precision: + if 'inference' in args.mode: + summary.iloc[0:1, 4:5] = passrate_values['target_amp_bf16_inference'] + summary.iloc[1:2, 4:5] = geomean_values['target_amp_bf16_inference'] + if 'training' in args.mode: + summary.iloc[2:3, 4:5] = passrate_values['target_amp_bf16_training'] + summary.iloc[3:4, 4:5] = geomean_values['target_amp_bf16_training'] + if 'amp_fp16' in args.precision: + if 'inference' in args.mode: + summary.iloc[4:5, 4:5] = passrate_values['target_amp_fp16_inference'] + summary.iloc[5:6, 4:5] = geomean_values['target_amp_fp16_inference'] + if 'training' in args.mode: + summary.iloc[6:7, 4:5] = passrate_values['target_amp_fp16_training'] + summary.iloc[7:8, 4:5] = geomean_values['target_amp_fp16_training'] + + if 'bfloat16' in args.precision: + if 'inference' in args.mode: + summary.iloc[8:9, 4:5] = passrate_values['target_bfloat16_inference'] + summary.iloc[9:10, 4:5] = geomean_values['target_bfloat16_inference'] + if 'training' in args.mode: + summary.iloc[10:11, 4:5] = passrate_values['target_bfloat16_training'] + summary.iloc[11:12, 4:5] = geomean_values['target_bfloat16_training'] + if 'float16' in args.precision: + if 'inference' in args.mode: + summary.iloc[12:13, 4:5] = passrate_values['target_float16_inference'] + summary.iloc[13:14, 4:5] = geomean_values['target_float16_inference'] + if 'training' in args.mode: + summary.iloc[14:15, 4:5] = passrate_values['target_float16_training'] + summary.iloc[15:16, 4:5] = geomean_values['target_float16_training'] + if 'float32' in args.precision: + if 'inference' in args.mode: + summary.iloc[16:17, 4:5] = passrate_values['target_float32_inference'] + summary.iloc[17:18, 4:5] = geomean_values['target_float32_inference'] + if 'training' in args.mode: + summary.iloc[18:19, 4:5] = passrate_values['target_float32_training'] + summary.iloc[19:20, 4:5] = geomean_values['target_float32_training'] + + if args.reference is not None: + if 'amp_bf16' in args.precision: + if 'inference' in args.mode: + summary.iloc[0:1, 7:8] = passrate_values['reference_amp_bf16_inference'] + summary.iloc[1:2, 7:8] = geomean_values['reference_amp_bf16_inference'] + if 'training' in args.mode: + summary.iloc[2:3, 7:8] = passrate_values['reference_amp_bf16_training'] + summary.iloc[3:4, 7:8] = geomean_values['reference_amp_bf16_training'] + if 'amp_fp16' in args.precision: + if 'inference' in args.mode: + summary.iloc[4:5, 7:8] = passrate_values['reference_amp_fp16_inference'] + summary.iloc[5:6, 7:8] = geomean_values['reference_amp_fp16_inference'] + if 'training' in args.mode: + summary.iloc[6:7, 7:8] = passrate_values['reference_amp_fp16_training'] + summary.iloc[7:8, 7:8] = geomean_values['reference_amp_fp16_training'] + + if 'bfloat16' in args.precision: + if 'inference' in args.mode: + summary.iloc[8:9, 7:8] = passrate_values['reference_bfloat16_inference'] + summary.iloc[9:10, 7:8] = geomean_values['reference_bfloat16_inference'] + if 'training' in args.mode: + summary.iloc[10:11, 7:8] = passrate_values['reference_bfloat16_training'] + summary.iloc[11:12, 7:8] = geomean_values['reference_bfloat16_training'] + if 'float16' in args.precision: + if 'inference' in args.mode: + summary.iloc[12:13, 7:8] = passrate_values['reference_float16_inference'] + summary.iloc[13:14, 7:8] = geomean_values['reference_float16_inference'] + if 'training' in args.mode: + summary.iloc[14:15, 7:8] = passrate_values['reference_float16_training'] + summary.iloc[15:16, 7:8] = geomean_values['reference_float16_training'] + if 'float32' in args.precision: + if 'inference' in args.mode: + summary.iloc[16:17, 7:8] = passrate_values['reference_float32_inference'] + summary.iloc[17:18, 7:8] = geomean_values['reference_float32_inference'] + if 'training' in args.mode: + summary.iloc[18:19, 7:8] = passrate_values['reference_float32_training'] + summary.iloc[19:20, 7:8] = geomean_values['reference_float32_training'] + + print(f"===================={scenario}Summary=============================") + print(summary) + + sf = StyleFrame(summary) + for i in range(1, 11): + sf.set_column_width(i, 22) + for j in range(1, 22): + sf.set_row_height(j, 30) + sf.to_excel(sheet_name=suite + '_' + scenario + '_Summary', excel_writer=excel) + + else: + data = { + 'Test Secnario': ['AMP_BF16 Inference', 'AMP_BF16 Training', 'AMP_FP16 Inference', 'AMP_FP16 Training', 'BF16 Inference', 'BF16 Training', 'FP16 Inference', 'FP16 Training', 'FP32 Inference', 'FP32 Training'], + 'Comp Item': ['Pass Rate', 'Pass Rate', 'Pass Rate', 'Pass Rate', 'Pass Rate', 'Pass Rate', 'Pass Rate', 'Pass Rate', 'Pass Rate', 'Pass Rate'], + 'compiler': ['inductor', 'inductor', 'inductor', 'inductor', 'inductor', 'inductor', 'inductor', 'inductor', 'inductor', 'inductor'], + 'torchbench': [' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '], + 'huggingface': [' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '], + 'timm_models ': [' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '], + 'refer_torchbench ': [' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '], + 'refer_huggingface ': [' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '], + 'refer_timm_models ': [' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', ' '] + } + summary = pd.DataFrame(data) + + if suite == 'huggingface': + if 'amp_bf16' in args.precision: + if 'inference' in args.mode: + # passrate + summary.iloc[0:1, 4:5] = passrate_values['target_amp_bf16_inference'] + if 'training' in args.mode: + summary.iloc[1:2, 4:5] = passrate_values['target_amp_bf16_training'] + + if 'amp_fp16' in args.precision: + if 'inference' in args.mode: + summary.iloc[2:3, 4:5] = passrate_values['target_amp_fp16_inference'] + if 'training' in args.mode: + summary.iloc[3:4, 4:5] = passrate_values['target_amp_fp16_training'] + + if 'bfloat16' in args.precision: + if 'inference' in args.mode: + summary.iloc[4:5, 4:5] = passrate_values['target_bfloat16_inference'] + if 'training' in args.mode: + summary.iloc[5:6, 4:5] = passrate_values['target_bfloat16_training'] + if 'float16' in args.precision: + if 'inference' in args.mode: + summary.iloc[6:7, 4:5] = passrate_values['target_float16_inference'] + if 'training' in args.mode: + summary.iloc[7:8, 4:5] = passrate_values['target_float16_training'] + if 'float32' in args.precision: + if 'inference' in args.mode: + summary.iloc[8:9, 4:5] = passrate_values['target_float32_inference'] + if 'training' in args.mode: + summary.iloc[9:10, 4:5] = passrate_values['target_float32_training'] + + if args.reference is not None: + if 'amp_bf16' in args.precision: + if 'inference' in args.mode: + # passrate + summary.iloc[0:1, 7:8] = passrate_values['reference_amp_bf16_inference'] + if 'training' in args.mode: + summary.iloc[1:2, 7:8] = passrate_values['reference_amp_bf16_training'] + if 'amp_fp16' in args.precision: + if 'inference' in args.mode: + summary.iloc[2:3, 7:8] = passrate_values['reference_amp_fp16_inference'] + if 'training' in args.mode: + summary.iloc[3:4, 7:8] = passrate_values['reference_amp_fp16_training'] + + if 'bfloat16' in args.precision: + if 'inference' in args.mode: + summary.iloc[4:5, 7:8] = passrate_values['reference_bfloat16_inference'] + if 'training' in args.mode: + summary.iloc[5:6, 7:8] = passrate_values['reference_bfloat16_training'] + if 'float16' in args.precision: + if 'inference' in args.mode: + summary.iloc[6:7, 7:8] = passrate_values['reference_float16_inference'] + if 'training' in args.mode: + summary.iloc[7:8, 7:8] = passrate_values['reference_float16_training'] + if 'float32' in args.precision: + if 'inference' in args.mode: + summary.iloc[8:9, 7:8] = passrate_values['reference_float32_inference'] + if 'training' in args.mode: + summary.iloc[9:10, 7:8] = passrate_values['reference_float32_training'] + + if suite == 'timm_models': + if 'amp_bf16' in args.precision: + if 'inference' in args.mode: + # passrate + summary.iloc[0:1, 5:6] = passrate_values['target_amp_bf16_inference'] + if 'training' in args.mode: + summary.iloc[1:2, 5:6] = passrate_values['target_amp_bf16_training'] + if 'amp_fp16' in args.precision: + if 'inference' in args.mode: + summary.iloc[2:3, 5:6] = passrate_values['target_amp_fp16_inference'] + if 'training' in args.mode: + summary.iloc[3:4, 5:6] = passrate_values['target_amp_fp16_training'] + + if 'bfloat16' in args.precision: + if 'inference' in args.mode: + summary.iloc[4:5, 5:6] = passrate_values['target_bfloat16_inference'] + if 'training' in args.mode: + summary.iloc[5:6, 5:6] = passrate_values['target_bfloat16_training'] + if 'float16' in args.precision: + if 'inference' in args.mode: + summary.iloc[6:7, 5:6] = passrate_values['target_float16_inference'] + if 'training' in args.mode: + summary.iloc[7:8, 5:6] = passrate_values['target_float16_training'] + if 'float32' in args.precision: + if 'inference' in args.mode: + summary.iloc[8:9, 5:6] = passrate_values['target_float32_inference'] + if 'training' in args.mode: + summary.iloc[9:10, 5:6] = passrate_values['target_float32_training'] + + if args.reference is not None: + if 'amp_bf16' in args.precision: + if 'inference' in args.mode: + # passrate + summary.iloc[0:1, 8:9] = passrate_values['reference_amp_bf16_inference'] + if 'training' in args.mode: + summary.iloc[1:2, 8:9] = passrate_values['reference_amp_bf16_training'] + if 'amp_fp16' in args.precision: + if 'inference' in args.mode: + summary.iloc[2:3, 8:9] = passrate_values['reference_amp_fp16_inference'] + if 'training' in args.mode: + summary.iloc[3:4, 8:9] = passrate_values['reference_amp_fp16_training'] + + if 'bfloat16' in args.precision: + if 'inference' in args.mode: + summary.iloc[4:5, 8:9] = passrate_values['reference_bfloat16_inference'] + if 'training' in args.mode: + summary.iloc[5:6, 8:9] = passrate_values['reference_bfloat16_training'] + if 'float16' in args.precision: + if 'inference' in args.mode: + summary.iloc[6:7, 8:9] = passrate_values['reference_float16_inference'] + if 'training' in args.mode: + summary.iloc[7:8, 8:9] = passrate_values['reference_float16_training'] + if 'float32' in args.precision: + if 'inference' in args.mode: + summary.iloc[8:9, 8:9] = passrate_values['reference_float32_inference'] + if 'training' in args.mode: + summary.iloc[9:10, 8:9] = passrate_values['reference_float32_training'] + + if suite == 'torchbench': + if 'amp_bf16' in args.precision: + if 'inference' in args.mode: + # passrate + summary.iloc[0:1, 3:4] = passrate_values['target_amp_bf16_inference'] + if 'training' in args.mode: + summary.iloc[1:2, 3:4] = passrate_values['target_amp_bf16_training'] + if 'amp_fp16' in args.precision: + if 'inference' in args.mode: + summary.iloc[2:3, 3:4] = passrate_values['target_amp_fp16_inference'] + if 'training' in args.mode: + summary.iloc[3:4, 3:4] = passrate_values['target_amp_fp16_training'] + + if 'bfloat16' in args.precision: + if 'inference' in args.mode: + summary.iloc[4:5, 3:4] = passrate_values['target_bfloat16_inference'] + if 'training' in args.mode: + summary.iloc[5:6, 3:4] = passrate_values['target_bfloat16_training'] + if 'float16' in args.precision: + if 'inference' in args.mode: + summary.iloc[6:7, 3:4] = passrate_values['target_float16_inference'] + if 'training' in args.mode: + summary.iloc[7:8, 3:4] = passrate_values['target_float16_training'] + if 'float32' in args.precision: + if 'inference' in args.mode: + summary.iloc[8:9, 3:4] = passrate_values['target_float32_inference'] + if 'training' in args.mode: + summary.iloc[9:10, 3:4] = passrate_values['target_float32_training'] + + if args.reference is not None: + if 'amp_bf16' in args.precision: + if 'inference' in args.mode: + # passrate + summary.iloc[0:1, 6:7] = passrate_values['reference_amp_bf16_inference'] + if 'training' in args.mode: + summary.iloc[1:2, 6:7] = passrate_values['reference_amp_bf16_training'] + + if 'amp_fp16' in args.precision: + if 'inference' in args.mode: + summary.iloc[2:3, 6:7] = passrate_values['reference_amp_fp16_inference'] + if 'training' in args.mode: + summary.iloc[3:4, 6:7] = passrate_values['reference_amp_fp16_training'] + + if 'bfloat16' in args.precision: + if 'inference' in args.mode: + summary.iloc[4:5, 6:7] = passrate_values['reference_bfloat16_inference'] + if 'training' in args.mode: + summary.iloc[5:6, 6:7] = passrate_values['reference_bfloat16_training'] + if 'float16' in args.precision: + if 'inference' in args.mode: + summary.iloc[6:7, 6:7] = passrate_values['reference_float16_inference'] + if 'training' in args.mode: + summary.iloc[7:8, 6:7] = passrate_values['reference_float16_training'] + if 'float32' in args.precision: + if 'inference' in args.mode: + summary.iloc[8:9, 6:7] = passrate_values['reference_float32_inference'] + if 'training' in args.mode: + summary.iloc[9:10, 6:7] = passrate_values['reference_float32_training'] + + print(f"===================={scenario} Summary=============================") + print(summary) + + sf = StyleFrame(summary) + for i in range(1, 10): + sf.set_column_width(i, 22) + for j in range(1, 12): + sf.set_row_height(j, 30) + sf.to_excel(sheet_name=suite + '_' + scenario + '_Summary', excel_writer=excel) + + +def generate_report(excel, scenario_list, precision_list, mode_list, suite_list): + for sc in scenario_list: + for s in suite_list: + for p in precision_list: + for m in mode_list: + update_details(sc, p, m, s, excel) + update_summary(excel, sc, s) + + +def excel_postprocess(file, scenario, precison, mode, suite): + wb = file.book + # Summary + #ws = wb['Summary'] + #for i in range(2, 21, 2): + # ws.merge_cells(start_row=i, end_row=i + 1, start_column=1, end_column=1) + # details + for sc in scenario: + for s in suite: + for p in precison: + for m in mode: + wdt = wb[s + '_' + p + '_' + m[0:3] + '_' + sc[0:3]] + wdt.merge_cells(start_row=1, end_row=2, start_column=1, end_column=1) + wdt.merge_cells(start_row=1, end_row=1, start_column=3, end_column=7) + wdt.merge_cells(start_row=1, end_row=1, start_column=8, end_column=12) + wdt.merge_cells(start_row=1, end_row=1, start_column=13, end_column=16) + wb.save(file) + + +if __name__ == '__main__': + excel = StyleFrame.ExcelWriter('inductor_log/Inductor_E2E_Test_Report.xlsx') + generate_report(excel, args.scenario, args.precision, args.mode, args.suite) + excel_postprocess(excel, args.scenario, args.precision, args.mode, args.suite) diff --git a/.github/workflows/nightly_ondemand.yml b/.github/workflows/nightly_ondemand.yml index 4f56edb78..959915dcb 100644 --- a/.github/workflows/nightly_ondemand.yml +++ b/.github/workflows/nightly_ondemand.yml @@ -62,7 +62,7 @@ on: permissions: read-all concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref_name }}-${{ github.ref_type == 'branch' && github.sha }}-${{ github.event_name == 'workflow_dispatch' }}-${{ github.event_name == 'schedule' }}-${{ inputs.suite }}-${{ inputs.dt }}-${{ inputs.mode }}-${{ inputs.scenario }}-${{ inputs.triton }}-${{ inputs.model }} + group: ${{ github.workflow }}-${{ github.sha }}-${{ github.event_name }}-${{ inputs.pytorch }}-${{ inputs.keep_torch_xpu_ops }}-${{ inputs.ut }}-${{ inputs.triton }}-${{ inputs.suite }}-${{ inputs.dt }}-${{ inputs.mode }}-${{ inputs.scenario }}-${{ inputs.model }}-${{ inputs.python }} cancel-in-progress: true jobs: @@ -288,7 +288,7 @@ jobs: Tests-Failure-And-Report: if: ${{ ! cancelled() }} - runs-on: pvc_e2e + runs-on: [ self-hosted, Linux ] permissions: issues: write env: diff --git a/.github/workflows/nightly_ondemand_rolling.yml b/.github/workflows/nightly_ondemand_rolling.yml new file mode 100644 index 000000000..4016e7e36 --- /dev/null +++ b/.github/workflows/nightly_ondemand_rolling.yml @@ -0,0 +1,372 @@ +name: Nightly-OnDemand Tests Rolling + +on: + schedule: + # GMT+8 21:30 every workday + - cron: '30 13 * * 0-4' + # GMT+8 0:30 Saturday + - cron: '30 16 * * 5' + workflow_dispatch: + inputs: + pytorch: + required: false + type: string + default: 'main' + description: Pytorch branch/commit + keep_torch_xpu_ops: + required: false + type: string + default: 'false' + description: Keep torch-xpu-ops pin. `true` means use pined commit + ut: + required: false + type: string + default: 'torch_xpu' + description: UT scope. `op_example,op_extended,op_ut,torch_xpu`. Delimiter is comma + triton: + required: false + type: string + default: '' + description: Triton commit. Use pytorch pined commit by default + suite: + required: true + type: string + default: 'huggingface' + description: Dynamo benchmarks test suite. `huggingface,timm_models,torchbench`. Delimiter is comma + dt: + required: true + type: string + default: 'float32' + description: Data precision of the test. `float32,bfloat16,float16,amp_bf16,amp_fp16`. Delimiter is comma + mode: + required: true + type: string + default: 'inference' + description: Test mode. `inference,training`. Delimiter is comma + scenario: + required: true + type: string + default: 'accuracy' + description: Test scenario. `accuracy,performance`. Delimiter is comma + model: + required: false + type: string + default: '' + description: Model. Will only run this one mode if set + python: + required: false + type: string + default: '3.10' + description: Python version + +permissions: read-all + +concurrency: + group: ${{ github.workflow }}-${{ github.sha }}-${{ github.event_name }}-${{ inputs.pytorch }}-${{ inputs.keep_torch_xpu_ops }}-${{ inputs.ut }}-${{ inputs.triton }}-${{ inputs.suite }}-${{ inputs.dt }}-${{ inputs.mode }}-${{ inputs.scenario }}-${{ inputs.model }}-${{ inputs.python }} + cancel-in-progress: true + +jobs: + Linux-Nightly-Ondemand-UT-Tests-Rolling: + if: github.event_name == 'schedule' || ${{ inputs.ut_suite }} + uses: ./.github/workflows/_linux_ut.yml + env: + ZE_AFFINITY_MASK: 0 + with: + keep_torch_xpu_ops: ${{ github.event_name == 'schedule' && 'false' || inputs.keep_torch_xpu_ops }} + ut: ${{ github.event_name == 'schedule' && 'op_example,op_extended,op_ut,torch_xpu' || inputs.ut }} + pytorch: ${{ github.event_name == 'schedule' && 'main' || inputs.pytorch }} + python: ${{ github.event_name == 'schedule' && '3.10' || inputs.python }} + runner: pvc_rolling + + Linux-Weekly-UT-Tests-ABI-0-Rolling: + if: github.event_name == 'schedule' && github.event.schedule == '30 16 * * 5' + uses: ./.github/workflows/_linux_ut.yml + env: + ZE_AFFINITY_MASK: 0 + with: + abi: 0 + ut: op_example,op_extended,op_ut,torch_xpu + runner: pvc_rolling + + Linux-Nightly-Ondemand-E2E-Tests-Rolling: + runs-on: pvc_rolling + # Don't run on forked repos + if: github.repository_owner == 'intel' + timeout-minutes: 3600 + env: + pytorch: ${{ github.event_name == 'schedule' && 'main' || inputs.pytorch }} + keep_torch_xpu_ops: ${{ github.event_name == 'schedule' && 'false' || inputs.keep_torch_xpu_ops }} + ut: ${{ github.event_name == 'schedule' && 'op_example,op_extended,op_ut,torch_xpu' || inputs.ut }} + python: ${{ github.event_name == 'schedule' && '3.10' || inputs.python }} + outputs: + TORCH_BRANCH_ID: ${{ steps.pinned.outputs.TORCH_BRANCH_ID }} + TORCH_COMMIT_ID: ${{ steps.pinned.outputs.TORCH_COMMIT_ID }} + DRIVER_VERSION: ${{ steps.pinned.outputs.DRIVER_VERSION }} + BUNDLE_VERSION: ${{ steps.pinned.outputs.BUNDLE_VERSION }} + OS_PRETTY_NAME: ${{ steps.pinned.outputs.OS_PRETTY_NAME }} + GCC_VERSION: ${{ steps.pinned.outputs.GCC_VERSION }} + TORCHBENCH_COMMIT_ID: ${{ steps.pinned.outputs.TORCHBENCH_COMMIT_ID }} + TORCHVISION_COMMIT_ID: ${{ steps.pinned.outputs.TORCHVISION_COMMIT_ID }} + TORCHAUDIO_COMMIT_ID: ${{ steps.pinned.outputs.TORCHAUDIO_COMMIT_ID }} + TRANSFORMERS_VERSION: ${{ steps.pinned.outputs.TRANSFORMERS_VERSION }} + TIMM_COMMIT_ID: ${{ steps.pinned.outputs.TIMM_COMMIT_ID }} + TRITON_COMMIT_ID: ${{ steps.pinned.outputs.TRITON_COMMIT_ID }} + TIMEOUT_MODELS: ${{ steps.summary.outputs.TIMEOUT_MODELS }} + steps: + - name: Checkout torch-xpu-ops + uses: actions/checkout@v4 + - name: Prepare Conda ENV + run: | + which conda && conda clean -ay + conda remove --all -y -n e2e_ci || rm -rf $(dirname ${CONDA_EXE})/../envs/e2e_ci + conda create -n e2e_ci python=${{ env.python }} cmake ninja -y + source activate e2e_ci + pip install mkl-static mkl-include + pip install pandas scipy tqdm + - name: Prepare Stock Pytorch + run: | + pwd + cd ../ && rm -rf pytorch + source activate e2e_ci + git clone https://github.com/pytorch/pytorch pytorch + cd pytorch && git checkout ${{ env.pytorch }} + # apply PRs for stock pytorch + pip install requests + python ../torch-xpu-ops/.github/scripts/apply_torch_pr.py + git status && git show -s + git submodule sync && git submodule update --init --recursive + if [[ ${{ env.keep_torch_xpu_ops }} == 'true' ]]; then + echo "Don't replace torch-xpu-ops!" + else + rm -rf third_party/torch-xpu-ops && cp -r ../torch-xpu-ops third_party/ + # Workaround for torch-xpu-ops ci test + sed -i "s/checkout --quiet \${TORCH_XPU_OPS_COMMIT}/log -n 1/g" caffe2/CMakeLists.txt + fi + - name: Identify pinned versions + id: pinned + run: | + cd ../pytorch + if [ -z ${{ inputs.triton }} ]; then + echo "TRITON_COMMIT_ID=$(<.ci/docker/ci_commit_pins/triton-xpu.txt)" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}" + else + echo "TRITON_COMMIT_ID=${{ inputs.triton }}" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}" + fi + echo "TORCH_BRANCH_ID=$(git rev-parse --abbrev-ref HEAD)" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}" + echo "TORCH_COMMIT_ID=$(git rev-parse HEAD)" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}" + echo "TORCHBENCH_COMMIT_ID=$(> "${GITHUB_ENV}" + echo "TORCHVISION_COMMIT_ID=$(<.github/ci_commit_pins/vision.txt)" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}" + echo "TORCHAUDIO_COMMIT_ID=$(<.github/ci_commit_pins/audio.txt)" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}" + echo "TRANSFORMERS_VERSION=$(<.ci/docker/ci_commit_pins/huggingface.txt)" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}" + echo "TIMM_COMMIT_ID=$(<.ci/docker/ci_commit_pins/timm.txt)" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}" + echo "MODEL_ONLY_NAME=${{ inputs.model }}" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}" + source /opt/intel/oneapi/pytorch-gpu-dev-0.5/oneapi-vars.sh + echo "DRIVER_VERSION=$(dkms status 2>&1 |grep 'intel-i915-dkms' |sed 's/.*\///;s/,.*//')" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}" + echo "BUNDLE_VERSION=$(dpcpp --version 2>&1 |grep 'DPC++/C++' |sed 's/.*(//;s/).*//')" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}" + . /etc/os-release + echo "OS_PRETTY_NAME=${PRETTY_NAME}" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}" + echo "GCC_VERSION=$(gcc -dumpversion)" |tee -a "${GITHUB_OUTPUT}" >> "${GITHUB_ENV}" + echo ${GITHUB_ENV} + - name: Triton Installation + run: | + source activate e2e_ci + cd ../pytorch + TRITON_REPO="https://github.com/intel/intel-xpu-backend-for-triton" + echo ${TRITON_REPO}@${TRITON_COMMIT_ID} + pip install --force-reinstall "git+${TRITON_REPO}@${TRITON_COMMIT_ID}#subdirectory=python" + - name: Build Pytorch XPU + run: | + source activate e2e_ci + source .github/scripts/env.sh + cd ../pytorch + pip install -r requirements.txt + export CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH}:${CONDA_PREFIX:-"$(dirname $(which conda))/../"} + python setup.py bdist_wheel + pip install --force-reinstall dist/*.whl + - name: Show GITHUB_ENV + run: | + echo "$GITHUB_ENV" + rm -rf ../pytorch/inductor_log + rm -rf /tmp/torchinductor_* + + # Nihglty launch + - name: Nightly Huggingface FP32/BF16/FP16 Inference & Training Accuracy Test + if: github.event_name == 'schedule' && github.event.schedule == '30 13 * * 0-4' + uses: ./.github/actions/inductor-xpu-e2e-test + with: + suite: huggingface + env_prepare: true + dt: float32,bfloat16,float16 + mode: inference,training + scenario: accuracy + hf_token: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} + - name: Nightly Torchbench BF16 Training Accuracy Test + if: github.event_name == 'schedule' && github.event.schedule == '30 13 * * 0-4' + uses: ./.github/actions/inductor-xpu-e2e-test + with: + suite: torchbench + dt: bfloat16 + mode: training + scenario: accuracy + env_prepare: true + hf_token: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} + - name: Nightly Timm_models FP16 Training Accuracy Test + if: github.event_name == 'schedule' && github.event.schedule == '30 13 * * 0-4' + uses: ./.github/actions/inductor-xpu-e2e-test + with: + suite: timm_models + dt: float16 + mode: training + scenario: accuracy + env_prepare: true + hf_token: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} + # Weekly launch + - name: Weekly Huggingface Full Test + if: github.event_name == 'schedule' && github.event.schedule == '30 16 * * 5' + uses: ./.github/actions/inductor-xpu-e2e-test + with: + suite: huggingface + env_prepare: true + dt: float32,bfloat16,float16,amp_bf16,amp_fp16 + mode: inference,training + scenario: accuracy,performance + hf_token: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} + - name: Weekly Torchbench Full Test + if: github.event_name == 'schedule' && github.event.schedule == '30 16 * * 5' + uses: ./.github/actions/inductor-xpu-e2e-test + with: + suite: torchbench + env_prepare: true + dt: float32,bfloat16,float16,amp_bf16,amp_fp16 + mode: inference,training + scenario: accuracy,performance + hf_token: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} + - name: Weekly Timm_models Full Test + if: github.event_name == 'schedule' && github.event.schedule == '30 16 * * 5' + uses: ./.github/actions/inductor-xpu-e2e-test + with: + suite: timm_models + env_prepare: true + dt: float32,bfloat16,float16,amp_bf16,amp_fp16 + mode: inference,training + scenario: accuracy,performance + hf_token: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} + # On-demand launch + - name: OnDemand Test (${{ inputs.suite }} ${{ inputs.dt }} ${{ inputs.mode }} ${{ inputs.scenario }}) + if: github.event_name != 'schedule' + uses: ./.github/actions/inductor-xpu-e2e-test + with: + suite: ${{ inputs.suite }} + env_prepare: true + dt: ${{ inputs.dt }} + mode: ${{ inputs.mode }} + scenario: ${{ inputs.scenario }} + hf_token: ${{ secrets.HUGGING_FACE_HUB_TOKEN }} + - name: Summarize archieve files + id: summary + if: ${{ ! cancelled() }} + run: | + rm -rf ${{ github.workspace }}/upload_files + cp -r ${{ github.workspace }}/../pytorch/inductor_log ${{ github.workspace }}/upload_files + mkdir -p ${{ github.workspace }}/../../_backup/ && cd ${{ github.workspace }}/../../_backup/ + find . -type f -name "*.tgz" -mtime +3 -delete # delete files older than 3 days + tar zcf xpu-inductor-${GITHUB_RUN_ID}.tgz -C ${{ github.workspace }}/upload_files/ . # backup logs + failed_models=$(grep "Real failed models: *[1-9]" ${{ github.workspace }}/upload_files/summary_accuracy.log |wc -l || true) + timeout_models=$(grep "timeout models: *[1-9]" ${{ github.workspace }}/upload_files/summary_accuracy.log |wc -l || true) + if [ ${timeout_models} -ne 0 ];then + TIMEOUT_MODELS="$( + grep -E "timeout models: [1-9]|Summary for" ${{ github.workspace }}/upload_files/summary_accuracy.log |grep "timeout" -B 1 + )" + echo "${TIMEOUT_MODELS}" |sed 's/Summary/\\nSummary/g;s/Timeout/\\nTimeout/g' |tee -a "${GITHUB_OUTPUT}" + grep -E "timeout models: [1-9]|Summary for" ${{ github.workspace }}/upload_files/summary_accuracy.log |grep "timeout" -B 1 + fi + if [ ${failed_models} -ne 0 ];then + grep -E "Real failed models: [1-9]|Summary for" ${{ github.workspace }}/upload_files/summary_accuracy.log |grep "failed" -B 1 + exit 1 + fi + - name: Upload Inductor XPU E2E Data + if: ${{ ! cancelled() }} + uses: actions/upload-artifact@v4 + with: + name: Inductor-XPU-E2E-Data-${{ github.event.pull_request.number || github.sha }} + path: ${{ github.workspace }}/upload_files + + Tests-Failure-And-Report: + if: ${{ ! cancelled() }} + runs-on: [ self-hosted, Linux ] + permissions: + issues: write + env: + GH_TOKEN: ${{ github.token }} + python: ${{ github.event_name == 'schedule' && '3.10' || inputs.python }} + needs: Linux-Nightly-Ondemand-E2E-Tests + steps: + - name: Report github issue for XPU OPS nightly + if: github.repository_owner == 'intel' + run: | + set -xe + # Test env + build_url="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" + repo="${{ github.repository }}" + TORCH_BRANCH_ID="${{ needs.Linux-Nightly-Ondemand-E2E-Tests.outputs.TORCH_BRANCH_ID }}" + TORCH_COMMIT_ID="${{ needs.Linux-Nightly-Ondemand-E2E-Tests.outputs.TORCH_COMMIT_ID }}" + DRIVER_VERSION="${{ needs.Linux-Nightly-Ondemand-E2E-Tests.outputs.DRIVER_VERSION }}" + BUNDLE_VERSION="${{ needs.Linux-Nightly-Ondemand-E2E-Tests.outputs.BUNDLE_VERSION }}" + OS_PRETTY_NAME="${{ needs.Linux-Nightly-Ondemand-E2E-Tests.outputs.OS_PRETTY_NAME }}" + GCC_VERSION="${{ needs.Linux-Nightly-Ondemand-E2E-Tests.outputs.GCC_VERSION }}" + TORCHBENCH_COMMIT_ID="${{ needs.Linux-Nightly-Ondemand-E2E-Tests.outputs.TORCHBENCH_COMMIT_ID }}" + TORCHVISION_COMMIT_ID="${{ needs.Linux-Nightly-Ondemand-E2E-Tests.outputs.TORCHVISION_COMMIT_ID }}" + TORCHAUDIO_COMMIT_ID="${{ needs.Linux-Nightly-Ondemand-E2E-Tests.outputs.TORCHAUDIO_COMMIT_ID }}" + TRANSFORMERS_VERSION="${{ needs.Linux-Nightly-Ondemand-E2E-Tests.outputs.TRANSFORMERS_VERSION }}" + TIMM_COMMIT_ID="${{ needs.Linux-Nightly-Ondemand-E2E-Tests.outputs.TIMM_COMMIT_ID }}" + TRITON_COMMIT_ID="${{ needs.Linux-Nightly-Ondemand-E2E-Tests.outputs.TRITON_COMMIT_ID }}" + TIMEOUT_MODELS="${{ needs.Linux-Nightly-Ondemand-E2E-Tests.outputs.TIMEOUT_MODELS }}" + # Test status + if [ "${{ needs.Linux-Nightly-Ondemand-E2E-Tests.result }}" == "success" ];then + test_status=Success + elif [ "${{ needs.Linux-Nightly-Ondemand-E2E-Tests.result }}" == "failure" ];then + test_status=Failure + cc_comment="CC ${{ secrets.NIGHTLY_EMAIL_LIST }}" + else + test_status=None + exit 0 + fi + # Test Type + if [ "${GITHUB_EVENT_NAME}" == "workflow_dispatch" ];then + test_type="On-demand" + test_issue_id=426 + cc_comment="CC @${GITHUB_TRIGGERING_ACTOR}" + elif [ "${{ github.event.schedule }}" == "30 16 * * 5" ];then + test_type="Weekly" + test_issue_id=432 + else + test_type="Nightly" + test_issue_id=432 + fi + # Test report + echo -e "**${test_status}** $test_type Rolling Test on $(date +'%F'), See: $build_url\n" > ${{ github.workspace }}/report.txt + printf "Torch-xpu-ops | PyTorch | Triton\n--- | --- | ---\n${GITHUB_WORKFLOW_SHA:0:7} on ${GITHUB_REF_NAME} | " >> ${{ github.workspace }}/report.txt + printf "[${TORCH_COMMIT_ID:0:7}](https://github.com/pytorch/pytorch/commit/${TORCH_COMMIT_ID:0:7}) on $TORCH_BRANCH_ID | " >> ${{ github.workspace }}/report.txt + echo -e "[${TRITON_COMMIT_ID:0:7}](https://github.com/intel/intel-xpu-backend-for-triton/commit/${TRITON_COMMIT_ID:0:7}) \n" >> ${{ github.workspace }}/report.txt + printf "Transformers | Timm | Torchbench | Torchvision | Torchaudio\n--- | --- | --- | --- | ---\n" >> ${{ github.workspace }}/report.txt + printf "[${TRANSFORMERS_VERSION:0:7}](https://github.com/huggingface/transformers/commit/${TRANSFORMERS_VERSION:0:7}) | " >> ${{ github.workspace }}/report.txt + printf "[${TIMM_COMMIT_ID:0:7}](https://github.com/huggingface/pytorch-image-models/commit/${TIMM_COMMIT_ID:0:7}) | " >> ${{ github.workspace }}/report.txt + printf "[${TORCHBENCH_COMMIT_ID:0:7}](https://github.com/pytorch/benchmark/commit/${TORCHBENCH_COMMIT_ID:0:7}) | " >> ${{ github.workspace }}/report.txt + printf "[${TORCHVISION_COMMIT_ID:0:7}](https://github.com/pytorch/vision/commit/${TORCHVISION_COMMIT_ID:0:7}) | " >> ${{ github.workspace }}/report.txt + echo -e "[${TORCHAUDIO_COMMIT_ID:0:7}](https://github.com/pytorch/audio/commit/${TORCHAUDIO_COMMIT_ID:0:7}) \n" >> ${{ github.workspace }}/report.txt + printf "Device | OS | GCC | Python | Driver(DKMS) | Bundle(DPCPP)\n--- | --- | --- | --- | --- | ---\n" >> ${{ github.workspace }}/report.txt + echo -e "$RUNNER_NAME | $OS_PRETTY_NAME | $GCC_VERSION | ${{ env.python }} | rolling-$DRIVER_VERSION| $BUNDLE_VERSION \n" >> ${{ github.workspace }}/report.txt + if [ "${GITHUB_EVENT_NAME}" == "workflow_dispatch" ];then + test_scope="${{ inputs.suite }}/${{ inputs.dt }}/${{ inputs.mode }}/${{ inputs.scenario }}" + if [ "${{ inputs.triton }}" != "" ];then + test_scope+="; triton=${{ inputs.triton }}" + fi + if [ "${{ inputs.model }}" != "" ];then + test_scope+="; model=${{ inputs.model }}" + fi + echo -e "Inputs | $test_scope\n--- | --- \n" >> ${{ github.workspace }}/report.txt + fi + echo "$TIMEOUT_MODELS" >> ${{ github.workspace }}/report.txt + echo "$cc_comment" >> ${{ github.workspace }}/report.txt + # Report + report_txt=$(cat ${{ github.workspace }}/report.txt) + gh --repo $repo issue comment $test_issue_id --body "$report_txt" diff --git a/test/xpu/run_test_with_skip.py b/test/xpu/run_test_with_skip.py index 2267d940c..3f2e371d4 100644 --- a/test/xpu/run_test_with_skip.py +++ b/test/xpu/run_test_with_skip.py @@ -1172,6 +1172,8 @@ def launch_test(test_case, skip_list=None, exe_list=None): "test_to_nn_TransformerEncoder_eval_mode_swap_True_set_grad_True_xpu_float32", "test_to_nn_TransformerEncoder_train_mode_swap_True_set_grad_True_xpu_float32", "test_to_nn_Transformer_swap_True_set_grad_True_xpu_float32", + #issue 746, adjust tolerence + "test_non_contiguous_tensors_nn_Conv3d_xpu_float32", ) res += launch_test("test_modules_xpu.py", skip_list) @@ -2905,9 +2907,13 @@ def launch_test(test_case, skip_list=None, exe_list=None): res += launch_test("nn/test_convolution_xpu.py", skip_list) # test_dynamic_shapes - - -res += launch_test("test_dynamic_shapes_xpu.py") +skip_list = ( + # issue 746, new ut failures introduced by new pytorch + "test_method_fn_add_first_type_int_second_type_float", + "test_method_fn_mul_first_type_int_second_type_float", + "test_method_fn_sub_first_type_int_second_type_float", +) +res += launch_test("test_dynamic_shapes_xpu.py", skip_list) # test_load_state_dict