From 2ed8d60ad06f09fbac972eeb13841edf68951760 Mon Sep 17 00:00:00 2001 From: mengfeil Date: Thu, 26 Dec 2024 15:28:17 +0800 Subject: [PATCH 1/2] update weekly accuracy referenche --- .../inductor_timm_models_inference.csv | 2 +- .../inductor_timm_models_training.csv | 8 ++++---- .../inductor_torchbench_inference.csv | 12 +++++------ .../inductor_torchbench_training.csv | 20 +++++++++---------- 4 files changed, 21 insertions(+), 21 deletions(-) diff --git a/.github/ci_expected_accuracy/inductor_timm_models_inference.csv b/.github/ci_expected_accuracy/inductor_timm_models_inference.csv index ffaa26f62..66ed332bb 100644 --- a/.github/ci_expected_accuracy/inductor_timm_models_inference.csv +++ b/.github/ci_expected_accuracy/inductor_timm_models_inference.csv @@ -59,4 +59,4 @@ twins_pcpvt_base,pass,pass,pass,pass,pass visformer_small,pass,pass,pass,pass,pass vit_base_patch16_224,pass,pass,pass,pass,pass volo_d1_224,pass,pass,pass,pass,pass -xcit_large_24_p8_224,pass,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run +xcit_large_24_p8_224,pass,pass,pass,pass,pass diff --git a/.github/ci_expected_accuracy/inductor_timm_models_training.csv b/.github/ci_expected_accuracy/inductor_timm_models_training.csv index c605c9bdf..6bbb653f0 100644 --- a/.github/ci_expected_accuracy/inductor_timm_models_training.csv +++ b/.github/ci_expected_accuracy/inductor_timm_models_training.csv @@ -14,7 +14,7 @@ dla102,pass,pass,pass,pass,pass dm_nfnet_f0,pass,pass,pass,pass,pass dpn107,pass,pass,pass,pass,pass eca_botnext26ts_256,pass,pass,pass,pass,pass -eca_halonext26ts,pass,pass,fail_to_run,pass,pass +eca_halonext26ts,pass,pass,pass,pass,pass ese_vovnet19b_dw,pass,pass,pass,pass,pass fbnetc_100,pass,pass,pass,pass,pass fbnetv3_b,pass,pass,pass,pass,pass @@ -25,8 +25,8 @@ gmixer_24_224,pass,pass,pass,pass,pass gmlp_s16_224,pass,pass,pass,pass,pass hrnet_w18,pass,pass,pass,pass,pass inception_v3,pass,pass,pass,pass,pass -jx_nest_base,pass,pass,fail_accuracy,pass,pass -lcnet_050,pass,pass,fail_accuracy,pass,pass +jx_nest_base,pass,pass,pass,pass,pass +lcnet_050,pass,pass,pass,pass,pass levit_128,pass,pass,pass,pass,pass mixer_b16_224,pass,pass,pass,pass,pass mixnet_l,pass,pass,pass,pass,pass @@ -59,4 +59,4 @@ twins_pcpvt_base,pass,pass,pass,pass,pass visformer_small,pass,pass,pass,pass,pass vit_base_patch16_224,pass,pass,pass,pass,pass volo_d1_224,pass,pass,pass,pass,pass -xcit_large_24_p8_224,pass_due_to_skip,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run +xcit_large_24_p8_224,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip diff --git a/.github/ci_expected_accuracy/inductor_torchbench_inference.csv b/.github/ci_expected_accuracy/inductor_torchbench_inference.csv index 4825aa41f..792bfac0b 100644 --- a/.github/ci_expected_accuracy/inductor_torchbench_inference.csv +++ b/.github/ci_expected_accuracy/inductor_torchbench_inference.csv @@ -25,9 +25,9 @@ detectron2_maskrcnn,fail_to_run,eager_fail_to_run,fail_to_run,eager_fail_to_run, detectron2_maskrcnn_r_101_c4,fail_accuracy,eager_fail_to_run,fail_accuracy,eager_fail_to_run,fail_accuracy detectron2_maskrcnn_r_101_fpn,fail_accuracy,eager_fail_to_run,fail_accuracy,eager_fail_to_run,fail_accuracy detectron2_maskrcnn_r_50_c4,pass,eager_fail_to_run,fail_accuracy,eager_fail_to_run,fail_accuracy -detectron2_maskrcnn_r_50_fpn,fail_accuracy,eager_fail_to_run,fail_accuracy,eager_fail_to_run,fail_accuracy +detectron2_maskrcnn_r_50_fpn,pass,eager_fail_to_run,fail_accuracy,eager_fail_to_run,fail_accuracy dlrm,pass,pass,pass,pass,pass -doctr_det_predictor,pass,pass,pass,eager_fail_to_run,pass +doctr_det_predictor,pass,pass,pass,pass,pass doctr_reco_predictor,pass,pass,pass,pass,pass drq,pass,pass,pass,pass,pass fastNLP_Bert,pass,pass,pass,pass,pass @@ -68,7 +68,7 @@ nvidia_deeprecommender,pass,pass,pass,pass,pass opacus_cifar10,pass,pass,pass,pass,pass phlippe_densenet,pass,pass,pass,pass,pass phlippe_resnet,pass,pass,pass,pass,pass -pyhpc_equation_of_state,pass,pass,fail_accuracy,pass,pass +pyhpc_equation_of_state,pass,pass,pass,pass,pass pyhpc_isoneutral_mixing,pass,pass,pass,pass,pass pyhpc_turbulent_kinetic_energy,pass,pass,pass,pass,pass pytorch_CycleGAN_and_pix2pix,pass,pass,pass,pass,pass @@ -86,7 +86,7 @@ simple_gpt,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_t simple_gpt_tp_manual,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load soft_actor_critic,pass,pass,pass,pass,pass speech_transformer,pass,pass,pass,pass,pass -squeezenet1_1,pass,fail_accuracy,fail_accuracy,pass,pass +squeezenet1_1,pass,fail_accuracy,pass,pass,pass stable_diffusion_text_encoder,pass,pass,pass,pass,pass stable_diffusion_unet,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip tacotron2,pass,pass,pass,fail_to_run,fail_to_run @@ -98,8 +98,8 @@ timm_resnest,pass,pass,pass,pass,pass timm_vision_transformer,pass,pass,pass,pass,pass timm_vision_transformer_large,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip timm_vovnet,pass,pass,pass,pass,pass -torch_multimodal_clip,pass,pass,pass,eager_fail_to_run,eager_fail_to_run -tts_angular,pass,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run +torch_multimodal_clip,pass,pass,pass,pass,pass +tts_angular,pass,pass,pass,pass,pass vgg16,pass,pass,pass,pass,pass vision_maskrcnn,pass,pass,pass,eager_fail_to_run,eager_fail_to_run yolov3,pass,pass,pass,pass,pass diff --git a/.github/ci_expected_accuracy/inductor_torchbench_training.csv b/.github/ci_expected_accuracy/inductor_torchbench_training.csv index dc766eac0..4c1bc6aba 100644 --- a/.github/ci_expected_accuracy/inductor_torchbench_training.csv +++ b/.github/ci_expected_accuracy/inductor_torchbench_training.csv @@ -1,11 +1,11 @@ name,float32,bfloat16,float16,amp_bf16,amp_fp16 -torchrec_dlrm,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run +torchrec_dlrm,pass,eager_fail_to_run,eager_fail_to_run,pass,pass BERT_pytorch,pass,pass,pass,pass,pass Background_Matting,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip DALLE2_pytorch,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run LearningToPaint,pass,pass,pass,pass,pass -Super_SloMo,eager_two_runs_differ,pass,pass,eager_two_runs_differ,pass -alexnet,eager_two_runs_differ,pass,pass,pass,pass +Super_SloMo,eager_two_runs_differ,pass,pass,pass,pass +alexnet,pass,pass,pass,pass,pass basic_gnn_edgecnn,pass,pass,pass,pass,pass basic_gnn_gcn,pass,pass,pass,pass,pass basic_gnn_gin,pass,pass,pass,pass,pass @@ -26,13 +26,13 @@ detectron2_maskrcnn_r_101_c4,eager_fail_to_run,eager_fail_to_run,eager_fail_to_r detectron2_maskrcnn_r_101_fpn,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run detectron2_maskrcnn_r_50_c4,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run detectron2_maskrcnn_r_50_fpn,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run -dlrm,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run +dlrm,pass,pass,pass,pass,pass doctr_det_predictor,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run doctr_reco_predictor,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run drq,pass,pass,pass,pass,pass fastNLP_Bert,pass,pass,pass,pass,pass functorch_dp_cifar10,fail_accuracy,fail_accuracy,fail_accuracy,pass,pass -functorch_maml_omniglot,pass,pass,pass,fail_accuracy,pass +functorch_maml_omniglot,pass,pass,pass,eager_fail_to_run,pass hf_Albert,pass,pass,pass,pass,pass hf_Bart,pass,pass,pass,pass,pass hf_Bert,pass,pass,pass,pass,pass @@ -59,7 +59,7 @@ maml_omniglot,pass,pass,pass,pass,pass microbench_unbacked_tolist_sum,pass,pass,pass,pass,pass mnasnet1_0,pass,pass,pass,pass,pass mobilenet_v2,pass,pass,pass,pass,pass -mobilenet_v2_quantized_qat,fail_accuracy,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run +mobilenet_v2_quantized_qat,pass,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run mobilenet_v3_large,pass,pass,pass,pass,pass moco,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load moondream,pass,pass,pass,pass,pass @@ -77,7 +77,7 @@ pytorch_unet,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip resnet152,pass,pass,pass,pass,pass resnet18,pass,pass,pass,pass,pass resnet50,pass,pass,pass,pass,pass -resnet50_quantized_qat,eager_two_runs_differ,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run +resnet50_quantized_qat,pass,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run resnext50_32x4d,pass,pass,pass,pass,pass sam,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run sam_fast,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run @@ -86,7 +86,7 @@ simple_gpt,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_t simple_gpt_tp_manual,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load soft_actor_critic,pass,pass,pass,pass,pass speech_transformer,pass,pass,pass,pass,pass -squeezenet1_1,pass,fail_accuracy,fail_accuracy,fail_accuracy,pass +squeezenet1_1,pass,pass,pass,pass,pass stable_diffusion_text_encoder,pass,pass,pass,pass,pass stable_diffusion_unet,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip tacotron2,fail_to_run,fail_to_run,fail_to_run,fail_to_run,fail_to_run @@ -98,8 +98,8 @@ timm_resnest,pass,pass,pass,pass,pass timm_vision_transformer,pass,pass,pass,pass,pass timm_vision_transformer_large,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip timm_vovnet,pass,pass,pass,pass,pass -torch_multimodal_clip,pass,pass,pass,eager_fail_to_run,eager_fail_to_run -tts_angular,pass,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run +torch_multimodal_clip,pass,pass,pass,pass,pass +tts_angular,pass,pass,pass,pass,pass vgg16,pass,pass,pass,pass,pass vision_maskrcnn,pass,pass,pass,eager_fail_to_run,eager_fail_to_run yolov3,pass,pass,pass,pass,pass From a905468d9cdb226af1ad1967698d174db5180e44 Mon Sep 17 00:00:00 2001 From: mengfeil Date: Thu, 26 Dec 2024 15:56:45 +0800 Subject: [PATCH 2/2] Update failed models with known issues --- .../inductor_huggingface_training.csv | 2 +- .../inductor_timm_models_inference.csv | 40 ++++++------ .../inductor_timm_models_training.csv | 62 +++++++++---------- .../inductor_torchbench_inference.csv | 38 ++++++------ .../inductor_torchbench_training.csv | 34 +++++----- 5 files changed, 88 insertions(+), 88 deletions(-) diff --git a/.github/ci_expected_accuracy/inductor_huggingface_training.csv b/.github/ci_expected_accuracy/inductor_huggingface_training.csv index a75d3d225..fa73dc6fc 100644 --- a/.github/ci_expected_accuracy/inductor_huggingface_training.csv +++ b/.github/ci_expected_accuracy/inductor_huggingface_training.csv @@ -13,7 +13,7 @@ CamemBert,pass,pass,pass,pass,pass DebertaForMaskedLM,pass,pass,pass,pass,pass DebertaForQuestionAnswering,pass,pass,pass,pass,pass DebertaV2ForMaskedLM,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip -DebertaV2ForQuestionAnswering,pass,pass,pass,pass,pass +DebertaV2ForQuestionAnswering,fail_accuracy,fail_accuracy,fail_accuracy,pass,pass DistilBertForMaskedLM,pass,pass,pass,pass,pass DistilBertForQuestionAnswering,pass,pass,pass,pass,pass DistillGPT2,pass,pass,pass,pass,pass diff --git a/.github/ci_expected_accuracy/inductor_timm_models_inference.csv b/.github/ci_expected_accuracy/inductor_timm_models_inference.csv index 66ed332bb..42e454c6c 100644 --- a/.github/ci_expected_accuracy/inductor_timm_models_inference.csv +++ b/.github/ci_expected_accuracy/inductor_timm_models_inference.csv @@ -1,5 +1,5 @@ name,float32,bfloat16,float16,amp_bf16,amp_fp16 -adv_inception_v3,pass,pass,pass,pass,pass +adv_inception_v3,pass,pass,pass,fail_accuracy,pass beit_base_patch16_224,pass,pass,pass,pass,pass botnet26t_256,pass,pass,pass,pass,pass cait_m36_384,pass,pass,pass,pass,pass @@ -10,48 +10,48 @@ convnext_base,pass,pass,pass,pass,pass crossvit_9_240,pass,pass,pass,pass,pass cspdarknet53,pass,pass,pass,pass,pass deit_base_distilled_patch16_224,pass,pass,pass,pass,pass -dla102,pass,pass,pass,pass,pass -dm_nfnet_f0,pass,pass,pass,pass,pass -dpn107,pass,pass,pass,pass,pass +dla102,pass,fail_accuracy,pass,fail_accuracy,pass +dm_nfnet_f0,pass,fail_accuracy,pass,pass,pass +dpn107,pass,fail_accuracy,pass,fail_accuracy,pass eca_botnext26ts_256,pass,pass,pass,pass,pass eca_halonext26ts,pass,pass,pass,pass,pass ese_vovnet19b_dw,pass,pass,pass,pass,pass -fbnetc_100,pass,pass,pass,pass,pass +fbnetc_100,pass,fail_accuracy,pass,fail_accuracy,pass fbnetv3_b,pass,pass,pass,pass,pass gernet_l,pass,pass,pass,pass,pass -ghostnet_100,pass,pass,pass,pass,pass -gluon_inception_v3,pass,pass,pass,pass,pass +ghostnet_100,pass,fail_accuracy,pass,fail_accuracy,pass +gluon_inception_v3,pass,pass,pass,fail_accuracy,pass gmixer_24_224,pass,pass,pass,pass,pass gmlp_s16_224,pass,pass,pass,pass,pass -hrnet_w18,pass,pass,pass,pass,pass -inception_v3,pass,pass,pass,pass,pass +hrnet_w18,pass,fail_accuracy,pass,fail_accuracy,pass +inception_v3,pass,fail_accuracy,pass,fail_accuracy,pass jx_nest_base,pass,pass,pass,pass,pass -lcnet_050,pass,pass,pass,pass,pass +lcnet_050,pass,fail_accuracy,pass,fail_accuracy,pass levit_128,pass,pass,pass,pass,pass mixer_b16_224,pass,pass,pass,pass,pass mixnet_l,pass,pass,pass,pass,pass -mnasnet_100,pass,pass,pass,pass,pass -mobilenetv2_100,pass,pass,pass,pass,pass +mnasnet_100,pass,fail_accuracy,pass,fail_accuracy,pass +mobilenetv2_100,pass,pass,pass,fail_accuracy,pass mobilenetv3_large_100,pass,pass,pass,pass,pass mobilevit_s,pass,pass,pass,pass,pass nfnet_l0,pass,pass,pass,pass,pass pit_b_224,pass,pass,pass,pass,pass pnasnet5large,pass,pass,pass,pass,pass poolformer_m36,pass,pass,pass,pass,pass -regnety_002,pass,pass,pass,pass,pass +regnety_002,pass,fail_accuracy,pass,fail_accuracy,pass repvgg_a2,pass,pass,pass,pass,pass -res2net101_26w_4s,pass,pass,pass,pass,pass -res2net50_14w_8s,pass,pass,pass,pass,pass -res2next50,pass,pass,pass,pass,pass +res2net101_26w_4s,pass,pass,pass,fail_accuracy,pass +res2net50_14w_8s,pass,pass,pass,fail_accuracy,pass +res2next50,pass,pass,pass,fail_accuracy,pass resmlp_12_224,pass,pass,pass,pass,pass resnest101e,pass,pass,pass,pass,pass -rexnet_100,pass,pass,pass,pass,pass +rexnet_100,pass,pass,pass,fail_accuracy,pass sebotnet33ts_256,pass,pass,pass,pass,pass selecsls42b,pass,pass,pass,pass,pass -spnasnet_100,pass,pass,pass,pass,pass +spnasnet_100,pass,fail_accuracy,pass,fail_accuracy,pass swin_base_patch4_window7_224,pass,pass,pass,pass,pass -swsl_resnext101_32x16d,pass,pass,pass,pass,pass -tf_efficientnet_b0,pass,pass,pass,pass,pass +swsl_resnext101_32x16d,pass,fail_accuracy,pass,fail_accuracy,pass +tf_efficientnet_b0,pass,fail_accuracy,pass,fail_accuracy,pass tf_mixnet_l,pass,pass,pass,pass,pass tinynet_a,pass,pass,pass,pass,pass tnt_s_patch16_224,pass,pass,pass,pass,pass diff --git a/.github/ci_expected_accuracy/inductor_timm_models_training.csv b/.github/ci_expected_accuracy/inductor_timm_models_training.csv index 6bbb653f0..6da0ad06f 100644 --- a/.github/ci_expected_accuracy/inductor_timm_models_training.csv +++ b/.github/ci_expected_accuracy/inductor_timm_models_training.csv @@ -1,62 +1,62 @@ name,float32,bfloat16,float16,amp_bf16,amp_fp16 -adv_inception_v3,pass,pass,pass,pass,pass +adv_inception_v3,pass,fail_accuracy,pass,fail_accuracy,pass beit_base_patch16_224,pass,pass,pass,pass,pass botnet26t_256,pass,pass,pass,pass,pass cait_m36_384,pass,pass,pass,pass,pass coat_lite_mini,pass,pass,pass,pass,pass convit_base,pass,pass,pass,pass,pass -convmixer_768_32,pass,pass,pass,pass,pass -convnext_base,pass,pass,fail_accuracy,pass,pass +convmixer_768_32,pass,fail_accuracy,pass,fail_accuracy,pass +convnext_base,pass,fail_accuracy,fail_accuracy,pass,pass crossvit_9_240,pass,pass,pass,pass,pass cspdarknet53,pass,pass,pass,pass,pass deit_base_distilled_patch16_224,pass,pass,pass,pass,pass -dla102,pass,pass,pass,pass,pass -dm_nfnet_f0,pass,pass,pass,pass,pass +dla102,pass,fail_accuracy,pass,fail_accuracy,pass +dm_nfnet_f0,pass,fail_accuracy,pass,pass,pass dpn107,pass,pass,pass,pass,pass eca_botnext26ts_256,pass,pass,pass,pass,pass -eca_halonext26ts,pass,pass,pass,pass,pass -ese_vovnet19b_dw,pass,pass,pass,pass,pass -fbnetc_100,pass,pass,pass,pass,pass -fbnetv3_b,pass,pass,pass,pass,pass +eca_halonext26ts,pass,pass,pass,fail_accuracy,pass +ese_vovnet19b_dw,pass,fail_accuracy,pass,fail_accuracy,pass +fbnetc_100,pass,fail_accuracy,pass,fail_accuracy,pass +fbnetv3_b,pass,fail_accuracy,pass,fail_accuracy,pass gernet_l,pass,pass,pass,pass,pass -ghostnet_100,pass,pass,pass,pass,pass -gluon_inception_v3,pass,pass,pass,pass,pass +ghostnet_100,pass,fail_accuracy,pass,fail_accuracy,pass +gluon_inception_v3,pass,fail_accuracy,pass,fail_accuracy,pass gmixer_24_224,pass,pass,pass,pass,pass gmlp_s16_224,pass,pass,pass,pass,pass -hrnet_w18,pass,pass,pass,pass,pass -inception_v3,pass,pass,pass,pass,pass +hrnet_w18,pass,fail_accuracy,pass,fail_accuracy,pass +inception_v3,pass,fail_accuracy,pass,fail_accuracy,pass jx_nest_base,pass,pass,pass,pass,pass -lcnet_050,pass,pass,pass,pass,pass +lcnet_050,pass,fail_accuracy,pass,fail_accuracy,pass levit_128,pass,pass,pass,pass,pass mixer_b16_224,pass,pass,pass,pass,pass -mixnet_l,pass,pass,pass,pass,pass -mnasnet_100,pass,pass,pass,pass,pass -mobilenetv2_100,pass,pass,pass,pass,pass -mobilenetv3_large_100,pass,pass,pass,pass,pass +mixnet_l,pass,fail_accuracy,pass,fail_accuracy,pass +mnasnet_100,pass,fail_accuracy,pass,fail_accuracy,pass +mobilenetv2_100,pass,fail_accuracy,pass,fail_accuracy,pass +mobilenetv3_large_100,pass,fail_accuracy,pass,fail_accuracy,pass mobilevit_s,pass,pass,pass,pass,pass nfnet_l0,pass,pass,pass,pass,pass pit_b_224,pass,pass,pass,pass,pass -pnasnet5large,pass,pass,pass,pass,pass +pnasnet5large,pass,fail_accuracy,pass,fail_accuracy,pass poolformer_m36,pass,pass,pass,pass,pass -regnety_002,pass,pass,pass,pass,pass -repvgg_a2,pass,pass,pass,pass,pass -res2net101_26w_4s,pass,pass,pass,pass,pass -res2net50_14w_8s,pass,pass,pass,pass,pass -res2next50,pass,pass,pass,pass,pass +regnety_002,pass,fail_accuracy,pass,fail_accuracy,pass +repvgg_a2,pass,fail_accuracy,pass,fail_accuracy,pass +res2net101_26w_4s,pass,fail_accuracy,pass,fail_accuracy,pass +res2net50_14w_8s,pass,fail_accuracy,pass,fail_accuracy,pass +res2next50,pass,fail_accuracy,pass,fail_accuracy,pass resmlp_12_224,pass,pass,pass,pass,pass resnest101e,pass,pass,pass,pass,pass -rexnet_100,pass,pass,pass,pass,pass +rexnet_100,pass,fail_accuracy,pass,fail_accuracy,pass sebotnet33ts_256,pass,pass,pass,pass,pass -selecsls42b,pass,pass,pass,pass,pass -spnasnet_100,pass,pass,pass,pass,pass +selecsls42b,pass,fail_accuracy,pass,fail_accuracy,pass +spnasnet_100,pass,fail_accuracy,pass,fail_accuracy,pass swin_base_patch4_window7_224,pass,pass,pass,pass,pass -swsl_resnext101_32x16d,pass,pass,pass,pass,pass -tf_efficientnet_b0,pass,pass,pass,pass,pass -tf_mixnet_l,pass,pass,pass,pass,pass +swsl_resnext101_32x16d,pass,fail_accuracy,pass,fail_accuracy,pass +tf_efficientnet_b0,pass,fail_accuracy,pass,fail_accuracy,pass +tf_mixnet_l,pass,fail_accuracy,pass,fail_accuracy,pass tinynet_a,pass,pass,pass,pass,pass tnt_s_patch16_224,pass,pass,pass,pass,pass twins_pcpvt_base,pass,pass,pass,pass,pass -visformer_small,pass,pass,pass,pass,pass +visformer_small,pass,fail_accuracy,pass,fail_accuracy,pass vit_base_patch16_224,pass,pass,pass,pass,pass volo_d1_224,pass,pass,pass,pass,pass xcit_large_24_p8_224,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip diff --git a/.github/ci_expected_accuracy/inductor_torchbench_inference.csv b/.github/ci_expected_accuracy/inductor_torchbench_inference.csv index 792bfac0b..f26592d43 100644 --- a/.github/ci_expected_accuracy/inductor_torchbench_inference.csv +++ b/.github/ci_expected_accuracy/inductor_torchbench_inference.csv @@ -1,5 +1,5 @@ name,float32,bfloat16,float16,amp_bf16,amp_fp16 -torchrec_dlrm,pass,eager_fail_to_run,eager_fail_to_run,pass,pass +torchrec_dlrm,pass,eager_fail_to_run,eager_fail_to_run,fail_to_run,fail_to_run BERT_pytorch,pass,pass,pass,pass,pass Background_Matting,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip DALLE2_pytorch,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run @@ -13,7 +13,7 @@ basic_gnn_sage,pass,pass,pass,pass,pass cm3leon_generate,pass,pass,pass,pass,pass dcgan,pass,pass,pass,pass,pass demucs,pass,pass,pass,pass,pass -densenet121,pass,pass,pass,pass,pass +densenet121,pass,pass,pass,fail_accuracy,pass detectron2_fasterrcnn_r_101_c4,pass,eager_fail_to_run,fail_accuracy,eager_fail_to_run,fail_accuracy detectron2_fasterrcnn_r_101_dc5,pass,eager_fail_to_run,fail_accuracy,eager_fail_to_run,fail_accuracy detectron2_fasterrcnn_r_101_fpn,pass,eager_fail_to_run,fail_accuracy,eager_fail_to_run,fail_accuracy @@ -32,7 +32,7 @@ doctr_reco_predictor,pass,pass,pass,pass,pass drq,pass,pass,pass,pass,pass fastNLP_Bert,pass,pass,pass,pass,pass functorch_dp_cifar10,pass,pass,pass,pass,pass -functorch_maml_omniglot,pass,pass,pass,pass,pass +functorch_maml_omniglot,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run hf_Albert,pass,pass,pass,pass,pass hf_Bart,pass,pass,pass,pass,pass hf_Bert,pass,pass,pass,pass,pass @@ -44,7 +44,7 @@ hf_GPT2_large,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_ski hf_Longformer,pass,pass,pass,pass,pass hf_Reformer,pass,pass,pass,pass,pass hf_T5,pass,pass,pass,pass,pass -hf_T5_base,pass,pass,pass,pass,pass +hf_T5_base,pass,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run hf_T5_generate,pass,pass,pass,pass,pass hf_T5_large,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip hf_Whisper,pass,pass,pass,pass,pass @@ -53,19 +53,19 @@ hf_distil_whisper,pass,pass,pass,pass,pass lennard_jones,pass,pass,pass,pass,pass llama,pass,pass,pass,pass,pass llama_v2_7b_16h,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip -llava,eager_fail_to_run,pass,pass,eager_fail_to_run,eager_fail_to_run +llava,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run maml,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip -maml_omniglot,pass,pass,pass,pass,pass +maml_omniglot,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run microbench_unbacked_tolist_sum,pass,pass,pass,pass,pass -mnasnet1_0,pass,pass,pass,pass,pass -mobilenet_v2,pass,pass,pass,pass,pass +mnasnet1_0,pass,fail_accuracy,pass,fail_accuracy,pass +mobilenet_v2,pass,fail_accuracy,pass,fail_accuracy,pass mobilenet_v2_quantized_qat,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load mobilenet_v3_large,pass,pass,pass,pass,pass moco,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load moondream,pass,pass,pass,pass,pass nanogpt,pass,pass,pass,pass,pass nvidia_deeprecommender,pass,pass,pass,pass,pass -opacus_cifar10,pass,pass,pass,pass,pass +opacus_cifar10,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run phlippe_densenet,pass,pass,pass,pass,pass phlippe_resnet,pass,pass,pass,pass,pass pyhpc_equation_of_state,pass,pass,pass,pass,pass @@ -74,30 +74,30 @@ pyhpc_turbulent_kinetic_energy,pass,pass,pass,pass,pass pytorch_CycleGAN_and_pix2pix,pass,pass,pass,pass,pass pytorch_stargan,pass,pass,pass,pass,pass pytorch_unet,pass,pass,pass,pass,pass -resnet152,pass,pass,pass,pass,pass -resnet18,pass,pass,pass,pass,pass +resnet152,pass,fail_accuracy,pass,fail_accuracy,pass +resnet18,pass,pass,pass,fail_accuracy,pass resnet50,pass,pass,pass,pass,pass resnet50_quantized_qat,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load -resnext50_32x4d,pass,pass,pass,pass,pass +resnext50_32x4d,pass,pass,pass,fail_accuracy,pass sam,pass,pass,pass,pass,pass sam_fast,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run -shufflenet_v2_x1_0,pass,pass,pass,pass,pass +shufflenet_v2_x1_0,pass,fail_accuracy,pass,fail_accuracy,pass simple_gpt,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load simple_gpt_tp_manual,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load soft_actor_critic,pass,pass,pass,pass,pass speech_transformer,pass,pass,pass,pass,pass squeezenet1_1,pass,fail_accuracy,pass,pass,pass -stable_diffusion_text_encoder,pass,pass,pass,pass,pass -stable_diffusion_unet,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip +stable_diffusion_text_encoder,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run +stable_diffusion_unet,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run tacotron2,pass,pass,pass,fail_to_run,fail_to_run timm_efficientdet,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load timm_efficientnet,pass,pass,pass,pass,pass -timm_nfnet,pass,pass,pass,pass,pass -timm_regnet,pass,pass,pass,pass,pass -timm_resnest,pass,pass,pass,pass,pass +timm_nfnet,pass,fail_accuracy,pass,pass,pass +timm_regnet,pass,pass,pass,fail_accuracy,pass +timm_resnest,pass,fail_accuracy,pass,fail_accuracy,pass timm_vision_transformer,pass,pass,pass,pass,pass timm_vision_transformer_large,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip -timm_vovnet,pass,pass,pass,pass,pass +timm_vovnet,pass,fail_accuracy,pass,fail_accuracy,pass torch_multimodal_clip,pass,pass,pass,pass,pass tts_angular,pass,pass,pass,pass,pass vgg16,pass,pass,pass,pass,pass diff --git a/.github/ci_expected_accuracy/inductor_torchbench_training.csv b/.github/ci_expected_accuracy/inductor_torchbench_training.csv index 4c1bc6aba..4b4322c88 100644 --- a/.github/ci_expected_accuracy/inductor_torchbench_training.csv +++ b/.github/ci_expected_accuracy/inductor_torchbench_training.csv @@ -13,7 +13,7 @@ basic_gnn_sage,pass,pass,pass,pass,pass cm3leon_generate,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run dcgan,pass,pass,pass,pass,pass demucs,fail_accuracy,eager_fail_to_run,eager_fail_to_run,fail_accuracy,fail_accuracy -densenet121,pass,pass,pass,pass,pass +densenet121,pass,pass,pass,fail_accuracy,pass detectron2_fasterrcnn_r_101_c4,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run detectron2_fasterrcnn_r_101_dc5,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run detectron2_fasterrcnn_r_101_fpn,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run @@ -32,7 +32,7 @@ doctr_reco_predictor,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager drq,pass,pass,pass,pass,pass fastNLP_Bert,pass,pass,pass,pass,pass functorch_dp_cifar10,fail_accuracy,fail_accuracy,fail_accuracy,pass,pass -functorch_maml_omniglot,pass,pass,pass,eager_fail_to_run,pass +functorch_maml_omniglot,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run hf_Albert,pass,pass,pass,pass,pass hf_Bart,pass,pass,pass,pass,pass hf_Bert,pass,pass,pass,pass,pass @@ -55,10 +55,10 @@ llama,pass,pass,pass,pass,pass llama_v2_7b_16h,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip llava,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run maml,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run -maml_omniglot,pass,pass,pass,pass,pass +maml_omniglot,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run microbench_unbacked_tolist_sum,pass,pass,pass,pass,pass -mnasnet1_0,pass,pass,pass,pass,pass -mobilenet_v2,pass,pass,pass,pass,pass +mnasnet1_0,pass,fail_accuracy,pass,fail_accuracy,pass +mobilenet_v2,pass,fail_accuracy,pass,fail_accuracy,pass mobilenet_v2_quantized_qat,pass,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run mobilenet_v3_large,pass,pass,pass,pass,pass moco,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load @@ -74,33 +74,33 @@ pyhpc_turbulent_kinetic_energy,model_fail_to_load,model_fail_to_load,model_fail_ pytorch_CycleGAN_and_pix2pix,eager_two_runs_differ,eager_two_runs_differ,pass,eager_two_runs_differ,pass pytorch_stargan,pass,pass,pass,pass,pass pytorch_unet,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip -resnet152,pass,pass,pass,pass,pass -resnet18,pass,pass,pass,pass,pass -resnet50,pass,pass,pass,pass,pass +resnet152,pass,fail_accuracy,pass,fail_accuracy,pass +resnet18,pass,pass,pass,fail_accuracy,pass +resnet50,pass,pass,pass,fail_accuracy,pass resnet50_quantized_qat,pass,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run -resnext50_32x4d,pass,pass,pass,pass,pass +resnext50_32x4d,pass,pass,pass,fail_accuracy,pass sam,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run sam_fast,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run -shufflenet_v2_x1_0,pass,pass,pass,pass,pass +shufflenet_v2_x1_0,pass,fail_accuracy,pass,fail_accuracy,pass simple_gpt,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load simple_gpt_tp_manual,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load soft_actor_critic,pass,pass,pass,pass,pass speech_transformer,pass,pass,pass,pass,pass squeezenet1_1,pass,pass,pass,pass,pass -stable_diffusion_text_encoder,pass,pass,pass,pass,pass -stable_diffusion_unet,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip +stable_diffusion_text_encoder,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run +stable_diffusion_unet,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run tacotron2,fail_to_run,fail_to_run,fail_to_run,fail_to_run,fail_to_run timm_efficientdet,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load timm_efficientnet,pass,pass,pass,pass,pass -timm_nfnet,pass,pass,pass,pass,pass -timm_regnet,pass,fail_accuracy,pass,pass,pass -timm_resnest,pass,pass,pass,pass,pass +timm_nfnet,pass,fail_accuracy,pass,pass,pass +timm_regnet,pass,fail_accuracy,fail_accuracy,fail_accuracy,fail_accuracy +timm_resnest,pass,fail_accuracy,pass,fail_accuracy,pass timm_vision_transformer,pass,pass,pass,pass,pass timm_vision_transformer_large,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip -timm_vovnet,pass,pass,pass,pass,pass +timm_vovnet,pass,fail_accuracy,pass,fail_accuracy,pass torch_multimodal_clip,pass,pass,pass,pass,pass tts_angular,pass,pass,pass,pass,pass vgg16,pass,pass,pass,pass,pass -vision_maskrcnn,pass,pass,pass,eager_fail_to_run,eager_fail_to_run +vision_maskrcnn,eager_fail_to_run,pass,pass,eager_fail_to_run,eager_fail_to_run yolov3,pass,pass,pass,pass,pass hf_Roberta_base,pass,pass,pass,pass,pass