Update weekly accuracy reference

.github/ci_expected_accuracy/inductor_huggingface_training.csv

@@ -13,7 +13,7 @@ CamemBert,pass,pass,pass,pass,pass
 DebertaForMaskedLM,pass,pass,pass,pass,pass
 DebertaForQuestionAnswering,pass,pass,pass,pass,pass
 DebertaV2ForMaskedLM,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip
-DebertaV2ForQuestionAnswering,pass,pass,pass,pass,pass
+DebertaV2ForQuestionAnswering,fail_accuracy,fail_accuracy,fail_accuracy,pass,pass
 DistilBertForMaskedLM,pass,pass,pass,pass,pass
 DistilBertForQuestionAnswering,pass,pass,pass,pass,pass
 DistillGPT2,pass,pass,pass,pass,pass

.github/ci_expected_accuracy/inductor_timm_models_inference.csv

@@ -1,5 +1,5 @@
 name,float32,bfloat16,float16,amp_bf16,amp_fp16
-adv_inception_v3,pass,pass,pass,pass,pass
+adv_inception_v3,pass,pass,pass,fail_accuracy,pass
 beit_base_patch16_224,pass,pass,pass,pass,pass
 botnet26t_256,pass,pass,pass,pass,pass
 cait_m36_384,pass,pass,pass,pass,pass
@@ -10,53 +10,53 @@ convnext_base,pass,pass,pass,pass,pass
 crossvit_9_240,pass,pass,pass,pass,pass
 cspdarknet53,pass,pass,pass,pass,pass
 deit_base_distilled_patch16_224,pass,pass,pass,pass,pass
-dla102,pass,pass,pass,pass,pass
-dm_nfnet_f0,pass,pass,pass,pass,pass
-dpn107,pass,pass,pass,pass,pass
+dla102,pass,fail_accuracy,pass,fail_accuracy,pass
+dm_nfnet_f0,pass,fail_accuracy,pass,pass,pass
+dpn107,pass,fail_accuracy,pass,fail_accuracy,pass
 eca_botnext26ts_256,pass,pass,pass,pass,pass
 eca_halonext26ts,pass,pass,pass,pass,pass
 ese_vovnet19b_dw,pass,pass,pass,pass,pass
-fbnetc_100,pass,pass,pass,pass,pass
+fbnetc_100,pass,fail_accuracy,pass,fail_accuracy,pass
 fbnetv3_b,pass,pass,pass,pass,pass
 gernet_l,pass,pass,pass,pass,pass
-ghostnet_100,pass,pass,pass,pass,pass
-gluon_inception_v3,pass,pass,pass,pass,pass
+ghostnet_100,pass,fail_accuracy,pass,fail_accuracy,pass
+gluon_inception_v3,pass,pass,pass,fail_accuracy,pass
 gmixer_24_224,pass,pass,pass,pass,pass
 gmlp_s16_224,pass,pass,pass,pass,pass
-hrnet_w18,pass,pass,pass,pass,pass
-inception_v3,pass,pass,pass,pass,pass
+hrnet_w18,pass,fail_accuracy,pass,fail_accuracy,pass
+inception_v3,pass,fail_accuracy,pass,fail_accuracy,pass
 jx_nest_base,pass,pass,pass,pass,pass
-lcnet_050,pass,pass,pass,pass,pass
+lcnet_050,pass,fail_accuracy,pass,fail_accuracy,pass
 levit_128,pass,pass,pass,pass,pass
 mixer_b16_224,pass,pass,pass,pass,pass
 mixnet_l,pass,pass,pass,pass,pass
-mnasnet_100,pass,pass,pass,pass,pass
-mobilenetv2_100,pass,pass,pass,pass,pass
+mnasnet_100,pass,fail_accuracy,pass,fail_accuracy,pass
+mobilenetv2_100,pass,pass,pass,fail_accuracy,pass
 mobilenetv3_large_100,pass,pass,pass,pass,pass
 mobilevit_s,pass,pass,pass,pass,pass
 nfnet_l0,pass,pass,pass,pass,pass
 pit_b_224,pass,pass,pass,pass,pass
 pnasnet5large,pass,pass,pass,pass,pass
 poolformer_m36,pass,pass,pass,pass,pass
-regnety_002,pass,pass,pass,pass,pass
+regnety_002,pass,fail_accuracy,pass,fail_accuracy,pass
 repvgg_a2,pass,pass,pass,pass,pass
-res2net101_26w_4s,pass,pass,pass,pass,pass
-res2net50_14w_8s,pass,pass,pass,pass,pass
-res2next50,pass,pass,pass,pass,pass
+res2net101_26w_4s,pass,pass,pass,fail_accuracy,pass
+res2net50_14w_8s,pass,pass,pass,fail_accuracy,pass
+res2next50,pass,pass,pass,fail_accuracy,pass
 resmlp_12_224,pass,pass,pass,pass,pass
 resnest101e,pass,pass,pass,pass,pass
-rexnet_100,pass,pass,pass,pass,pass
+rexnet_100,pass,pass,pass,fail_accuracy,pass
 sebotnet33ts_256,pass,pass,pass,pass,pass
 selecsls42b,pass,pass,pass,pass,pass
-spnasnet_100,pass,pass,pass,pass,pass
+spnasnet_100,pass,fail_accuracy,pass,fail_accuracy,pass
 swin_base_patch4_window7_224,pass,pass,pass,pass,pass
-swsl_resnext101_32x16d,pass,pass,pass,pass,pass
-tf_efficientnet_b0,pass,pass,pass,pass,pass
+swsl_resnext101_32x16d,pass,fail_accuracy,pass,fail_accuracy,pass
+tf_efficientnet_b0,pass,fail_accuracy,pass,fail_accuracy,pass
 tf_mixnet_l,pass,pass,pass,pass,pass
 tinynet_a,pass,pass,pass,pass,pass
 tnt_s_patch16_224,pass,pass,pass,pass,pass
 twins_pcpvt_base,pass,pass,pass,pass,pass
 visformer_small,pass,pass,pass,pass,pass
 vit_base_patch16_224,pass,pass,pass,pass,pass
 volo_d1_224,pass,pass,pass,pass,pass
-xcit_large_24_p8_224,pass,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run
+xcit_large_24_p8_224,pass,pass,pass,pass,pass

.github/ci_expected_accuracy/inductor_timm_models_training.csv

@@ -1,62 +1,62 @@
 name,float32,bfloat16,float16,amp_bf16,amp_fp16
-adv_inception_v3,pass,pass,pass,pass,pass
+adv_inception_v3,pass,fail_accuracy,pass,fail_accuracy,pass
 beit_base_patch16_224,pass,pass,pass,pass,pass
 botnet26t_256,pass,pass,pass,pass,pass
 cait_m36_384,pass,pass,pass,pass,pass
 coat_lite_mini,pass,pass,pass,pass,pass
 convit_base,pass,pass,pass,pass,pass
-convmixer_768_32,pass,pass,pass,pass,pass
-convnext_base,pass,pass,fail_accuracy,pass,pass
+convmixer_768_32,pass,fail_accuracy,pass,fail_accuracy,pass
+convnext_base,pass,fail_accuracy,fail_accuracy,pass,pass
 crossvit_9_240,pass,pass,pass,pass,pass
 cspdarknet53,pass,pass,pass,pass,pass
 deit_base_distilled_patch16_224,pass,pass,pass,pass,pass
-dla102,pass,pass,pass,pass,pass
-dm_nfnet_f0,pass,pass,pass,pass,pass
+dla102,pass,fail_accuracy,pass,fail_accuracy,pass
+dm_nfnet_f0,pass,fail_accuracy,pass,pass,pass
 dpn107,pass,pass,pass,pass,pass
 eca_botnext26ts_256,pass,pass,pass,pass,pass
-eca_halonext26ts,pass,pass,fail_to_run,pass,pass
-ese_vovnet19b_dw,pass,pass,pass,pass,pass
-fbnetc_100,pass,pass,pass,pass,pass
-fbnetv3_b,pass,pass,pass,pass,pass
+eca_halonext26ts,pass,pass,pass,fail_accuracy,pass
+ese_vovnet19b_dw,pass,fail_accuracy,pass,fail_accuracy,pass
+fbnetc_100,pass,fail_accuracy,pass,fail_accuracy,pass
+fbnetv3_b,pass,fail_accuracy,pass,fail_accuracy,pass
 gernet_l,pass,pass,pass,pass,pass
-ghostnet_100,pass,pass,pass,pass,pass
-gluon_inception_v3,pass,pass,pass,pass,pass
+ghostnet_100,pass,fail_accuracy,pass,fail_accuracy,pass
+gluon_inception_v3,pass,fail_accuracy,pass,fail_accuracy,pass
 gmixer_24_224,pass,pass,pass,pass,pass
 gmlp_s16_224,pass,pass,pass,pass,pass
-hrnet_w18,pass,pass,pass,pass,pass
-inception_v3,pass,pass,pass,pass,pass
-jx_nest_base,pass,pass,fail_accuracy,pass,pass
-lcnet_050,pass,pass,fail_accuracy,pass,pass
+hrnet_w18,pass,fail_accuracy,pass,fail_accuracy,pass
+inception_v3,pass,fail_accuracy,pass,fail_accuracy,pass
+jx_nest_base,pass,pass,pass,pass,pass
+lcnet_050,pass,fail_accuracy,pass,fail_accuracy,pass
 levit_128,pass,pass,pass,pass,pass
 mixer_b16_224,pass,pass,pass,pass,pass
-mixnet_l,pass,pass,pass,pass,pass
-mnasnet_100,pass,pass,pass,pass,pass
-mobilenetv2_100,pass,pass,pass,pass,pass
-mobilenetv3_large_100,pass,pass,pass,pass,pass
+mixnet_l,pass,fail_accuracy,pass,fail_accuracy,pass
+mnasnet_100,pass,fail_accuracy,pass,fail_accuracy,pass
+mobilenetv2_100,pass,fail_accuracy,pass,fail_accuracy,pass
+mobilenetv3_large_100,pass,fail_accuracy,pass,fail_accuracy,pass
 mobilevit_s,pass,pass,pass,pass,pass
 nfnet_l0,pass,pass,pass,pass,pass
 pit_b_224,pass,pass,pass,pass,pass
-pnasnet5large,pass,pass,pass,pass,pass
+pnasnet5large,pass,fail_accuracy,pass,fail_accuracy,pass
 poolformer_m36,pass,pass,pass,pass,pass
-regnety_002,pass,pass,pass,pass,pass
-repvgg_a2,pass,pass,pass,pass,pass
-res2net101_26w_4s,pass,pass,pass,pass,pass
-res2net50_14w_8s,pass,pass,pass,pass,pass
-res2next50,pass,pass,pass,pass,pass
+regnety_002,pass,fail_accuracy,pass,fail_accuracy,pass
+repvgg_a2,pass,fail_accuracy,pass,fail_accuracy,pass
+res2net101_26w_4s,pass,fail_accuracy,pass,fail_accuracy,pass
+res2net50_14w_8s,pass,fail_accuracy,pass,fail_accuracy,pass
+res2next50,pass,fail_accuracy,pass,fail_accuracy,pass
 resmlp_12_224,pass,pass,pass,pass,pass
 resnest101e,pass,pass,pass,pass,pass
-rexnet_100,pass,pass,pass,pass,pass
+rexnet_100,pass,fail_accuracy,pass,fail_accuracy,pass
 sebotnet33ts_256,pass,pass,pass,pass,pass
-selecsls42b,pass,pass,pass,pass,pass
-spnasnet_100,pass,pass,pass,pass,pass
+selecsls42b,pass,fail_accuracy,pass,fail_accuracy,pass
+spnasnet_100,pass,fail_accuracy,pass,fail_accuracy,pass
 swin_base_patch4_window7_224,pass,pass,pass,pass,pass
-swsl_resnext101_32x16d,pass,pass,pass,pass,pass
-tf_efficientnet_b0,pass,pass,pass,pass,pass
-tf_mixnet_l,pass,pass,pass,pass,pass
+swsl_resnext101_32x16d,pass,fail_accuracy,pass,fail_accuracy,pass
+tf_efficientnet_b0,pass,fail_accuracy,pass,fail_accuracy,pass
+tf_mixnet_l,pass,fail_accuracy,pass,fail_accuracy,pass
 tinynet_a,pass,pass,pass,pass,pass
 tnt_s_patch16_224,pass,pass,pass,pass,pass
 twins_pcpvt_base,pass,pass,pass,pass,pass
-visformer_small,pass,pass,pass,pass,pass
+visformer_small,pass,fail_accuracy,pass,fail_accuracy,pass
 vit_base_patch16_224,pass,pass,pass,pass,pass
 volo_d1_224,pass,pass,pass,pass,pass
-xcit_large_24_p8_224,pass_due_to_skip,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run
+xcit_large_24_p8_224,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip

.github/ci_expected_accuracy/inductor_torchbench_inference.csv

@@ -1,5 +1,5 @@
 name,float32,bfloat16,float16,amp_bf16,amp_fp16
-torchrec_dlrm,pass,eager_fail_to_run,eager_fail_to_run,pass,pass
+torchrec_dlrm,pass,eager_fail_to_run,eager_fail_to_run,fail_to_run,fail_to_run
 BERT_pytorch,pass,pass,pass,pass,pass
 Background_Matting,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip
 DALLE2_pytorch,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run
@@ -13,7 +13,7 @@ basic_gnn_sage,pass,pass,pass,pass,pass
 cm3leon_generate,pass,pass,pass,pass,pass
 dcgan,pass,pass,pass,pass,pass
 demucs,pass,pass,pass,pass,pass
-densenet121,pass,pass,pass,pass,pass
+densenet121,pass,pass,pass,fail_accuracy,pass
 detectron2_fasterrcnn_r_101_c4,pass,eager_fail_to_run,fail_accuracy,eager_fail_to_run,fail_accuracy
 detectron2_fasterrcnn_r_101_dc5,pass,eager_fail_to_run,fail_accuracy,eager_fail_to_run,fail_accuracy
 detectron2_fasterrcnn_r_101_fpn,pass,eager_fail_to_run,fail_accuracy,eager_fail_to_run,fail_accuracy
@@ -25,14 +25,14 @@ detectron2_maskrcnn,fail_to_run,eager_fail_to_run,fail_to_run,eager_fail_to_run,
 detectron2_maskrcnn_r_101_c4,fail_accuracy,eager_fail_to_run,fail_accuracy,eager_fail_to_run,fail_accuracy
 detectron2_maskrcnn_r_101_fpn,fail_accuracy,eager_fail_to_run,fail_accuracy,eager_fail_to_run,fail_accuracy
 detectron2_maskrcnn_r_50_c4,pass,eager_fail_to_run,fail_accuracy,eager_fail_to_run,fail_accuracy
-detectron2_maskrcnn_r_50_fpn,fail_accuracy,eager_fail_to_run,fail_accuracy,eager_fail_to_run,fail_accuracy
+detectron2_maskrcnn_r_50_fpn,pass,eager_fail_to_run,fail_accuracy,eager_fail_to_run,fail_accuracy
 dlrm,pass,pass,pass,pass,pass
-doctr_det_predictor,pass,pass,pass,eager_fail_to_run,pass
+doctr_det_predictor,pass,pass,pass,pass,pass
 doctr_reco_predictor,pass,pass,pass,pass,pass
 drq,pass,pass,pass,pass,pass
 fastNLP_Bert,pass,pass,pass,pass,pass
 functorch_dp_cifar10,pass,pass,pass,pass,pass
-functorch_maml_omniglot,pass,pass,pass,pass,pass
+functorch_maml_omniglot,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run
 hf_Albert,pass,pass,pass,pass,pass
 hf_Bart,pass,pass,pass,pass,pass
 hf_Bert,pass,pass,pass,pass,pass
@@ -44,7 +44,7 @@ hf_GPT2_large,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_ski
 hf_Longformer,pass,pass,pass,pass,pass
 hf_Reformer,pass,pass,pass,pass,pass
 hf_T5,pass,pass,pass,pass,pass
-hf_T5_base,pass,pass,pass,pass,pass
+hf_T5_base,pass,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run
 hf_T5_generate,pass,pass,pass,pass,pass
 hf_T5_large,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip
 hf_Whisper,pass,pass,pass,pass,pass
@@ -53,53 +53,53 @@ hf_distil_whisper,pass,pass,pass,pass,pass
 lennard_jones,pass,pass,pass,pass,pass
 llama,pass,pass,pass,pass,pass
 llama_v2_7b_16h,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip
-llava,eager_fail_to_run,pass,pass,eager_fail_to_run,eager_fail_to_run
+llava,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run
 maml,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip
-maml_omniglot,pass,pass,pass,pass,pass
+maml_omniglot,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run
 microbench_unbacked_tolist_sum,pass,pass,pass,pass,pass
-mnasnet1_0,pass,pass,pass,pass,pass
-mobilenet_v2,pass,pass,pass,pass,pass
+mnasnet1_0,pass,fail_accuracy,pass,fail_accuracy,pass
+mobilenet_v2,pass,fail_accuracy,pass,fail_accuracy,pass
 mobilenet_v2_quantized_qat,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load
 mobilenet_v3_large,pass,pass,pass,pass,pass
 moco,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load
 moondream,pass,pass,pass,pass,pass
 nanogpt,pass,pass,pass,pass,pass
 nvidia_deeprecommender,pass,pass,pass,pass,pass
-opacus_cifar10,pass,pass,pass,pass,pass
+opacus_cifar10,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run
 phlippe_densenet,pass,pass,pass,pass,pass
 phlippe_resnet,pass,pass,pass,pass,pass
-pyhpc_equation_of_state,pass,pass,fail_accuracy,pass,pass
+pyhpc_equation_of_state,pass,pass,pass,pass,pass
 pyhpc_isoneutral_mixing,pass,pass,pass,pass,pass
 pyhpc_turbulent_kinetic_energy,pass,pass,pass,pass,pass
 pytorch_CycleGAN_and_pix2pix,pass,pass,pass,pass,pass
 pytorch_stargan,pass,pass,pass,pass,pass
 pytorch_unet,pass,pass,pass,pass,pass
-resnet152,pass,pass,pass,pass,pass
-resnet18,pass,pass,pass,pass,pass
+resnet152,pass,fail_accuracy,pass,fail_accuracy,pass
+resnet18,pass,pass,pass,fail_accuracy,pass
 resnet50,pass,pass,pass,pass,pass
 resnet50_quantized_qat,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load
-resnext50_32x4d,pass,pass,pass,pass,pass
+resnext50_32x4d,pass,pass,pass,fail_accuracy,pass
 sam,pass,pass,pass,pass,pass
 sam_fast,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run
-shufflenet_v2_x1_0,pass,pass,pass,pass,pass
+shufflenet_v2_x1_0,pass,fail_accuracy,pass,fail_accuracy,pass
 simple_gpt,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load
 simple_gpt_tp_manual,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load
 soft_actor_critic,pass,pass,pass,pass,pass
 speech_transformer,pass,pass,pass,pass,pass
-squeezenet1_1,pass,fail_accuracy,fail_accuracy,pass,pass
-stable_diffusion_text_encoder,pass,pass,pass,pass,pass
-stable_diffusion_unet,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip
+squeezenet1_1,pass,fail_accuracy,pass,pass,pass
+stable_diffusion_text_encoder,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run
+stable_diffusion_unet,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run
 tacotron2,pass,pass,pass,fail_to_run,fail_to_run
 timm_efficientdet,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load,model_fail_to_load
 timm_efficientnet,pass,pass,pass,pass,pass
-timm_nfnet,pass,pass,pass,pass,pass
-timm_regnet,pass,pass,pass,pass,pass
-timm_resnest,pass,pass,pass,pass,pass
+timm_nfnet,pass,fail_accuracy,pass,pass,pass
+timm_regnet,pass,pass,pass,fail_accuracy,pass
+timm_resnest,pass,fail_accuracy,pass,fail_accuracy,pass
 timm_vision_transformer,pass,pass,pass,pass,pass
 timm_vision_transformer_large,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip,pass_due_to_skip
-timm_vovnet,pass,pass,pass,pass,pass
-torch_multimodal_clip,pass,pass,pass,eager_fail_to_run,eager_fail_to_run
-tts_angular,pass,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run,eager_fail_to_run
+timm_vovnet,pass,fail_accuracy,pass,fail_accuracy,pass
+torch_multimodal_clip,pass,pass,pass,pass,pass
+tts_angular,pass,pass,pass,pass,pass
 vgg16,pass,pass,pass,pass,pass
 vision_maskrcnn,pass,pass,pass,eager_fail_to_run,eager_fail_to_run
 yolov3,pass,pass,pass,pass,pass