Xfail 405b as sharding vmfb is unsupported

archana-ramalingam · archana-ramalingam · commit 8069f24cbd39 · 2024-10-28T14:47:43.000-05:00
diff --git a/sharktank/tests/evaluate/perplexity_vmfb_test.py b/sharktank/tests/evaluate/perplexity_vmfb_test.py
@@ -164,6 +164,9 @@ def test_llama3_8B_fp8(self):
             msg=f"Current perplexity deviates baseline by {perplexity_difference}",
         )
 
+    @pytest.mark.xfail(
+        reason="Sharding is not supported yet",
+    )
     @longrun
     def test_llama3_405B_f16_decomposed(self):
 
@@ -267,7 +270,7 @@ def test_llama3_405B_fp8_decomposed(self):
         )
 
     @pytest.mark.xfail(
-        reason="Non-decomposed attention is not supported yet",
+        reason="FP8 model is unsupported",
     )
     @longrun
     def test_llama3_405B_fp8(self):

Original file line number	Diff line number	Diff line change
`@@ -164,6 +164,9 @@ def test_llama3_8B_fp8(self):`
`164`	`164`	`msg=f"Current perplexity deviates baseline by {perplexity_difference}",`
`165`	`165`	`)`
`166`	`166`
	`167`	`+ @pytest.mark.xfail(`
	`168`	`+ reason="Sharding is not supported yet",`
	`169`	`+ )`
`167`	`170`	`@longrun`
`168`	`171`	`def test_llama3_405B_f16_decomposed(self):`
`169`	`172`
`@@ -267,7 +270,7 @@ def test_llama3_405B_fp8_decomposed(self):`
`267`	`270`	`)`
`268`	`271`
`269`	`272`	`@pytest.mark.xfail(`
`270`		`- reason="Non-decomposed attention is not supported yet",`
	`273`	`+ reason="FP8 model is unsupported",`
`271`	`274`	`)`
`272`	`275`	`@longrun`
`273`	`276`	`def test_llama3_405B_fp8(self):`