use llm-foundry for ICL metrics (mlfoundations#287)

* use llm-foundry for ICL metrics

* replaced InContextLearningQAAccuracy with InContextLearningGenerationExactMatchAccuracy, and removed the commented line for code evals

* resolve lint check

* Update requirements_test.txt

* Update requirements_test.txt

---------

Co-authored-by: Achal Dave <[email protected]>
Co-authored-by: jmercat <[email protected]>
Co-authored-by: sedrick-keh-tri <[email protected]>

Author: 4 people

open_lm/utils/llm_foundry_wrapper.py

@@ -4,14 +4,14 @@
 """Implements a Hugging Causal LM wrapped inside a :class:`.ComposerModel`."""
 
 from typing import Mapping, Union
-
-from composer.metrics.nlp import (
+from llmfoundry.eval.metrics.nlp import (
     InContextLearningLMAccuracy,
     InContextLearningLMExpectedCalibrationError,
     InContextLearningMCExpectedCalibrationError,
     InContextLearningMultipleChoiceAccuracy,
-    InContextLearningQAAccuracy,
-    InContextLearningCodeEvalAccuracy,
+    InContextLearningGenerationExactMatchAccuracy,
+)
+from composer.metrics.nlp import (
     LanguageCrossEntropy,
     LanguagePerplexity,
 )
@@ -33,10 +33,9 @@
     LanguagePerplexity(),
     InContextLearningLMAccuracy(),
     InContextLearningMultipleChoiceAccuracy(),
-    InContextLearningQAAccuracy(),
+    InContextLearningGenerationExactMatchAccuracy(),
     InContextLearningLMExpectedCalibrationError(),
     InContextLearningMCExpectedCalibrationError(),
-    InContextLearningCodeEvalAccuracy(),
 ]
 
 

requirements_test.txt

@@ -3,4 +3,4 @@ pytest-cov==3.0.0
 pytest-xdist==2.5.0
 pytest==7.0.1
 tensorboard==2.14.1
-llm-foundry==0.7.0
+llm-foundry==0.9.0