Use stratified sampling for JCoLA

Stability-AI · Oct 20, 2023 · 2f25eda · 2f25eda
1 parent 6ab2dd7
commit 2f25eda
Showing 1 changed file with 5 additions and 1 deletion.
diff --git a/lm_eval/tasks/ja/jcola.py b/lm_eval/tasks/ja/jcola.py
@@ -23,7 +23,7 @@
 
 
 class JCoLA(CoLA):
-    VERSION = 0.1
+    VERSION = 0.2
     PROMPT_VERSION = 0.0
     DATASET_PATH = "shunk031/JGLUE"
     DATASET_NAME = "JCoLA"
@@ -43,6 +43,10 @@ def construct_requests(self, doc, ctx):
         ll_false, _ = rf.loglikelihood(ctx, " %s" % self.CHOICES[0])
         return ll_true, ll_false
 
+    def fewshot_context(self, doc, num_fewshot, provide_description=None, rnd=None, description=None, stratified=False):
+        # Use stratified sampling
+        return super().fewshot_context(doc, num_fewshot, provide_description, rnd, description, stratified=True)
+
 
 class JCoLAWithJAAlpacaPrompt(JCoLA):
     """