From ff702e704511e58cd60f162efd4a8b1727b0005f Mon Sep 17 00:00:00 2001
From: mkshing <mktshing4@gmail.com>
Date: Thu, 12 Oct 2023 14:28:21 +0900
Subject: [PATCH 1/3] Add prompt version `0.21` for JCommonsenseQA

---
 lm_eval/tasks/ja/jcommonsenseqa.py | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/lm_eval/tasks/ja/jcommonsenseqa.py b/lm_eval/tasks/ja/jcommonsenseqa.py
index 0cc6e91bd6..961dbfea7e 100644
--- a/lm_eval/tasks/ja/jcommonsenseqa.py
+++ b/lm_eval/tasks/ja/jcommonsenseqa.py
@@ -134,6 +134,26 @@ def doc_to_target(self, doc):
         return f"{doc['gold']}"
 
 
+class JCommonsenseQAWithFintanPromptV21(JCommonsenseQA):
+    VERSION = 1.1
+    PROMPT_VERSION = 0.21
+    DESCRIPTION = "与えられた選択肢の中から、最適な答えを選んでください。 \n\n"
+
+    def doc_to_text(self, doc):
+        """
+        与えられた選択肢の中から、最適な答えを選んでください。
+
+        質問：{question}
+        選択肢：
+        - {choice0}
+        - {choice4}
+        回答：
+        """
+        choices = "\n".join([f"- {choice}" for choice in doc["choices"]])
+        input_text = f"質問：{doc['goal']}\n選択肢：\n{choices}\n回答："
+        return input_text
+
+
 class JCommonsenseQAWithJAAlpacaPrompt(JCommonsenseQA):
     """
     This prompt format was inspired by the below data in fujiki/japanese_alpaca_data.
@@ -246,6 +266,7 @@ def doc_to_text(self, doc):
 VERSIONS = [
     JCommonsenseQA,
     JCommonsenseQAWithFintanPrompt,
+    JCommonsenseQAWithFintanPromptV21,
     JCommonsenseQAWithJAAlpacaPrompt,
     JCommonsenseQAWithRinnaInstructionSFT,
     JCommonsenseQAWithRinnaBilingualInstructionSFT,

From ca467ca90a665dede9076233258434e25ad30407 Mon Sep 17 00:00:00 2001
From: mkshing <mktshing4@gmail.com>
Date: Thu, 12 Oct 2023 16:20:35 +0900
Subject: [PATCH 2/3] rename to "0.2.1" and add warning

---
 lm_eval/tasks/ja/jcommonsenseqa.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/lm_eval/tasks/ja/jcommonsenseqa.py b/lm_eval/tasks/ja/jcommonsenseqa.py
index 961dbfea7e..e5110c5467 100644
--- a/lm_eval/tasks/ja/jcommonsenseqa.py
+++ b/lm_eval/tasks/ja/jcommonsenseqa.py
@@ -8,6 +8,8 @@
 Homepage: https://github.com/yahoojapan/JGLUE
 """
 import os
+import time
+
 from lm_eval.base import MultipleChoiceTask, rf
 import numpy as np
 
@@ -118,6 +120,7 @@ class JCommonsenseQAWithFintanPrompt(JCommonsenseQA):
     DESCRIPTION = (
         "質問と回答の選択肢を入力として受け取り、選択肢から回答を選択してください。なお、回答は選択肢の番号(例:0)でするものとします。 \n\n"
     )
+    DID_WARNING = False
 
     def doc_to_text(self, doc):
         """
@@ -125,6 +128,14 @@ def doc_to_text(self, doc):
         選択肢:0.choice0,1.choice1, ...,4.choice4
         回答:
         """
+        if not self.DID_WARNING:
+            print(
+                "#" * 100
+                + "\n\nprompt version `0.2` for JCommonsenseQA tends to output low scores! We highly recommend using `0.2.1` instead!\n\n"
+                + "#" * 100
+            )
+            self.DID_WARNING = True
+            time.sleep(5)
         choices = ",".join(
             [f"{idx}.{choice}" for idx, choice in enumerate(doc["choices"])]
         )
@@ -136,7 +147,7 @@ def doc_to_target(self, doc):
 
 class JCommonsenseQAWithFintanPromptV21(JCommonsenseQA):
     VERSION = 1.1
-    PROMPT_VERSION = 0.21
+    PROMPT_VERSION = "0.2.1"
     DESCRIPTION = "与えられた選択肢の中から、最適な答えを選んでください。 \n\n"
 
     def doc_to_text(self, doc):

From c623c82719e375c9c5ad64dd34030ad90e0d93ca Mon Sep 17 00:00:00 2001
From: mkshing <mktshing4@gmail.com>
Date: Tue, 24 Oct 2023 13:40:25 +0900
Subject: [PATCH 3/3] change print to warnings

---
 lm_eval/tasks/ja/jcommonsenseqa.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lm_eval/tasks/ja/jcommonsenseqa.py b/lm_eval/tasks/ja/jcommonsenseqa.py
index e5110c5467..c7d94dc95a 100644
--- a/lm_eval/tasks/ja/jcommonsenseqa.py
+++ b/lm_eval/tasks/ja/jcommonsenseqa.py
@@ -8,6 +8,7 @@
 Homepage: https://github.com/yahoojapan/JGLUE
 """
 import os
+import warnings
 import time
 
 from lm_eval.base import MultipleChoiceTask, rf
@@ -129,7 +130,7 @@ def doc_to_text(self, doc):
         回答:
         """
         if not self.DID_WARNING:
-            print(
+            warnings.warn(
                 "#" * 100
                 + "\n\nprompt version `0.2` for JCommonsenseQA tends to output low scores! We highly recommend using `0.2.1` instead!\n\n"
                 + "#" * 100