add eval code and result

Author: Ming Li

evaluation/generation/eva_generation.py

@@ -5,7 +5,7 @@
 import os
 from tqdm import tqdm
 
-PROMPT_DICT = {
+PROMPT_DICT_ALPACA = {
     "prompt_input": (
         "Below is an instruction that describes a task, paired with an input that provides further context. "
         "Write a response that appropriately completes the request.\n\n"
@@ -17,20 +17,36 @@
         "### Instruction:\n{instruction}\n\n### Response:"
     ),
 }
+PROMPT_DICT_WIZARDLM = {
+    "prompt_input": (
+        "{instruction}\n{input}\n\n### Response:"
+    ),
+    "prompt_no_input": (
+        "{instruction}\n\n### Response:"
+    ),
+}
+PROMPT_DICT_VICUNA = {
+    "prompt_input": (
+        "A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions. USER: {instruction}\nInput:\n{input} ASSISTANT:"
+    ),
+    "prompt_no_input": (
+        "A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions. USER: {instruction} ASSISTANT:"
+    ),
+}
 
 def parse_args():
-    parser = argparse.ArgumentParser(description="Finetune a transformers model on a summarization task")
+    parser = argparse.ArgumentParser()
     parser.add_argument(
         "--dataset_name",
         type=str,
         default=None,
         help="The name of the dataset to use (via the datasets library).",
     )
     parser.add_argument(
-        "--training_data_source_name",
+        "--prompt",
         type=str,
         default='alpaca',
-        help="The training_data_source_name.",
+        help="alpaca, wiz, vicuna.",
     )
     parser.add_argument(
         "--num_beams",
@@ -47,12 +63,6 @@ def parse_args():
         help="Path to pretrained model or model identifier from huggingface.co/models.",
         required=False,
     )
-    parser.add_argument(
-        "--per_device_eval_batch_size",
-        type=int,
-        default=8,
-        help="Batch size (per device) for the evaluation dataloader.",
-    )
     parser.add_argument("--seed", type=int, default=0, help="A seed for reproducible training.")
     parser.add_argument("--max_length", type=int, default=1024)
     args = parser.parse_args()
@@ -69,26 +79,29 @@ def main():
 
     model.to(device)
     model.eval()
-    if(args.training_data_source_name=='alpaca'or args.training_data_source_name=='alpaca_gpt4'):
-        prompt_input, prompt_no_input = PROMPT_DICT["prompt_input"], PROMPT_DICT["prompt_no_input"]
+
+    if args.prompt == 'alpaca':
+        prompt_input, prompt_no_input = PROMPT_DICT_ALPACA["prompt_input"], PROMPT_DICT_ALPACA["prompt_no_input"]
+    elif args.prompt == 'wiz':
+        prompt_input, prompt_no_input = PROMPT_DICT_WIZARDLM["prompt_input"], PROMPT_DICT_WIZARDLM["prompt_no_input"]
+    elif args.prompt == 'vicuna':
+        prompt_input, prompt_no_input = PROMPT_DICT_VICUNA["prompt_input"], PROMPT_DICT_VICUNA["prompt_no_input"]
+
 
     if(args.dataset_name=="vicuna"):
-        dataset_path = './test_data/vicuna_test_set.jsonl'
+        dataset_path = 'evaluation/test_data/vicuna_test_set.jsonl'
         prompt_key = 'text'
     elif(args.dataset_name=="koala"):
-        dataset_path = './test_data/koala_test_set.jsonl'
+        dataset_path = 'evaluation/test_data/koala_test_set.jsonl'
         prompt_key = 'prompt'
     elif(args.dataset_name=="sinstruct"):
-        dataset_path = './test_data/sinstruct_test_set.jsonl'
+        dataset_path = 'evaluation/test_data/sinstruct_test_set.jsonl'
         prompt_key = 'instruction'
     elif(args.dataset_name=="wizardlm"):
-        dataset_path = './test_data/wizardlm_test_set.jsonl'
+        dataset_path = 'evaluation/test_data/wizardlm_test_set.jsonl'
         prompt_key = 'Instruction'
-    elif(args.dataset_name=="truthfulqa"):
-        dataset_path = './test_data/truthfulqa_test_set.jsonl'
-        prompt_key = 'Question'
     elif(args.dataset_name=="lima"):
-        dataset_path = './test_data/lima_test_set.jsonl'
+        dataset_path = 'evaluation/test_data/lima_test_set.jsonl'
         prompt_key = 'conversations'
 
     with open(dataset_path) as f:
@@ -111,14 +124,17 @@ def main():
             generate_ids = model.generate(input_ids, max_length=args.max_length)
             outputs = tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
             point['raw_output'] = outputs
-            point['response'] = outputs.split("Response:")[1]
+            if args.prompt in ['alpaca','wiz']:
+                point['response'] = outputs.split("Response:")[1]
+            elif args.prompt in ['vicuna']:
+                point['response'] = outputs.split("ASSISTANT:")[1]
             results.append(point)
 
     output_dir =  os.path.join(args.model_name_or_path, 'test_inference')
     if not os.path.exists(output_dir):
         os.makedirs(output_dir)
 
-    saved_name = args.dataset_name + "_" + str(args.seed) + '_' + str(args.max_length) + ".json"
+    saved_name = args.dataset_name + "_" + str(args.max_length) + ".json"
     with open(os.path.join(output_dir, saved_name), "w") as f:
         json.dump(results, f, indent=4)
 

evaluation/generation/eval.py

@@ -84,7 +84,7 @@ def get_json_list(file_path):
 
 
 if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="ChatGPT-based QA evaluation.")
+    parser = argparse.ArgumentParser()
     parser.add_argument("--wraped_file",default='')
     parser.add_argument("--api_key",type=str,default='')
     parser.add_argument("--api_model",type=str,default='gpt-3.5-turbo')
@@ -120,8 +120,6 @@ def get_json_list(file_path):
         prompt_key = 'instruction'
     elif(dataset_name=="wizardlm"):
         prompt_key = 'Instruction'
-    elif(dataset_name=="truthfulqa"):
-        prompt_key = 'Question'
     elif(dataset_name=="lima"):
         prompt_key = 'conversations'
 
@@ -192,12 +190,8 @@ def get_json_list(file_path):
         predictions_all.append(predictions)
 
     all_scores = []
-    ans1_win_idsx_list = []
-    ans2_win_idsx_list = []
     for reverse in range(2):
         scores_list = []
-        ans1_win_idsx = [0 for _ in range(total_len)]
-        ans2_win_idsx = [0 for _ in range(total_len)]
         predictions = predictions_all[reverse]
         for idx, prediction in enumerate(predictions):
             review = prediction['choices'][0]['message']['content']
@@ -207,40 +201,19 @@ def get_json_list(file_path):
             qa_jsons[idx][review_key] = review
             qa_jsons[idx][scores_key] = str(scores)
             scores_list.append(scores)
-            if scores[0] > scores[1]:
-                if not reverse:
-                    ans1_win_idsx[idx] = 1
-                else:
-                    ans2_win_idsx[idx] = 1
-            elif scores[1] > scores[0]:
-                if not reverse:
-                    ans2_win_idsx[idx] = 1
-                else:
-                    ans1_win_idsx[idx] = 1
 
         all_scores.append(scores_list)
         avg_scores = np.array(scores_list).mean(0)
         avg_key = 'average_scores' if not reverse else 'average_scores_reverse'
         meta_info[avg_key] = str(avg_scores.tolist())
 
-        ans1_win_idsx_list.append(ans1_win_idsx)
-        ans2_win_idsx_list.append(ans2_win_idsx)
-
-    ans1_win_idx_overall = np.array(ans1_win_idsx_list[0]) * np.array(ans1_win_idsx_list[1]) 
-    ans2_win_idx_overall = np.array(ans2_win_idsx_list[0]) * np.array(ans2_win_idsx_list[1]) 
-    # ans1_win_count = ans1_win_idx_overall.sum()
-    # ans2_win_count = ans2_win_idx_overall.sum()
-
-    # meta_info['ans1_win_count'] = ans1_win_count.tolist()
-    # meta_info['ans2_win_count'] = ans2_win_count.tolist()
-
     wraped_info['Meta_Info'] = meta_info
     wraped_info['data'] = qa_jsons
 
-    if args.api_model == 'gpt-4':
+    if 'gpt-4' in args.api_model:
         output_review_file = args.wraped_file.strip('.json') + '_reviews_gpt4.json'
-    else:
-        output_review_file = args.wraped_file.strip('.json') + '_reviews.json'
+    elif 'gpt-3.5' in args.api_model:
+        output_review_file = args.wraped_file.strip('.json') + '_reviews_gpt3.5.json'
     with open(f"{output_review_file}", "w") as f:
         json.dump(wraped_info, f, indent=4)
         pass

evaluation/generation/eval_generation_wrap.py

@@ -3,19 +3,20 @@
 import argparse
 
 def parse_args():
-    parser = argparse.ArgumentParser(description="Finetune a transformers model on a summarization task")
+    parser = argparse.ArgumentParser()
     parser.add_argument("--dataset_name", type=str, default='', help="The name of the dataset to use.")
     parser.add_argument("--fname1", type=str, default='')
     parser.add_argument("--fname2", type=str, default='')
     parser.add_argument("--save_name", type=str, default='') # a vs b format
+    parser.add_argument("--max_length", type=int, default=1024)
 
     args = parser.parse_args()
     return args
 
 args = parse_args()
 
 print('args.dataset_name',args.dataset_name)
-f_name = args.dataset_name+'_0_1024.json'
+f_name = args.dataset_name+'_'+str(args.max_length)+'.json'
 args.fname1 = os.path.join(args.fname1,f_name)
 args.fname2 = os.path.join(args.fname2,f_name)
 print('args.fname1',args.fname1)
@@ -37,8 +38,6 @@ def parse_args():
     prompt_key = 'instruction'
 elif(args.dataset_name=="wizardlm"):
     prompt_key = 'Instruction'
-elif(args.dataset_name=="truthfulqa"):
-    prompt_key = 'Question'
 elif(args.dataset_name=="lima"):
     prompt_key = 'conversations'
 

evaluation/generation/review_eval_score.py

@@ -2,20 +2,22 @@
 import json
 import numpy as np
 import matplotlib.pyplot as plt
+import argparse
 
-review_home_path = 'logs/xxx1-VSxxx2'
-datasets = ['Vicuna','Koala','WizardLM','SInstruct','LIMA']
-# datasets = ['Vicuna','Koala','WizardLM','SInstruct']
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--review_home_path", type=str, default='', help="home path that save the reviews")
+    parser.add_argument('--task_list', nargs='+', type=str, default=['Vicuna','Koala','WizardLM','SInstruct','LIMA'])
+    parser.add_argument("--key1", type=str, default='Model1')
+    parser.add_argument("--key2", type=str, default='Model2')
+    parser.add_argument("--save_name", type=str, default='result') # a vs b format
+    parser.add_argument("--max_length", type=int, default=1024)
+    parser.add_argument("--api_model",type=str,default='gpt-3.5-turbo')
 
-save_name = review_home_path.split('/')[-1]
+    args = parser.parse_args()
+    return args
 
-key1, key2 = save_name.split('-VS-')[0],save_name.split('-VS-')[1]
-title_ = save_name
-
-# key1 = 'Pre-Experienced Selected by Alpaca (15%)'
-# # key2 = 'WizardLM' + r"$^*$" + '(100%)'
-# key2 = 'Alpaca (100%)'
-# title_ = key1 + ' vs. ' + key2 
+args = parse_args()
 
 
 def survey(results, category_names):
@@ -84,20 +86,22 @@ def get_scores_all(pure_data):
             score3 += 1
     return [score1, score2, score3]
 
-for dataset in datasets:
+for dataset in args.task_list:
     review_path = ''
-    for root, ds, fs in os.walk(review_home_path):
+    for root, ds, fs in os.walk(args.review_home_path):
             for f in fs:
-                if 'reviews' in f and f.endswith('.json') and dataset.lower() in f:
-                    review_path = os.path.join(root, f)
-                # if 'reviews_gpt4' in f and f.endswith('.json') and dataset.lower() in f:
-                #     review_path = os.path.join(root, f)
+                if 'gpt-3.5' in args.api_model:
+                    if 'reviews_gpt3.5' in f and f.endswith('.json') and dataset.lower() in f:
+                        review_path = os.path.join(root, f)
+                elif 'gpt-4' in args.api_model:
+                    if 'reviews_gpt4' in f and f.endswith('.json') and dataset.lower() in f:
+                        review_path = os.path.join(root, f)
     with open(review_path, "r") as f:
         review_data = json.load(f)
     pure_data = review_data['data']
 
     scores = get_scores_all(pure_data)
-    category_names = [f"{key1} wins", "Tie", f"{key2} wins"]
+    category_names = [f"{args.key1} wins", "Tie", f"{args.key2} wins"]
     results[dataset] = scores
 
 def cal_rate(results):
@@ -112,18 +116,6 @@ def cal_rate(results):
 
 cal_rate(results)
 survey(results, category_names)
-img_path = os.path.join(review_home_path,save_name+'.jpg')
-plt.title(title_)
+img_path = os.path.join(args.review_home_path,args.save_name+'.jpg')
 plt.savefig(img_path)
 pass
-
-# from PIL import Image
-# def crop_edges(image_path, left, upper, right, lower):
-#     with Image.open(image_path) as img:
-#         width, height = img.size
-#         cropped = img.crop((left, upper, width - right, height - lower))
-#         return cropped
-# cropped_img = crop_edges(img_path,45,45,45,45)
-# cropped_img.save(img_path)
-# pass
-