Add fine-tuned blip2 eval results for held-out set

yukw777 · Mar 4, 2024 · 02fef29 · 02fef29
1 parent f458fe5
commit 02fef29
Showing 1 changed file with 26 additions and 9 deletions.
diff --git a/figures/icl_eval_figures.ipynb b/figures/icl_eval_figures.ipynb
@@ -101,6 +101,22 @@
     "        \"BLEU\": [0.06718, 0.1507, 0.1992, 0.2373, 0.2834, 0.2931, 0.2913],\n",
     "        \"meta\": {\"shots\": shots, \"linestyle\": \"-\"},\n",
     "    },\n",
+    "    \"FT BLIP-2 OPT-2.7B\": {\n",
+    "        \"STS-CE\": [0.4719, 0.4743, 0.4316, 0.4454, 0.4557, 0.4699, 0.4749],\n",
+    "        \"STS-BE\": [0.5313, 0.5434, 0.5148, 0.5266, 0.5332, 0.5432, 0.5482],\n",
+    "        \"BERTScore-F1\": [0.6432, 0.6435, 0.6456, 0.6464, 0.6446, 0.652, 0.6368],\n",
+    "        \"ROUGE-L\": [0.5368, 0.5139, 0.519, 0.5243, 0.5205, 0.5319, 0.5151],\n",
+    "        \"BLEU\": [0.1683, 0.1564, 0.1923, 0.1952, 0.1852, 0.1947, 0.1696],\n",
+    "        \"meta\": {\"shots\": shots, \"linestyle\": \"--\"},\n",
+    "    },\n",
+    "    \"FT BLIP-2 Flan-T5-xl\": {\n",
+    "        \"STS-CE\": [0.4849, 0.4755, 0.4776, 0.4933, 0.4934, 0.4641, 0.3997],\n",
+    "        \"STS-BE\": [0.5486, 0.5432, 0.5502, 0.5565, 0.5489, 0.5152, 0.444],\n",
+    "        \"BERTScore-F1\": [0.6602, 0.64, 0.645, 0.6362, 0.3001, -1.083, -4.232],\n",
+    "        \"ROUGE-L\": [0.5404, 0.5037, 0.5242, 0.5303, 0.5261, 0.4947, 0.4253],\n",
+    "        \"BLEU\": [0.188, 0.155, 0.1941, 0.1923, 0.1907, 0.1791, 0.1473],\n",
+    "        \"meta\": {\"shots\": shots, \"linestyle\": \"--\"},\n",
+    "    },\n",
     "    \"BLIP-2 OPT-2.7B\": {\n",
     "        \"STS-CE\": [0.1505, 0.3586, 0.4012, 0.432, 0.4575, 0.4469, 0.3809],\n",
     "        \"STS-BE\": [0.3094, 0.4233, 0.4862, 0.5222, 0.5486, 0.5422, 0.4755],\n",
@@ -141,15 +157,16 @@
     "        ],\n",
     "        \"meta\": {\"shots\": shots, \"linestyle\": \"--\"},\n",
     "    },\n",
-    "    \"VideoMAE-base\": {\n",
-    "        \"STS-CE\": [0.3799],\n",
-    "        \"STS-BE\": [0.479],\n",
-    "        \"BERTScore-F1\": [0.6516],\n",
-    "        \"ROUGE-L\": [0.4375],\n",
-    "        \"BLEU\": [0.0987],\n",
-    "        \"meta\": {\"shots\": [0], \"linestyle\": \"--\"},\n",
-    "    },\n",
-    "    \"VideoMAE-huge\": {\n",
+    "    # \"VideoMAE-base\": {\n",
+    "    #     \"STS-CE\": [0.3799],\n",
+    "    #     \"STS-BE\": [0.479],\n",
+    "    #     \"BERTScore-F1\": [0.6516],\n",
+    "    #     \"ROUGE-L\": [0.4375],\n",
+    "    #     \"BLEU\": [0.0987],\n",
+    "    #     \"meta\": {\"shots\": [0], \"linestyle\": \"--\"},\n",
+    "    # },\n",
+    "    # VideoMAE-huge\n",
+    "    \"VideoMAE\": {\n",
     "        \"STS-CE\": [0.4405],\n",
     "        \"STS-BE\": [0.5282],\n",
     "        \"BERTScore-F1\": [0.645],\n",