Upgrade to HF transformers 4.3.2

nbs/01-gpt2-with-value-head.ipynb

@@ -42,7 +42,7 @@
     "# export\n",
     "\n",
     "from transformers import GPT2LMHeadModel, GPT2Tokenizer, GPT2Model, GPT2PreTrainedModel\n",
-    "from transformers.modeling_utils import top_k_top_p_filtering\n",
+    "from transformers import top_k_top_p_filtering\n",
     "from torch import nn\n",
     "from torch.nn import Identity\n",
     "import torch.nn.functional as F\n",
@@ -129,7 +129,7 @@
     "    def forward(\n",
     "        self,\n",
     "        input_ids=None,\n",
-    "        past=None,\n",
+    "        past_key_values=None,\n",
     "        attention_mask=None,\n",
     "        token_type_ids=None,\n",
     "        position_ids=None,\n",
@@ -142,7 +142,7 @@
     "       \n",
     "        transformer_outputs = self.transformer(\n",
     "            input_ids,\n",
-    "            past=past,\n",
+    "            past_key_values=past_key_values,\n",
     "            attention_mask=attention_mask,\n",
     "            token_type_ids=token_type_ids,\n",
     "            position_ids=position_ids,\n",

requirements.txt

@@ -1,12 +1,11 @@
-
 -e .
 jupyterlab==2.0.1
 nbdev==0.2.16
 numpy==1.18.2
 pandas==1.0.3
-simpletransformers==0.21.4
-torch==1.4.0
+simpletransformers==0.60.9
+torch>=1.4.0
 tqdm==4.43.0
-transformers==2.6.0
-wandb==0.8.35
-matplotlib==3.2.1
+transformers==4.3.2
+wandb==0.10.20
+matplotlib==3.2.1

settings.ini

@@ -14,7 +14,7 @@ language = English
 custom_sidebar = False
 license = apache2
 status = 2
-requirements = torch>=1.4.0 transformers==2.6.0 numpy>=1.18.2
+requirements = torch>=1.4.0 transformers==4.3.2 numpy>=1.18.2
 nbs_path = ./nbs/
 doc_path = docs
 doc_host = https://lvwerra.github.io

trl/gpt2.py

@@ -5,7 +5,7 @@
 # Cell
 
 from transformers import GPT2LMHeadModel, GPT2Tokenizer, GPT2Model, GPT2PreTrainedModel
-from transformers.modeling_utils import top_k_top_p_filtering
+from transformers import top_k_top_p_filtering
 from torch import nn
 from torch.nn import Identity
 import torch.nn.functional as F
@@ -78,7 +78,7 @@ def detach_value_head(self):
     def forward(
         self,
         input_ids=None,
-        past=None,
+        past_key_values=None,
         attention_mask=None,
         token_type_ids=None,
         position_ids=None,
@@ -91,7 +91,7 @@ def forward(
 
         transformer_outputs = self.transformer(
             input_ids,
-            past=past,
+            past_key_values=past_key_values,
             attention_mask=attention_mask,
             token_type_ids=token_type_ids,
             position_ids=position_ids,