07-RoBERTa on NSP using Trainer.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 示例1\n",
    "\n",
    "https://github.com/huggingface/transformers/issues/7336"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import (BertForNextSentencePrediction,\n",
    "                          BertTokenizer,\n",
    "                          RobertaModel, RobertaTokenizer, Trainer,\n",
    "                          TrainingArguments)\n",
    "from transformers.data.datasets.language_modeling import TextDatasetForNextSentencePrediction\n",
    "from transformers.data.data_collator import DataCollatorForNextSentencePrediction\n",
    "from argparse import ArgumentParser\n",
    "\n",
    "\n",
    "def parse_args():\n",
    "    parser = ArgumentParser(\"Fine-tune RoBERTa in Next Sentence Prediction.\")\n",
    "    parser.add_argument(\"-m\", \"--model_path\", dest=\"model_path\", required=True, help=\"Path to RoBERTa model.\")\n",
    "    parser.add_argument(\"-o\", \"--output_path\", dest=\"output_path\", required=True, help=\"Path to directory of fine-tuned model.\")\n",
    "    parser.add_argument(\"-d\", \"--dataset_path\", dest=\"dataset_path\", required=True, help=\"Path to dataset.\")\n",
    "    args = parser.parse_args()\n",
    "    return args\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    args = parse_args()\n",
    "    tokenizer = RobertaTokenizer.from_pretrained(args.model_path)\n",
    "    finetune_model = BertForNextSentencePrediction.from_pretrained(args.model_path)\n",
    "\n",
    "    training_args = TrainingArguments(\n",
    "        output_dir=args.output_path,\n",
    "        num_train_epochs=3,\n",
    "        per_device_train_batch_size=1,\n",
    "        per_device_eval_batch_size=1,\n",
    "        warmup_steps=500,\n",
    "        weight_decay=0.01,\n",
    "        logging_dir='./logs',\n",
    "    )\n",
    "\n",
    "    data_collator = DataCollatorForNextSentencePrediction(\n",
    "        tokenizer=tokenizer,\n",
    "        mlm=False,\n",
    "        block_size=512,\n",
    "        nsp_probability=0.5,\n",
    "      )\n",
    "\n",
    "    train_dataset = TextDatasetForNextSentencePrediction(\n",
    "        tokenizer=tokenizer,\n",
    "        file_path=args.dataset_path,\n",
    "        block_size=512,\n",
    "    )\n",
    "\n",
    "    trainer = Trainer(\n",
    "        model=finetune_model,\n",
    "        args=training_args,\n",
    "        train_dataset=train_dataset,\n",
    "        data_collator=data_collator,\n",
    "    )\n",
    "\n",
    "    trainer.train()\n",
    "    trainer.save_model(args.output_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 示例2 BERT-SCRATCH.ipynb\n",
    "- https://discuss.huggingface.co/t/training-bert-from-scratch-mlm-nsp-on-a-new-domain/2075\n",
    "- https://colab.research.google.com/drive/12NHfXeUBo7RBl3Kffa-715i-Zpd0MOzP?usp=sharing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 示例3 mlm-and-nsp\n",
    "\n",
    "https://stackoverflow.com/questions/65646925/how-to-train-bert-from-scratch-on-a-new-domain-for-both-mlm-and-nsp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}