notebook/Inferring with BERT-like fine-tuned models-no-pipeline.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "7a0c8c7e-d051-4ac1-8666-203508902b0b",
   "metadata": {},
   "source": [
    "# Inferring with the models we fine-tuned\n",
    "## 1. Load model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "25ef2c5d-8d3f-4e01-98d3-65cce754af06",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_finetuned = \"xlm-roberta-large_text-mine\"\n",
    "pretrained = \"xlm-roberta-large\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c543f71c",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_finetuned = \"camembert/camembert-large_text-mine\"\n",
    "pretrained = \"camembert/camembert-large\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a43218bd-30c4-4481-9186-11f9b2cca252",
   "metadata": {},
   "outputs": [],
   "source": [
    "import transformers\n",
    "from transformers import pipeline\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a3a41a4e-20f6-456e-8af3-c45a55758e87",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import AutoTokenizer\n",
    "tokenizer = AutoTokenizer.from_pretrained(pretrained, is_split_into_words=True, model_max_length=512)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0fbb2340-21d6-4de7-864f-df76b09a0f2d",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import AutoModelForTokenClassification\n",
    "\n",
    "label_correspondance = {\n",
    "    0: \"aucun\",\n",
    "    1: \"geogFeat\",\n",
    "    2: \"geogFeat geogName\",\n",
    "    3: \"geogName\",\n",
    "    4: \"name\",\n",
    "    5: \"name geogName\"\n",
    "}\n",
    "\n",
    "id2label = label_correspondance\n",
    "label2id = {v: k for k, v in id2label.items()}\n",
    "\n",
    "model = AutoModelForTokenClassification.from_pretrained(\n",
    "    f\"./../models_finetuned/{model_finetuned}\",\n",
    "    id2label=id2label,\n",
    "    label2id=label2id,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "66453496-dd54-4de4-afd9-f08ae7c15f54",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "\n",
    "def textmine_pipeline(text, model=model, tokenizer=tokenizer):\n",
    "    tokens = tokenizer.encode_plus(text, is_split_into_words=True, return_tensors=\"pt\")\n",
    "    predictions = model(**tokens)\n",
    "    logits = predictions.logits\n",
    "    logits = logits\n",
    "    with torch.no_grad():\n",
    "        p = np.argmax(logits, axis=-1)\n",
    "    predictions_aggregated = []\n",
    "    subtoken_old = np.nan\n",
    "    for i, subtoken in enumerate(tokens.word_ids()[1:-1]):\n",
    "        if subtoken != subtoken_old:\n",
    "            predictions_aggregated.append(label_correspondance[p[0][i].item()])\n",
    "        subtoken_old = subtoken\n",
    "    return predictions_aggregated"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c7cddfb9-b694-4615-8484-267c4336d0be",
   "metadata": {},
   "outputs": [],
   "source": [
    "t = textmine_pipeline([\"Le\", \"port\", \"de\", \"plaisance\", \"de\", \"Cavalaire\", \"se\", \"trouve\", \"à\"])\n",
    "t"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "e562cc68-8b77-40ac-9a17-faa9bf89fdd0",
   "metadata": {},
   "source": [
    "## 2. Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8dad2cf0-0772-4e8e-a032-c86b3492b048",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_raw = pd.read_csv(\"./../data/test.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9c9c0051-b54f-4b59-9e83-5b94cd2506d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Rebuilding the sentences and making list of labels\n",
    "current_sentence = []\n",
    "list_sentences = []\n",
    "\n",
    "last_row = df_raw.iloc[-1][\"Id\"]\n",
    "# Iterate over each row in the DataFrame\n",
    "for index, row in df_raw.iterrows():\n",
    "    token = row['Token'].replace('\"','')\n",
    "    current_sentence.append(token)\n",
    "    \n",
    "    # Check if the current token ends with a period\n",
    "    if token.endswith('.') or index == last_row :       \n",
    "        # Update the 'Sentence' column with the rebuilt sentence\n",
    "        list_sentences.append(current_sentence)\n",
    "        # Reset the current sentence\n",
    "        current_sentence = []\n",
    "\n",
    "\n",
    "df = pd.DataFrame()\n",
    "df[\"tokens\"] = list_sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e6361271-a517-49ef-9f12-c21ee99021c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "df"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "32a93b98-93db-440d-9492-4324a1ee2d3e",
   "metadata": {},
   "source": [
    "## 3. Infer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ed88d36d-7d07-4718-875d-c5fb1ab1dd36",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm.notebook import tqdm\n",
    "tqdm.pandas()\n",
    "\n",
    "df[\"predicted\"] = df[\"tokens\"].progress_apply(textmine_pipeline)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "df1fecbc-03f2-4dcc-be9c-3fba9937cfdf",
   "metadata": {},
   "outputs": [],
   "source": [
    "df"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "89c8a011-5544-4cba-8150-428a814a7c84",
   "metadata": {},
   "source": [
    "## 4. Saving files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b5a378ae-8b40-4642-8057-cf73f293323a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_lists(row):\n",
    "    tokens = row['tokens']\n",
    "    predicted = row['predicted']\n",
    "    return list(zip(tokens, predicted))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2118f4c1-294c-4488-b44a-da5927f08d61",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df_test = df.apply(split_lists, axis=1)\n",
    "\n",
    "list_tokens = []\n",
    "for liste in df_test.tolist():\n",
    "    list_tokens.extend(liste)\n",
    "\n",
    "df_test2 = pd.DataFrame(list_tokens)\n",
    "df_test2 = df_test2.reset_index()\n",
    "df_test2 = df_test2.rename(columns={\"Index\": \"Id\", 0: \"Token\", 1: \"Label\"})\n",
    "\n",
    "df_test2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "134acdac-f0ee-4b51-8dc5-c3d5cd094126",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test2.to_csv(f\"./../output/{model_finetuned.replace('/', '_')}_nopipeline_test.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dab08bb8-89b8-4f28-96f2-53aff01f9d25",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_test2[[\"Id\", \"Label\"]].to_csv(f\"./../output/submission_remy_2.csv\", index=False)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "88c38911-0d60-4895-9953-469f4e4343c1",
   "metadata": {},
   "source": [
    "## 6 Others verification test on training date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "508bb212-d700-4684-afc5-49d8ad19038a",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_raw_train = pd.read_csv(\"./../data/train.csv\")\n",
    "# Rebuilding the sentences and making list of labels\n",
    "current_sentence = []\n",
    "list_sentences = []\n",
    "\n",
    "# Iterate over each row in the DataFrame\n",
    "for index, row in df_raw_train.iterrows():\n",
    "    token = row['Token'].replace('\"','')\n",
    "    current_sentence.append(token)\n",
    "    \n",
    "    # Check if the current token ends with a period\n",
    "    if token.endswith('.'):       \n",
    "        # Update the 'Sentence' column with the rebuilt sentence\n",
    "        list_sentences.append(current_sentence)\n",
    "        # Reset the current sentence\n",
    "        current_sentence = []\n",
    "\n",
    "\n",
    "df_train = pd.DataFrame()\n",
    "df_train[\"tokens\"] = list_sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "04aec845-d026-482d-a25f-b37eeac887b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "95613090-baf7-4f0c-891c-96fff4a53357",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train[\"predicted\"] = df_train[\"tokens\"].progress_apply(nlp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "692f4443-d7be-49a5-9325-c49030e9d844",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train[\"predicted_post\"] = df_train.progress_apply(lambda x: geogName(x.tokens, x.predicted), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8efe3f78-2d6f-4ac0-8acb-bbe55b08bff2",
   "metadata": {},
   "outputs": [],
   "source": [
    "id_number = 0\n",
    "list_tokens = []\n",
    "# for sentence in df_train[\"predicted\"]:\n",
    "for sentence in df_train[\"predicted_post\"]:\n",
    "    # print(sentence)\n",
    "    for ner_results in sentence:\n",
    "        results_dict = {\n",
    "            \"Id\": id_number,\n",
    "            \"Token\": f'\"\"\"{ner_results[0][\"word\"]}\"\"\"',\n",
    "            \"Label\": ner_results[0][\"entity_group\"]\n",
    "        }\n",
    "        list_tokens.append(results_dict)\n",
    "        id_number = id_number + 1\n",
    "\n",
    "df_test_train = pd.DataFrame(list_tokens)\n",
    "df_test_train.to_csv(f\"./../output/{model_finetuned.replace('/', '_')}_train.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ca018141-6a4b-4651-bc0a-d8df7f65306c",
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp([\"Le\", \"port\", \"de\", \"plaisance\", \"de\", \"Cavalaire\", \"se\", \"trouve\", \"à\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "58b73ef4-56d5-48ea-bc66-0e072eed2901",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(df_train.iloc[1][\"predicted\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "03f067ef-39bc-4bfa-a955-8dc03e7d8977",
   "metadata": {},
   "outputs": [],
   "source": [
    "def cleaning_label(label):\n",
    "    if label == \"geogName name\":\n",
    "        return \"name geogName\"\n",
    "    elif label == \"geogFeat geogName geogName\":\n",
    "        return \"geogFeat geogName\"\n",
    "    elif label == \"name geogName geogName\":\n",
    "        return \"name geogName\"\n",
    "    elif label == \"geogName geogName\":\n",
    "        return \"geogName\"\n",
    "    elif label == \"geogName geogFeat geogName\":\n",
    "        return \"geogFeat geogName\"\n",
    "    elif label == \"geogName geogFeat\":\n",
    "        return \"geogFeat geogName\"\n",
    "    elif label == \"geogName geogName name\":\n",
    "        return \"name geogName\"\n",
    "    elif label == \"geogName name geogName\":\n",
    "        return \"name geogName\"\n",
    "    elif label == \"geogFeat geogName geogName geogName\":\n",
    "        return \"geogFeat geogName\"\n",
    "    elif label == \"geogName geogName name geogName\":\n",
    "        return \"name geogName\"\n",
    "    else:\n",
    "        return label\n",
    "\n",
    "df_raw_train[\"Label\"] = df_raw_train[\"Label\"].apply(cleaning_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1782a4af-9d7e-4acc-9a04-40b4e1b24fc3",
   "metadata": {},
   "outputs": [],
   "source": [
    "label_correspondance = {\n",
    "    0: \"aucun\",\n",
    "    1: \"geogFeat\",\n",
    "    2: \"geogFeat geogName\",\n",
    "    3: \"geogName\",\n",
    "    4: \"name\",\n",
    "    5: \"name geogName\"\n",
    "}\n",
    "\n",
    "def label_id(desired_value):\n",
    "    # reverse Correspondance between label value and their index\n",
    "    for key, value in label_correspondance.items():\n",
    "        if value == desired_value:\n",
    "            return key\n",
    "\n",
    "def list_of_prediction(df_raw):\n",
    "    # Rebuilding the sentences and making list of labels\n",
    "    current_sentence = []\n",
    "    current_sentence_label = []\n",
    "    current_sentence_ner_tag = []\n",
    "    list_sentences = []\n",
    "    list_sentences_label = []\n",
    "    list_sentences_ner_tags = []\n",
    "\n",
    "    # Iterate over each row in the DataFrame\n",
    "    for index, row in df_raw.iterrows():\n",
    "        token = row['Token'].replace('\"','')\n",
    "        label = row['Label']\n",
    "        ner_tag = label_id(row[\"Label\"])\n",
    "        current_sentence.append(token)\n",
    "        current_sentence_label.append(label)\n",
    "        current_sentence_ner_tag.append(ner_tag)\n",
    "\n",
    "        # Check if the current token ends with a period\n",
    "        if token.endswith('.'):       \n",
    "            # Update the 'Sentence' column with the rebuilt sentence\n",
    "            list_sentences.append(current_sentence)\n",
    "            list_sentences_label.append(current_sentence_label)\n",
    "            list_sentences_ner_tags.append(current_sentence_ner_tag)\n",
    "\n",
    "            # Reset the current sentence\n",
    "            current_sentence = []\n",
    "            current_sentence_label = []\n",
    "            current_sentence_ner_tag = []\n",
    "\n",
    "    df = pd.DataFrame()\n",
    "    df[\"tokens\"] = list_sentences\n",
    "    df[\"labels\"] = list_sentences_label\n",
    "    df[\"ner_tags\"] = list_sentences_ner_tags\n",
    "    return df\n",
    "\n",
    "df_predicted_train = list_of_prediction(df_test_train)\n",
    "df_true_train = list_of_prediction(df_raw_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ef0d75ba-95c0-4fe4-af93-194e44b63af3",
   "metadata": {},
   "outputs": [],
   "source": [
    "from seqeval.metrics import classification_report\n",
    "from seqeval.metrics import f1_score\n",
    "\n",
    "label_correspondance = {\n",
    "    0: \"aucun\",\n",
    "    1: \"geogFeat\",\n",
    "    2: \"geogFeat geogName\",\n",
    "    3: \"geogName\",\n",
    "    4: \"name\",\n",
    "    5: \"name geogName\"\n",
    "}\n",
    "true_labels = [[label_names[l] for l in label if l != -100] for label in df_predicted_train[\"ner_tags\"]]\n",
    "true_predictions = [[label_names[l] for l in label if l != -100] for label in df_true_train[\"ner_tags\"]]\n",
    "\n",
    "print(classification_report(df_true_train[\"labels\"], df_predicted_train[\"labels\"]))\n",
    "all_metrics = metric.compute(predictions=true_predictions, references=true_labels)\n",
    "# print(all_metrics)\n",
    "all_metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4a475a84-69da-469f-8f7b-d2981d20b484",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_true_train[\"labels\"].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3df46be1-6a0c-4587-b06a-8acdc201c642",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_predicted_train[\"labels\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d812a02d-33e0-4f71-853a-6d37dec84bbb",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(pretrained, model_max_length=512)\n",
    "nlp_misc = pipeline(\"ner\", model=model, tokenizer=tokenizer,  device=0, framework=\"pt\", aggregation_strategy=\"first\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bbc308fa-4f39-451f-a24d-c05a87f6295b",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokens= [\"Le\", \"port\", \"de\", \"plaisance\", \"de\", \"Cavalaire\", \"se\", \"trouve\", \"à\"]\n",
    "nlp_misc(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "679e71ea-e06b-4d7a-9aa5-6f10a9bd0c4f",
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp([\"Le\", \"port\", \"de\", \"plaisance\", \"de\", \"Cavalaire\", \"se\", \"trouve\", \"à\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "41f74aee-6525-47ba-b804-19d95bf98b14",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokens = tokenizer([\"Le\", \"port\", \"de\", \"plaisance\", \"de\", \"Cavalaire\", \"se\", \"trouve\", \"à\"], is_split_into_words=True, return_tensors=\"pt\")\n",
    "tokens.to(0)\n",
    "# tokens = tokenizer(\"le port du lavandou\")\n",
    "print(tokens)\n",
    "# model.to(0)\n",
    "predictions = model(**tokens)\n",
    "print(predictions)\n",
    "import numpy as np\n",
    "import torch\n",
    "logits = predictions.logits\n",
    "logits = logits.cpu()\n",
    "with torch.no_grad():\n",
    "    p = np.argmax(logits, axis=-1)\n",
    "print(p)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6f82db01-19b5-452e-b382-9f519bc95212",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}