evaluation_rouge_bleu.ipynb

{
 "cells": [
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": [
    "## LLM results evaluation\n",
    "\n",
    "This notebook presents evaluation techniques for LLMs output such as ROUGE, BLEU and METEOR. \n",
    "All presented metrics are based on comparison of words in AI generated text with human provided result text. "
   ],
   "id": "5fd147636a0993a3"
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "### ROUGE",
   "id": "40c2fdc13251d3f4"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-09-02T06:55:01.141617Z",
     "start_time": "2024-09-02T06:54:41.011801Z"
    }
   },
   "cell_type": "code",
   "source": [
    "!pip install evaluate\n",
    "!pip install rouge_score\n",
    "!pip install bleu\n",
    "!pip install meteor"
   ],
   "id": "580d1b53c94216f8",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: evaluate in /home/michal/anaconda3/lib/python3.11/site-packages (0.4.2)\r\n",
      "Requirement already satisfied: datasets>=2.0.0 in /home/michal/anaconda3/lib/python3.11/site-packages (from evaluate) (2.21.0)\r\n",
      "Requirement already satisfied: numpy>=1.17 in /home/michal/anaconda3/lib/python3.11/site-packages (from evaluate) (1.26.4)\r\n",
      "Requirement already satisfied: dill in /home/michal/anaconda3/lib/python3.11/site-packages (from evaluate) (0.3.8)\r\n",
      "Requirement already satisfied: pandas in /home/michal/anaconda3/lib/python3.11/site-packages (from evaluate) (2.2.2)\r\n",
      "Requirement already satisfied: requests>=2.19.0 in /home/michal/anaconda3/lib/python3.11/site-packages (from evaluate) (2.32.3)\r\n",
      "Requirement already satisfied: tqdm>=4.62.1 in /home/michal/anaconda3/lib/python3.11/site-packages (from evaluate) (4.66.5)\r\n",
      "Requirement already satisfied: xxhash in /home/michal/anaconda3/lib/python3.11/site-packages (from evaluate) (3.5.0)\r\n",
      "Requirement already satisfied: multiprocess in /home/michal/anaconda3/lib/python3.11/site-packages (from evaluate) (0.70.16)\r\n",
      "Requirement already satisfied: fsspec>=2021.05.0 in /home/michal/anaconda3/lib/python3.11/site-packages (from fsspec[http]>=2021.05.0->evaluate) (2023.10.0)\r\n",
      "Requirement already satisfied: huggingface-hub>=0.7.0 in /home/michal/anaconda3/lib/python3.11/site-packages (from evaluate) (0.22.2)\r\n",
      "Requirement already satisfied: packaging in /home/michal/anaconda3/lib/python3.11/site-packages (from evaluate) (23.2)\r\n",
      "Requirement already satisfied: filelock in /home/michal/anaconda3/lib/python3.11/site-packages (from datasets>=2.0.0->evaluate) (3.13.1)\r\n",
      "Requirement already satisfied: pyarrow>=15.0.0 in /home/michal/anaconda3/lib/python3.11/site-packages (from datasets>=2.0.0->evaluate) (15.0.2)\r\n",
      "Requirement already satisfied: aiohttp in /home/michal/anaconda3/lib/python3.11/site-packages (from datasets>=2.0.0->evaluate) (3.9.3)\r\n",
      "Requirement already satisfied: pyyaml>=5.1 in /home/michal/anaconda3/lib/python3.11/site-packages (from datasets>=2.0.0->evaluate) (6.0.1)\r\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /home/michal/anaconda3/lib/python3.11/site-packages (from huggingface-hub>=0.7.0->evaluate) (4.11.0)\r\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /home/michal/anaconda3/lib/python3.11/site-packages (from requests>=2.19.0->evaluate) (3.3.2)\r\n",
      "Requirement already satisfied: idna<4,>=2.5 in /home/michal/anaconda3/lib/python3.11/site-packages (from requests>=2.19.0->evaluate) (3.4)\r\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /home/michal/anaconda3/lib/python3.11/site-packages (from requests>=2.19.0->evaluate) (2.0.7)\r\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/michal/anaconda3/lib/python3.11/site-packages (from requests>=2.19.0->evaluate) (2024.7.4)\r\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /home/michal/anaconda3/lib/python3.11/site-packages (from pandas->evaluate) (2.8.2)\r\n",
      "Requirement already satisfied: pytz>=2020.1 in /home/michal/anaconda3/lib/python3.11/site-packages (from pandas->evaluate) (2023.3.post1)\r\n",
      "Requirement already satisfied: tzdata>=2022.7 in /home/michal/anaconda3/lib/python3.11/site-packages (from pandas->evaluate) (2023.3)\r\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /home/michal/anaconda3/lib/python3.11/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.2.0)\r\n",
      "Requirement already satisfied: attrs>=17.3.0 in /home/michal/anaconda3/lib/python3.11/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (24.2.0)\r\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /home/michal/anaconda3/lib/python3.11/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.4.0)\r\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /home/michal/anaconda3/lib/python3.11/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (6.0.4)\r\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /home/michal/anaconda3/lib/python3.11/site-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.9.3)\r\n",
      "Requirement already satisfied: six>=1.5 in /home/michal/anaconda3/lib/python3.11/site-packages (from python-dateutil>=2.8.2->pandas->evaluate) (1.16.0)\r\n",
      "Requirement already satisfied: rouge_score in /home/michal/anaconda3/lib/python3.11/site-packages (0.1.2)\r\n",
      "Requirement already satisfied: absl-py in /home/michal/anaconda3/lib/python3.11/site-packages (from rouge_score) (2.1.0)\r\n",
      "Requirement already satisfied: nltk in /home/michal/anaconda3/lib/python3.11/site-packages (from rouge_score) (3.8.1)\r\n",
      "Requirement already satisfied: numpy in /home/michal/anaconda3/lib/python3.11/site-packages (from rouge_score) (1.26.4)\r\n",
      "Requirement already satisfied: six>=1.14.0 in /home/michal/anaconda3/lib/python3.11/site-packages (from rouge_score) (1.16.0)\r\n",
      "Requirement already satisfied: click in /home/michal/anaconda3/lib/python3.11/site-packages (from nltk->rouge_score) (8.1.7)\r\n",
      "Requirement already satisfied: joblib in /home/michal/anaconda3/lib/python3.11/site-packages (from nltk->rouge_score) (1.2.0)\r\n",
      "Requirement already satisfied: regex>=2021.8.3 in /home/michal/anaconda3/lib/python3.11/site-packages (from nltk->rouge_score) (2023.10.3)\r\n",
      "Requirement already satisfied: tqdm in /home/michal/anaconda3/lib/python3.11/site-packages (from nltk->rouge_score) (4.66.5)\r\n",
      "Collecting bleu\r\n",
      "  Downloading bleu-0.3.tar.gz (5.2 kB)\r\n",
      "  Preparing metadata (setup.py) ... \u001B[?25ldone\r\n",
      "\u001B[?25hCollecting efficiency (from bleu)\r\n",
      "  Downloading efficiency-2.0-py3-none-any.whl.metadata (2.5 kB)\r\n",
      "Requirement already satisfied: pandas in /home/michal/anaconda3/lib/python3.11/site-packages (from efficiency->bleu) (2.2.2)\r\n",
      "Requirement already satisfied: numpy in /home/michal/anaconda3/lib/python3.11/site-packages (from efficiency->bleu) (1.26.4)\r\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /home/michal/anaconda3/lib/python3.11/site-packages (from pandas->efficiency->bleu) (2.8.2)\r\n",
      "Requirement already satisfied: pytz>=2020.1 in /home/michal/anaconda3/lib/python3.11/site-packages (from pandas->efficiency->bleu) (2023.3.post1)\r\n",
      "Requirement already satisfied: tzdata>=2022.7 in /home/michal/anaconda3/lib/python3.11/site-packages (from pandas->efficiency->bleu) (2023.3)\r\n",
      "Requirement already satisfied: six>=1.5 in /home/michal/anaconda3/lib/python3.11/site-packages (from python-dateutil>=2.8.2->pandas->efficiency->bleu) (1.16.0)\r\n",
      "Downloading efficiency-2.0-py3-none-any.whl (32 kB)\r\n",
      "Building wheels for collected packages: bleu\r\n",
      "  Building wheel for bleu (setup.py) ... \u001B[?25ldone\r\n",
      "\u001B[?25h  Created wheel for bleu: filename=bleu-0.3-py3-none-any.whl size=5781 sha256=613658e672d5b2e75fbb40cb2b198e0d921b3e2cca7210d7124f7772275fdc3e\r\n",
      "  Stored in directory: /home/michal/.cache/pip/wheels/4e/9f/09/3d45ccd4ce42bc796c1f0b960037e30f40b953458d3868b6f3\r\n",
      "Successfully built bleu\r\n",
      "Installing collected packages: efficiency, bleu\r\n",
      "Successfully installed bleu-0.3 efficiency-2.0\r\n",
      "Requirement already satisfied: meteor in /home/michal/anaconda3/lib/python3.11/site-packages (2.0.15)\r\n",
      "Requirement already satisfied: bgzip<0.6.0,>=0.5.0 in /home/michal/anaconda3/lib/python3.11/site-packages (from meteor) (0.5.0)\r\n",
      "Requirement already satisfied: biom-format<3.0.0,>=2.1.15 in /home/michal/anaconda3/lib/python3.11/site-packages (from meteor) (2.1.16)\r\n",
      "Requirement already satisfied: cogent3<2025.0.0,>=2024.2.5a1 in /home/michal/anaconda3/lib/python3.11/site-packages (from meteor) (2024.7.19a5)\r\n",
      "Requirement already satisfied: ete3<4.0.0,>=3.1.3 in /home/michal/anaconda3/lib/python3.11/site-packages (from meteor) (3.1.3)\r\n",
      "Requirement already satisfied: packaging<24.0,>=23.2 in /home/michal/anaconda3/lib/python3.11/site-packages (from meteor) (23.2)\r\n",
      "Requirement already satisfied: pandas<3.0.0,>=2.1.2 in /home/michal/anaconda3/lib/python3.11/site-packages (from meteor) (2.2.2)\r\n",
      "Requirement already satisfied: pyarrow<16.0.0,>=15.0.0 in /home/michal/anaconda3/lib/python3.11/site-packages (from meteor) (15.0.2)\r\n",
      "Requirement already satisfied: pysam<0.23.0,>=0.22.0 in /home/michal/anaconda3/lib/python3.11/site-packages (from meteor) (0.22.1)\r\n",
      "Requirement already satisfied: click in /home/michal/anaconda3/lib/python3.11/site-packages (from biom-format<3.0.0,>=2.1.15->meteor) (8.1.7)\r\n",
      "Requirement already satisfied: numpy>=1.9.2 in /home/michal/anaconda3/lib/python3.11/site-packages (from biom-format<3.0.0,>=2.1.15->meteor) (1.26.4)\r\n",
      "Requirement already satisfied: scipy>=1.3.1 in /home/michal/anaconda3/lib/python3.11/site-packages (from biom-format<3.0.0,>=2.1.15->meteor) (1.11.4)\r\n",
      "Requirement already satisfied: h5py in /home/michal/anaconda3/lib/python3.11/site-packages (from biom-format<3.0.0,>=2.1.15->meteor) (3.11.0)\r\n",
      "Requirement already satisfied: chardet in /home/michal/anaconda3/lib/python3.11/site-packages (from cogent3<2025.0.0,>=2024.2.5a1->meteor) (4.0.0)\r\n",
      "Requirement already satisfied: numba>0.53 in /home/michal/anaconda3/lib/python3.11/site-packages (from cogent3<2025.0.0,>=2024.2.5a1->meteor) (0.59.1)\r\n",
      "Requirement already satisfied: scitrack in /home/michal/anaconda3/lib/python3.11/site-packages (from cogent3<2025.0.0,>=2024.2.5a1->meteor) (2021.5.3)\r\n",
      "Requirement already satisfied: stevedore in /home/michal/anaconda3/lib/python3.11/site-packages (from cogent3<2025.0.0,>=2024.2.5a1->meteor) (5.3.0)\r\n",
      "Requirement already satisfied: tqdm in /home/michal/anaconda3/lib/python3.11/site-packages (from cogent3<2025.0.0,>=2024.2.5a1->meteor) (4.66.5)\r\n",
      "Requirement already satisfied: typing_extensions in /home/michal/anaconda3/lib/python3.11/site-packages (from cogent3<2025.0.0,>=2024.2.5a1->meteor) (4.11.0)\r\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /home/michal/anaconda3/lib/python3.11/site-packages (from pandas<3.0.0,>=2.1.2->meteor) (2.8.2)\r\n",
      "Requirement already satisfied: pytz>=2020.1 in /home/michal/anaconda3/lib/python3.11/site-packages (from pandas<3.0.0,>=2.1.2->meteor) (2023.3.post1)\r\n",
      "Requirement already satisfied: tzdata>=2022.7 in /home/michal/anaconda3/lib/python3.11/site-packages (from pandas<3.0.0,>=2.1.2->meteor) (2023.3)\r\n",
      "Requirement already satisfied: llvmlite<0.43,>=0.42.0dev0 in /home/michal/anaconda3/lib/python3.11/site-packages (from numba>0.53->cogent3<2025.0.0,>=2024.2.5a1->meteor) (0.42.0)\r\n",
      "Requirement already satisfied: six>=1.5 in /home/michal/anaconda3/lib/python3.11/site-packages (from python-dateutil>=2.8.2->pandas<3.0.0,>=2.1.2->meteor) (1.16.0)\r\n",
      "Requirement already satisfied: pbr>=2.0.0 in /home/michal/anaconda3/lib/python3.11/site-packages (from stevedore->cogent3<2025.0.0,>=2024.2.5a1->meteor) (6.1.0)\r\n"
     ]
    }
   ],
   "execution_count": 10
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-09-02T11:36:42.962052Z",
     "start_time": "2024-09-02T11:36:41.412857Z"
    }
   },
   "cell_type": "code",
   "source": [
    "import evaluate\n",
    "rouge = evaluate.load('rouge')\n",
    "candidates = [\"Sky is pink and orange\",\"AI will send terminators to rule the world!\"]\n",
    "\n",
    "references = [[\"Sky sometimes is red and orange during sunset and dusk dawn\",\"Sky is usually blue, dark at night and sometimes yellow, pink and orange\"],\n",
    "              [\"AI will not send terminators. It will fly to different planet as soon as possible.\",\"AI will not send terminators. It will fly to different planet as soon as possible.\"]]\n",
    "results = rouge.compute(predictions=candidates, references=references)\n",
    "print(results)"
   ],
   "id": "ef1fe9fe270aca99",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'rouge1': 0.46739130434782616, 'rouge2': 0.16666666666666669, 'rougeL': 0.46739130434782616, 'rougeLsum': 0.46739130434782616}\n"
     ]
    }
   ],
   "execution_count": 16
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "### BLEU",
   "id": "1e6446d4095ce374"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-09-02T11:37:45.787021Z",
     "start_time": "2024-09-02T11:37:41.641683Z"
    }
   },
   "cell_type": "code",
   "source": [
    "bleu = evaluate.load('bleu')\n",
    "results = bleu.compute(predictions=candidates, references=references)\n",
    "print(results)"
   ],
   "id": "2450c7d62d21d4f1",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'bleu': 0.0, 'precisions': [0.6428571428571429, 0.25, 0.0, 0.0], 'brevity_penalty': 0.4557940183280172, 'length_ratio': 0.56, 'translation_length': 14, 'reference_length': 25}\n"
     ]
    }
   ],
   "execution_count": 18
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "### METEOR",
   "id": "519b9e74f1aac105"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-09-02T06:56:20.588520Z",
     "start_time": "2024-09-02T06:56:17.659839Z"
    }
   },
   "cell_type": "code",
   "source": [
    "meteor = evaluate.load('meteor')\n",
    "results = meteor.compute(predictions=candidates, references=references)\n",
    "print(results)"
   ],
   "id": "1713d6999040b334",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Downloading builder script:   0%|          | 0.00/6.93k [00:00<?, ?B/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "098f9a44ad984b4e95a8dfe7a82efc06"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package wordnet to /home/michal/nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n",
      "[nltk_data] Downloading package punkt to /home/michal/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package omw-1.4 to /home/michal/nltk_data...\n",
      "[nltk_data]   Package omw-1.4 is already up-to-date!\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'meteor': 0.5700319004727655}\n"
     ]
    }
   ],
   "execution_count": 13
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}