analysis_datasets.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Software Name : mislabeled-benchmark\n",
    "# SPDX-FileCopyrightText: Copyright (c) Orange Innovation\n",
    "# SPDX-License-Identifier: MIT\n",
    "# \n",
    "# This software is distributed under the MIT license,\n",
    "# see the \"LICENSE.md\" file for more details\n",
    "# or https://github.com/Orange-OpenSource/mislabeled-benchmark/blob/master/LICENSE.md\n",
    "\n",
    "from datasets import get_weak_datasets, datasets_ranked_by_time\n",
    "import os\n",
    "\n",
    "datasets_folder = os.path.join(os.path.expanduser(\"~\"), \"datasets\")\n",
    "\n",
    "weak_datasets = get_weak_datasets(\n",
    "    cache_folder=datasets_folder,\n",
    "    corruption=\"weak\",\n",
    "    seed=1,\n",
    "    datasets=datasets_ranked_by_time,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import math\n",
    "import operator\n",
    "from functools import reduce\n",
    "\n",
    "import numpy as np\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "from mislabeled.preprocessing import WeakLabelEncoder\n",
    "\n",
    "splits = [\"train\", \"validation\", \"test\"]\n",
    "stats = {}\n",
    "for dataset in weak_datasets.keys():\n",
    "    if dataset not in stats:\n",
    "        stats[dataset] = {}\n",
    "    stats[dataset][\"n_samples\"] = sum(\n",
    "        weak_datasets[dataset][split][\"data\"].shape[0] for split in splits\n",
    "    )\n",
    "    stats[dataset][\"task\"] = (\n",
    "        \"text\"\n",
    "        if \"raw\" in weak_datasets[dataset][\"train\"]\n",
    "        and isinstance(weak_datasets[dataset][\"train\"][\"raw\"], list)\n",
    "        else \"tabular\"\n",
    "    )\n",
    "    if dataset == \"cifar10\":\n",
    "        stats[dataset][\"task\"] = \"image\"\n",
    "    stats[dataset][\"n_features\"] = weak_datasets[dataset][\"train\"][\"data\"].shape[1]\n",
    "    if stats[dataset][\"task\"] == \"text\":\n",
    "        vocabulary = set()\n",
    "        for split in splits:\n",
    "            sentences = weak_datasets[dataset][split][\"raw\"]\n",
    "            for sentence in sentences:\n",
    "                words = sentence.split()\n",
    "                for word in words:\n",
    "                    vocabulary.add(word)\n",
    "        stats[dataset][\"n_original_features\"] = len(vocabulary)\n",
    "    else:\n",
    "        if \"raw\" in weak_datasets[dataset][\"train\"]:\n",
    "            stats[dataset][\"n_original_features\"] = weak_datasets[dataset][\"train\"][\n",
    "                \"raw\"\n",
    "            ].shape[1]\n",
    "        else:\n",
    "            stats[dataset][\"n_original_features\"] = stats[dataset][\"n_features\"]\n",
    "    stats[dataset][\"n_classes\"] = len(weak_datasets[dataset][\"train\"][\"target_names\"])\n",
    "    stats[dataset][\"priors\"] = np.zeros(stats[dataset][\"n_classes\"])\n",
    "    for split in splits:\n",
    "        stats[dataset][\"priors\"] += np.bincount(\n",
    "            weak_datasets[dataset][split][\"target\"],\n",
    "            minlength=stats[dataset][\"n_classes\"],\n",
    "        )\n",
    "    stats[dataset][\"priors\"] /= np.sum(stats[dataset][\"priors\"])\n",
    "    stats[dataset][\"priors\"] = stats[dataset][\"priors\"].tolist()\n",
    "    stats[dataset][\"n_weak_targets\"] = weak_datasets[dataset][\"train\"][\n",
    "        \"weak_targets\"\n",
    "    ].shape[1]\n",
    "    noisy_targets = WeakLabelEncoder(random_state=1).fit_transform(\n",
    "        weak_datasets[dataset][\"train\"][\"weak_targets\"]\n",
    "    )\n",
    "    unlabeled = noisy_targets == -1\n",
    "    stats[dataset][\"coverage\"] = math.floor((1 - np.mean(unlabeled)) * 100)\n",
    "    stats[dataset][\"noise_ratio\"] = np.mean(\n",
    "        noisy_targets[~unlabeled]\n",
    "        != np.asarray(weak_datasets[dataset][\"train\"][\"target\"])[~unlabeled]\n",
    "    )\n",
    "    noisy_targets[noisy_targets == -1] = stats[dataset][\"n_classes\"]\n",
    "    stats[dataset][\"noise_transition\"] = confusion_matrix(\n",
    "        noisy_targets,\n",
    "        weak_datasets[dataset][\"train\"][\"target\"],\n",
    "        normalize=\"pred\"\n",
    "    )\n",
    "    if stats[dataset][\"coverage\"] < 100:\n",
    "        pass\n",
    "        stats[dataset][\"noise_transition\"] = stats[dataset][\"noise_transition\"][:, 0:-1]\n",
    "    ergodicity = np.empty((stats[dataset][\"n_classes\"], stats[dataset][\"n_classes\"]))\n",
    "    for i in range(stats[dataset][\"n_classes\"]):\n",
    "        for j in range(stats[dataset][\"n_classes\"]):\n",
    "            ergodicity[i, j] = np.sum(\n",
    "                np.abs(\n",
    "                    stats[dataset][\"noise_transition\"][i]\n",
    "                    - stats[dataset][\"noise_transition\"][j]\n",
    "                )\n",
    "            )\n",
    "    stats[dataset][\"ergodicity\"] = 0.5 * np.max(ergodicity)\n",
    "    stats[dataset][\"noise_transition\"] = stats[dataset][\"noise_transition\"].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for dataset in weak_datasets.keys():\n",
    "    if dataset in [\"imdb136\", \"amazon\", \"professor_teacher\"]:\n",
    "        stats[dataset][\"benchmark\"] = \"weasel\"\n",
    "    elif dataset in [\"yoruba\", \"hausa\"]:\n",
    "        stats[dataset][\"benchmark\"] = \"waln\"\n",
    "    elif dataset in [\"cifar10\"]:\n",
    "        stats[dataset][\"benchmark\"] = \"cifar10n-agg\"\n",
    "    else:\n",
    "        stats[dataset][\"benchmark\"] = \"wrench\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "stats[\"professor-teacher\"] = stats.pop(\"professor_teacher\")\n",
    "stats.pop(\"imdb136\")\n",
    "stats.pop(\"cifar10\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def safe_num(num):\n",
    "    if isinstance(num, str):\n",
    "        num = float(num)\n",
    "    return float(\"{:.3g}\".format(abs(num)))\n",
    "\n",
    "\n",
    "def format_number(num):\n",
    "    num = safe_num(num)\n",
    "    sign = \"\"\n",
    "\n",
    "    metric = {\"T\": 1000000000000, \"B\": 1000000000, \"M\": 1000000, \"K\": 1000, \"\": 1}\n",
    "\n",
    "    for index in metric:\n",
    "        num_check = num / metric[index]\n",
    "\n",
    "        if num_check >= 1:\n",
    "            num = num_check\n",
    "            sign = index\n",
    "            break\n",
    "\n",
    "    return f\"{str(num).rstrip('0').rstrip('.')}{sign}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "def priors_to_img(priors):\n",
    "    n_classes = len(priors)\n",
    "    precision = 5\n",
    "    data = np.zeros((precision * n_classes, precision * n_classes), dtype=int)\n",
    "    rounded = np.round(np.asarray(priors) * precision * n_classes)\n",
    "    for i in range(n_classes):\n",
    "        j = 0\n",
    "        while j < precision * n_classes - rounded[i]:\n",
    "            data[j, i * precision : (i + 1) * precision] = 1\n",
    "            j += 1\n",
    "    fig, ax = plt.subplots(figsize=(2, 2))\n",
    "    ax.imshow(data, cmap=\"gray\", vmin=0, vmax=1)\n",
    "    ax.axis(\"off\")\n",
    "    return fig\n",
    "\n",
    "\n",
    "def t_to_img(t):\n",
    "    fig, ax = plt.subplots(figsize=(2, 2))\n",
    "    ax.imshow(1 - np.asarray(t), cmap=\"gray\", vmin=0, vmax=1)\n",
    "    ax.axis(\"off\")\n",
    "    return fig\n",
    "\n",
    "\n",
    "def plot_to_include_file(dataset, plot, folder=\"priors\"):\n",
    "    if not os.path.exists(folder):\n",
    "        os.makedirs(folder)\n",
    "    plot.savefig(\n",
    "        os.path.join(folder, f\"{dataset}.png\"), bbox_inches=\"tight\", transparent=True\n",
    "    )\n",
    "    return f\"\\\\parbox[c]{{16pt}}{{\\\\includegraphics[height=16pt]{{{folder}/{dataset}.png}}}}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_index_length(df, idx):\n",
    "    return len(df.filter(like=idx, axis=0))\n",
    "\n",
    "\n",
    "def format_index(df, idx):\n",
    "    return f\"\\\\rotatebox[origin=c]{{90}}{{{idx.upper()}}}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from functools import partial\n",
    "import pandas as pd\n",
    "\n",
    "\n",
    "stats_df = pd.DataFrame.from_dict(stats, orient=\"index\").reset_index(names=[\"dataset\"])\n",
    "columns_mapper = {\n",
    "    \"benchmark\": \"\\\\textbf{Benchmark}\",\n",
    "    \"task\": \"\\\\textbf{Task}\",\n",
    "    \"dataset\": \"\\\\textbf{Dataset}\",\n",
    "    \"n_samples\": \"$n$\",\n",
    "    \"n_original_features\": \"$d$\",\n",
    "    \"n_features\": \"$\\\\phi(d)$\",\n",
    "    \"n_classes\": \"$K$\",\n",
    "    \"priors\": \"$p(y)$\",\n",
    "    \"n_weak_targets\": \"LRs\",\n",
    "    \"noise_transition\": \"$\\\\mathbf{T}$\",\n",
    "    \"noise_ratio\": \"$p(\\\\tilde{y}\\\\neq y)$\",\n",
    "    \"coverage\": \"coverage\",\n",
    "}\n",
    "\n",
    "output_path = \"dataset_table\"\n",
    "\n",
    "\n",
    "for column in [\"n_samples\", \"n_original_features\", \"n_features\", \"n_weak_targets\"]:\n",
    "    stats_df[column] = stats_df[column].apply(format_number)\n",
    "\n",
    "for i, idx in enumerate(stats_df.index):\n",
    "    stats_df[\"priors\"][i] = plot_to_include_file(\n",
    "        stats_df[\"dataset\"][i],\n",
    "        priors_to_img(stats_df[\"priors\"][i]),\n",
    "        folder=os.path.join(output_path, \"priors\"),\n",
    "    )\n",
    "\n",
    "    stats_df[\"noise_transition\"][i] = plot_to_include_file(\n",
    "        stats_df[\"dataset\"][i],\n",
    "        t_to_img(stats_df[\"noise_transition\"][i]),\n",
    "        folder=os.path.join(output_path, \"noise_transitions\"),\n",
    "    )\n",
    "\n",
    "stats_df[\"coverage\"] = stats_df[\"coverage\"].apply(lambda cov: f\"{cov}\\%\")\n",
    "\n",
    "# for column in [\"benchmark\", \"task\"]:\n",
    "#     stats_df[column] = stats_df[column].apply(partial(format_index, stats_df))\n",
    "\n",
    "stats_df = stats_df.drop(\"ergodicity\", axis=1)\n",
    "stats_df = stats_df[columns_mapper.keys()]\n",
    "stats_df = stats_df.rename(columns_mapper, axis=1)\n",
    "\n",
    "index_columns = [\"benchmark\", \"task\", \"dataset\"]\n",
    "stats_df = stats_df.sort_values([columns_mapper[column] for column in index_columns])\n",
    "stats_df = stats_df.set_index([columns_mapper[column] for column in index_columns])\n",
    "\n",
    "\n",
    "columns_caption = {\n",
    "    \"dataset size\": \"n\",\n",
    "    \"number of raw features\": \"d\",\n",
    "    \"number of encoded features\": \"\\phi(d)\",\n",
    "    \"number of classes\": \"K\",\n",
    "    \"histogram of class priors\": \"p(y)\",\n",
    "    \"number of labeling rules\": \"\\\\text{LRs}\",\n",
    "    \"noise transition matrix\": \"\\\\mathbf{T}\",\n",
    "    \"noise ratio\": \"p(\\\\tilde{y}\\\\neq y)\",\n",
    "    \"percent of weakly unlabeled exemples\": \"\\\\text{coverage}\",\n",
    "}\n",
    "caption = \"Datased used to benchmark detectors. Columns:\"\n",
    "for name, symbol in columns_caption.items():\n",
    "    caption += f\" {name} ${symbol}$,\"\n",
    "caption = caption.rstrip(\",\")\n",
    "caption += \".\"\n",
    "\n",
    "with open(os.path.join(output_path, \"table.tex\"), \"w\") as table:\n",
    "    latex = stats_df.style.format(\n",
    "        precision=2, subset=[columns_mapper[\"noise_ratio\"]]\n",
    "    ).to_latex(\n",
    "        hrules=True,\n",
    "        column_format=\"l\" * 3 + \"r\" * (len(columns_mapper.keys()) - 3),\n",
    "        multirow_align=\"t\",\n",
    "        multicol_align=\"c\",\n",
    "        clines=\"skip-last;data\",\n",
    "        position=\"!h\",\n",
    "        label=\"tab:datasets\",\n",
    "        caption=caption,\n",
    "    )\n",
    "    latex = latex.split(\"\\n\")\n",
    "    latex.insert(1, \"\\\\centering\")\n",
    "    latex.insert(3, \"\\\\resizebox{\\\\columnwidth}{!}{\")\n",
    "    latex.insert(-2, \"}\")\n",
    "    for i, line in enumerate(latex):\n",
    "        if line == \"\\cline{1-12} \\cline{2-12}\":\n",
    "            latex[i] = \"\\cline{1-12}\"\n",
    "    table.write(\"\\n\".join(latex))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "stats[\"youtube\"][\"noise_transition\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}