medqa_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ItZn05k6XSka"
   },
   "source": [
    "# MedQA Relabelling Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "g8W29uraWAFk"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import functools\n",
    "import pandas as pd\n",
    "from matplotlib import pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "gU6N6NU5Jnpu"
   },
   "outputs": [],
   "source": [
    "COLORS = [\n",
    "    'salmon', 'orange', 'mediumseagreen', 'cornflowerblue'\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "5cSwnQFM8YHz"
   },
   "outputs": [],
   "source": [
    "input_file = 'medqa_relabelling.csv'\n",
    "with open(input_file, 'r') as f:\n",
    "  df = pd.read_csv(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "_NIOojeG68SP"
   },
   "outputs": [],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "piZ59gwhXU3u"
   },
   "source": [
    "## Voting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "-LbngG7kwP9T"
   },
   "outputs": [],
   "source": [
    "def compute_blind_errors(row):\n",
    "  \"\"\"Computes blind errors/mis-coverage, if GT is in rater's answers.\"\"\"\n",
    "  answer_idx = row['answer_idx']\n",
    "  responses = row['blind_answers']\n",
    "  if not isinstance(responses, list):\n",
    "    # No answer given.\n",
    "    return True\n",
    "  return answer_idx not in responses\n",
    "\n",
    "\n",
    "def compute_seen_errors(row):\n",
    "  \"\"\"Computes seen errors/mis-coverage, if GT is in rater's answers.\"\"\"\n",
    "  answer_idx = row['answer_idx']\n",
    "  responses = row['seen_answers']\n",
    "  if not isinstance(responses, list):\n",
    "    return True\n",
    "  return answer_idx not in responses\n",
    "\n",
    "\n",
    "def select_combined_answers(row):\n",
    "  \"\"\"Selects either seen or blind answers depending.\"\"\"\n",
    "  responses = row['seen_answers']\n",
    "  if not isinstance(responses, list):\n",
    "    # Rater did not change answer after revealing GT.\n",
    "    responses = row['blind_answers']\n",
    "  return responses\n",
    "\n",
    "\n",
    "def select_combined_answerable(row):\n",
    "  \"\"\"Selects either seen or blind answerable.\"\"\"\n",
    "  if pd.isnull(row['seen_answerable']):\n",
    "    return row['blind_answerable']\n",
    "  return row['seen_answerable']\n",
    "\n",
    "\n",
    "def compute_combined_errors(row):\n",
    "  \"\"\"Combined uses seen if seen_change, and blind answers otherwise.\"\"\"\n",
    "  answer_idx = row['answer_idx']\n",
    "  responses = row['combined_answers']\n",
    "  if not isinstance(responses, list):\n",
    "    return True\n",
    "  return answer_idx not in responses\n",
    "\n",
    "\n",
    "def compute_combined_size(row):\n",
    "  \"\"\"Computes size of combined answers.\"\"\"\n",
    "  responses = row['combined_answers']\n",
    "  if not isinstance(responses, list):\n",
    "    return 0\n",
    "  return len(responses)\n",
    "\n",
    "\n",
    "# Convert answers to lists.\n",
    "df['blind_answers'] = df['blind_answers'].apply(lambda x: eval(x) if isinstance(x, str) else x)\n",
    "df['seen_answers'] = df['seen_answers'].apply(lambda x: eval(x) if isinstance(x, str) else x)\n",
    "# Compute rater errors before and after revealing the GT.\n",
    "df['blind_errors'] = df.apply(compute_blind_errors, axis=1)\n",
    "df['seen_errors'] = df.apply(compute_seen_errors, axis=1)\n",
    "# Compute combined answer, answer size = # of selected options, errors.\n",
    "df['combined_answers'] = df.apply(select_combined_answers, axis=1)\n",
    "df['combined_size'] = df.apply(compute_combined_size, axis=1)\n",
    "df['combined_ambiguous'] = df['combined_size'] > 1\n",
    "df['combined_answerable'] = df.apply(select_combined_answerable, axis=1)\n",
    "df['combined_errors'] = df.apply(compute_combined_errors, axis=1)\n",
    "# If info_missing is False, then important_info_missing should also be False.\n",
    "df['important_info_missing'] = df['important_info_missing'].apply(\n",
    "    lambda x: x == True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "NdbYFZ_bwP9U"
   },
   "outputs": [],
   "source": [
    "# Columns that stay the same.\n",
    "same_keys = [\n",
    "    'qid', 'question', 'A', 'B', 'C', 'D', 'answer_idx',\n",
    "]\n",
    "# Columns that we want to vote over (i.e., aggregate rater opinions for).\n",
    "vote_keys = [\n",
    "    'blind_answerable', 'seen_answerable', 'combined_answerable',\n",
    "    'important_info_missing', 'info_missing',\n",
    "    'blind_errors', 'seen_errors', 'combined_errors',\n",
    "    'seen_change', 'combined_ambiguous',\n",
    "]\n",
    "keep_keys = ['blind_answers', 'seen_answers', 'combined_answers']\n",
    "vote_df = df[['qid'] + vote_keys + keep_keys]\n",
    "core_df = df[same_keys]\n",
    "core_df = core_df.drop_duplicates(['qid'])\n",
    "# We aggregate all rater opinions for the columns we want to vote on.\n",
    "vote_dfs = [core_df]\n",
    "for vote_key in vote_keys:\n",
    "  agg_df = vote_df.groupby('qid')[vote_key].apply(list).reset_index()\n",
    "  vote_dfs.append(agg_df)\n",
    "for keep_key in keep_keys:\n",
    "  agg_df = vote_df.groupby('qid')[keep_key].apply(list).reset_index()\n",
    "  vote_dfs.append(agg_df)\n",
    "# Merge all the individually aggregated columns.\n",
    "vote_df = functools.reduce(\n",
    "    lambda left, right: pd.merge(left, right, on=['qid']), vote_dfs)\n",
    "# Vote by requiring 2 or 3 rater majority.\n",
    "for vote_key in vote_keys:\n",
    "  vote_df[f'sum_{vote_key}'] = vote_df[vote_key].apply(\n",
    "      lambda xs: np.sum([x if isinstance(x, bool) else False for x in xs]))\n",
    "  for k in [1, 2, 3]:\n",
    "    vote_df[f'vote{k}_{vote_key}'] = vote_df[f'sum_{vote_key}'] >= k"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "52tHjmfJ24r5"
   },
   "source": [
    "## Simulate model predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "sXPXaFDw3xqo"
   },
   "outputs": [],
   "source": [
    "# We target an accuracy of 0.911 (in expectation), but fix an accuracy of 0.98\n",
    "# on examples that are likely unfilteres; this creates a scenario similar to the paper\n",
    "# where the model makes more mistakes on examples that are about to be filtered out.\n",
    "# This is the place where you can load your modal predictions!\n",
    "np.random.seed(42)\n",
    "num_filtered = int(np.sum(vote_df['vote1_combined_errors'] == True))\n",
    "filter_rate = num_filtered/vote_df.shape[0]\n",
    "accuracy_on_unfiltered = 0.98\n",
    "accuracy_on_filtered = (\n",
    "    (0.911 - accuracy_on_unfiltered * (1 - filter_rate)) / filter_rate\n",
    ")\n",
    "error_df = vote_df[['vote1_combined_errors', 'qid']]\n",
    "error_df['error'] = False\n",
    "error_df.loc[error_df['vote1_combined_errors'] == True, 'error'] = (\n",
    "    np.random.uniform(0, 1, (num_filtered,)) > accuracy_on_filtered\n",
    ")\n",
    "error_df.loc[error_df['vote1_combined_errors'] == False, 'error'] = (\n",
    "    np.random.uniform(0, 1, (error_df.shape[0] - num_filtered,)) > accuracy_on_unfiltered\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "dp3MzIqvL-oH"
   },
   "outputs": [],
   "source": [
    "df = pd.merge(df, error_df[['qid', 'error']], on=['qid'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "6vDypA6t96F6"
   },
   "source": [
    "## Evaluation results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "dQqEJqi-TBdu"
   },
   "outputs": [],
   "source": [
    "k = 3  # 3 = unanomous voting (main paper), 2 = majority voting (appendix).\n",
    "n_trials = 10  # 1000 in the paper.\n",
    "num_questions = []\n",
    "error_rates = []\n",
    "agg_df = df.groupby(['qid', 'error']).agg(list).reset_index()\n",
    "for t in list(range(n_trials)):\n",
    "  agg_df['sample_info_missing'] = agg_df['info_missing'].apply(\n",
    "      lambda x: np.sum(np.random.choice(x, 3, replace=True)) >= k)\n",
    "  agg_df['sample_combined_errors'] = agg_df['combined_errors'].apply(\n",
    "      lambda x: np.sum(np.random.choice(x, 3, replace=True)) >= k)\n",
    "  agg_df['sample_combined_ambiguous'] = agg_df['combined_ambiguous'].apply(\n",
    "      lambda x: np.sum(np.random.choice(x, 3, replace=True)) >= k)\n",
    "\n",
    "  qs = [\n",
    "      # All questions.\n",
    "      agg_df.shape[0],\n",
    "      # Filter questions that are missing information.\n",
    "      np.sum(agg_df['sample_info_missing'] == False),\n",
    "      # Filter label errors.\n",
    "      np.sum(np.logical_and(\n",
    "          agg_df['sample_info_missing'] == False,\n",
    "          agg_df['sample_combined_errors'] == False)),\n",
    "      # Filter ambiguous questions.\n",
    "      np.sum(np.logical_and.reduce((\n",
    "          agg_df['sample_info_missing'] == False,\n",
    "          agg_df['sample_combined_errors'] == False,\n",
    "          agg_df['sample_combined_ambiguous'] == False))),\n",
    "  ]\n",
    "  es = [\n",
    "      np.sum(agg_df['error']) / qs[0],\n",
    "      np.sum(np.logical_and(\n",
    "          agg_df['error'],\n",
    "          agg_df['sample_info_missing'] == False\n",
    "      )) / qs[1],\n",
    "      np.sum(np.logical_and.reduce((\n",
    "          agg_df['error'],\n",
    "          agg_df['sample_info_missing'] == False,\n",
    "          agg_df['sample_combined_errors'] == False,\n",
    "      ))) / qs[2],\n",
    "      np.sum(np.logical_and.reduce((\n",
    "          agg_df['error'],\n",
    "          agg_df['sample_info_missing'] == False,\n",
    "          agg_df['sample_combined_errors'] == False,\n",
    "          agg_df['sample_combined_ambiguous'] == False,\n",
    "      ))) / qs[3],\n",
    "  ]\n",
    "  num_questions.append(qs)\n",
    "  error_rates.append(es)\n",
    "  print(t)\n",
    "num_questions = np.array(num_questions)\n",
    "error_rates = np.array(error_rates)\n",
    "\n",
    "groups = ['Before', 'w/o\\nmissing info', 'w/o\\nlabel errors', 'w/o\\nambiguous']\n",
    "x = np.arange(len(groups))  # the label locations\n",
    "width = 0.45  # the width of the bars\n",
    "multiplier = 0\n",
    "\n",
    "fig, ax1 = plt.subplots(layout='constrained')\n",
    "ax2 = ax1.twinx()\n",
    "\n",
    "for ax, color, label, mean, std in [\n",
    "    (ax1, COLORS[3], 'Accuracy', 100 * (1 - np.mean(error_rates, axis=0)), np.std(100 * error_rates, axis=0)),\n",
    "    (ax2, COLORS[0], 'Questions', 100 * np.mean(num_questions / 1273, axis=0), 100 * np.std(num_questions / 1273, axis=0)),\n",
    "]:\n",
    "    offset = width * multiplier\n",
    "    rects = ax.bar(x + offset, mean, yerr=std, width=width, label=label, capsize=5, color=color)\n",
    "    ax.bar_label(rects, padding=3, fmt='%.1f', label_type='edge')\n",
    "    multiplier += 1\n",
    "\n",
    "ax1.spines['top'].set_visible(False)\n",
    "ax2.spines['top'].set_visible(False)\n",
    "ax1.set_ylabel('MedQA Accuracy', fontsize=12)\n",
    "ax2.set_ylabel('Fraction of questions', fontsize=12)\n",
    "ax1.set_xticks(x + width/2, groups, rotation=30, fontsize=12)\n",
    "ax1.legend(loc='upper right', bbox_to_anchor=(1, 1), ncols=1)\n",
    "ax2.legend(loc='upper right', bbox_to_anchor=(0.68, 1), ncols=1)\n",
    "ax1.set_ylim(90, 100)\n",
    "ax2.set_ylim(90, 100)\n",
    "ax.grid(False)\n",
    "plt.gcf().set_size_inches((5, 3))\n",
    "plt.savefig('medqa_filtering.pdf', dpi=300, format='pdf')\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}