example.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Imports\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import fairlearn.metrics as flm\n",
    "import sklearn.metrics as skm\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.exceptions import ConvergenceWarning\n",
    "\n",
    "from disagg import naive_estimate, structured_regression\n",
    "\n",
    "import warnings\n",
    "warnings.simplefilter(action='ignore', category=ConvergenceWarning)\n",
    "\n",
    "# Definitions of performance metrics\n",
    "def safe_ratio(num, den):\n",
    "    if den==0:\n",
    "        return np.nan\n",
    "    else:\n",
    "        return num / den\n",
    "\n",
    "def false_negative_rate(y_true, y_pred):\n",
    "    tn, fp, fn, tp = confusion_matrix(y_true, y_pred, labels=[0, 1]).ravel()\n",
    "    return safe_ratio(fn, fn+tp)\n",
    "\n",
    "def false_positive_rate(y_true, y_pred):\n",
    "    tn, fp, fn, tp = confusion_matrix(y_true, y_pred, labels=[0, 1]).ravel()\n",
    "    return safe_ratio(fp, fp+tn)\n",
    "\n",
    "def precision_score(y_true, y_pred):\n",
    "    tn, fp, fn, tp = confusion_matrix(y_true, y_pred, labels=[0, 1]).ravel()\n",
    "    return safe_ratio(tp, tp+fp)\n",
    "\n",
    "def roc_auc_score(y_true, y_pred):\n",
    "    if len(np.unique(y_true)) == 1:\n",
    "        return np.nan\n",
    "    else:\n",
    "        return skm.roc_auc_score(y_true, y_pred)\n",
    "\n",
    "METRICS = {\n",
    "    'auc': roc_auc_score,\n",
    "    'sel': flm.selection_rate,\n",
    "    'fnr': false_negative_rate,\n",
    "    'fpr': false_positive_rate,\n",
    "    'acc': skm.accuracy_score,\n",
    "    'ppv': precision_score\n",
    "}\n",
    "\n",
    "# Grid for cross-validation\n",
    "ALPHA_LIST = [20.0, 10.0, 5.0, 2.0, 1.0, 0.5, 0.2, 0.1, 0.05, 0.02, 0.01, 0.005]\n",
    "\n",
    "# Confidence interval values\n",
    "CI_VALS = [50, 80, 90, 95]\n",
    "\n",
    "# Fancy plotting of errors with clipping of values to the range of the metric\n",
    "def plot_errors(*, ax, counts, vals, ci_lower=None, ci_upper=None, offset, marker='o', clip=None, label):\n",
    "    skip_ci = False\n",
    "    if (ci_lower is None) or (ci_upper is None):\n",
    "        ci_lower = vals\n",
    "        ci_upper = vals\n",
    "        skip_ci = True\n",
    "    if clip is not None:\n",
    "        vals = vals.clip(clip[0], clip[1])\n",
    "        ci_lower = ci_lower.clip(clip[0], clip[1])\n",
    "        ci_upper = ci_upper.clip(clip[0], clip[1])\n",
    "    sorted_index = counts.sort_values().index\n",
    "    counts = counts.reindex(sorted_index)\n",
    "    vals = vals.reindex(sorted_index)\n",
    "    ci_lower = ci_lower.reindex(sorted_index).clip(upper=vals)\n",
    "    ci_upper = ci_upper.reindex(sorted_index).clip(lower=vals)\n",
    "\n",
    "    select = ~vals.isna()\n",
    "    xs = np.arange(len(vals))\n",
    "    yerr = np.array([vals[select]-ci_lower[select], ci_upper[select]-vals[select]])\n",
    "\n",
    "    if skip_ci:\n",
    "        ax.plot(xs[select]+offset, vals[select], marker=marker, ls='', label=label)\n",
    "    else:\n",
    "        ax.errorbar(xs[select]+offset, vals[select], yerr=yerr, marker=marker, ls='', label=label)\n",
    "    \n",
    "    ax.xaxis.set_tick_params(pad=12)\n",
    "    ax.set_xticks(xs)\n",
    "    ax.set_xticklabels(sorted_index, rotation=90, va='top', ha='center')\n",
    "    [ymin, ymax] = ax.get_ylim()\n",
    "    for i in range(len(vals)):\n",
    "        ax.text(i, ymin-(ymax-ymin)*0.03, f\"{counts[i]}\", va='top', ha='center', c='r', fontsize=8)    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Loading of diabetes evaluation data\n",
    "def load_eval(*, metric, eval_id):\n",
    "    \n",
    "    # Load evaluation dataset\n",
    "    df = pd.read_csv(f\"diabetes/evaluation/eval{eval_id:02d}.csv\")\n",
    "\n",
    "    # Load ground-truth and set threshold for classification decisions\n",
    "    df_truth = pd.read_csv('diabetes/evaluation/eval_all.csv')\n",
    "    threshold = df_truth.S.quantile(0.8)\n",
    "\n",
    "    # Generate ground-truth labels\n",
    "    for df_i in [df, df_truth]:\n",
    "        df_i['y'] = df_i.readmit_30_days.map({True: \"y1\", False: \"y0\"})\n",
    "        df_i['y_true'] = df_i.readmit_30_days.astype(float)\n",
    "        if metric == 'auc':\n",
    "            df_i['y_pred'] = df_i.S \n",
    "        else:\n",
    "            df_i['y_pred'] = 1.0*(df_i.S>threshold)\n",
    "\n",
    "    # Generate features for regression\n",
    "    cat1 = df[['race', 'gender', 'age', 'y']]\n",
    "    cat2 = pd.DataFrame({\n",
    "        'y_race': df['y'].str.cat(df['race'], sep='_'),\n",
    "        'y_gender': df['y'].str.cat(df['gender'], sep='_'),\n",
    "        'y_age': df['y'].str.cat(df['age'], sep='_'),\n",
    "        'race_gender': df['race'].str.cat(df['gender'], sep='_'),\n",
    "        'race_age': df['race'].str.cat(df['age'], sep='_'),\n",
    "        'gender_age': df['gender'].str.cat(df['age'], sep='_')})\n",
    "    cat3 = df[['grp_name']]\n",
    "    static_cov = df[['number_inpatient', 'number_diagnoses', 'number_emergency', 'number_outpatient', 'chf']].astype(float)\n",
    "    labels = df[['grp_name', 'y_true', 'y_pred']]\n",
    "    static_cov = (static_cov - static_cov.mean()) / static_cov.std()\n",
    "\n",
    "    dummies1 = pd.get_dummies(cat1)\n",
    "    dummies2 = pd.get_dummies(cat2)\n",
    "    dummies3 = pd.get_dummies(cat3)\n",
    "\n",
    "    processed_df = pd.concat([labels, dummies1, dummies2, dummies3, static_cov], axis=1)\n",
    "\n",
    "    # Consider three regression models called fa, fb, fc\n",
    "    model_features = {\n",
    "        'fa': list(dummies1) + list(dummies3),\n",
    "        'fb': list(dummies1) + list(dummies3) + list(static_cov),\n",
    "        'fc': list(dummies1) + list(dummies2) + list(dummies3) + list(static_cov),\n",
    "    }\n",
    "    return processed_df, model_features, df_truth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Example of a disaggregated evaluation\n",
    "metric = 'fpr'\n",
    "eval_id = 0\n",
    "\n",
    "df, model_features, df_truth = load_eval(metric=metric, eval_id=eval_id)\n",
    "naive = naive_estimate(\n",
    "        df=df, metric=METRICS[metric], ci=CI_VALS)\n",
    "lasso, _, _, _, _, _ = structured_regression(\n",
    "        df=df, metric=METRICS[metric], feature_names=model_features['fb'], alpha_list=ALPHA_LIST, cv=True, ci=CI_VALS)\n",
    "truth = naive_estimate(\n",
    "        df=df_truth, metric=METRICS[metric])\n",
    "\n",
    "fig, ax = plt.subplots(1, 1, figsize=[12,3])\n",
    "counts = naive['count']\n",
    "plot_errors(ax=ax, vals=naive.val, ci_lower=naive.ci95l, ci_upper=naive.ci95u, counts=counts, offset=-0.2, clip=[0,1], label='naive')\n",
    "plot_errors(ax=ax, vals=lasso.val, ci_lower=lasso.ci95l_rlpr, ci_upper=lasso.ci95u_rlpr, counts=counts, offset=0, clip=[0,1], label='lasso')\n",
    "plot_errors(ax=ax, vals=truth.val, counts=counts, offset=0.2, label='truth', marker='x')\n",
    "ax.legend()\n",
    "ax.set_ylabel(metric.upper())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Test implementation -- THIS TAKES A WHILE\n",
    "for metric in METRICS:\n",
    "    for eval_id in range(5):\n",
    "        df, model_features, df_truth = load_eval(metric=metric, eval_id=eval_id)\n",
    "        eval_str = f\"{eval_id:02d}_{metric}\"\n",
    "        print(f\"Evaluation dataset: {eval_str}\")\n",
    "        naive = naive_estimate(\n",
    "            df=df, metric=METRICS[metric], ci=CI_VALS)\n",
    "        lasso, _, _, _, _, _ = structured_regression(\n",
    "            df=df, metric=METRICS[metric], feature_names=model_features['fb'], alpha_list=ALPHA_LIST, cv=True, ci=CI_VALS)\n",
    "        naive_ref = pd.read_csv(f\"diabetes/results/res{eval_str}_naive.csv\", index_col='grp_name')\n",
    "        lasso_ref = pd.read_csv(f\"diabetes/results/res{eval_str}_lasso_fb.csv\", index_col='grp_name')\n",
    "        print(f\"  naive difference: {(naive-naive_ref).abs().max().max()}\")\n",
    "        print(f\"  lasso difference: {(lasso-lasso_ref).abs().max().max()}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}