prescriptor_robojudge.ipynb

{
 "cells": [
  {
   "cell_type": "raw",
   "metadata": {
    "scrolled": false
   },
   "source": [
    "# Copyright 2020 (c) Cognizant Digital Business, Evolutionary AI. All rights reserved. Issued under the Apache 2.0 License."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "scrolled": false
   },
   "source": [
    "# Prescriptor robojudge\n",
    "Notebook for evaluating prescriptions using standard predictor.\n",
    "\n",
    "The main metric computed is based on how many other prescriptions\n",
    "each submission dominates on the pareto front of stringency vs cases."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from covid_xprize.standard_predictor.xprize_predictor import NPI_COLUMNS\n",
    "from covid_xprize.scoring.prescriptor_scoring import weight_prescriptions_by_cost\n",
    "from covid_xprize.scoring.prescriptor_scoring import generate_cases_and_stringency_for_prescriptions\n",
    "from covid_xprize.scoring.prescriptor_scoring import compute_domination_df\n",
    "from covid_xprize.scoring.prescriptor_scoring import compute_pareto_set\n",
    "from covid_xprize.validation.prescriptor_validation import validate_submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Can set these longer for better evaluation. Will increase eval time\n",
    "START_DATE = \"2020-08-01\"\n",
    "END_DATE = \"2020-08-05\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Prepare Historical IP Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "from covid_xprize.scoring.predictor_scoring import load_dataset\n",
    "from covid_xprize.validation.scenario_generator import generate_scenario\n",
    "\n",
    "LATEST_DATA_URL = 'https://raw.githubusercontent.com/OxCGRT/covid-policy-tracker-legacy/main/legacy_data_202207/OxCGRT_latest.csv'\n",
    "GEO_FILE = \"countries_regions.csv\"\n",
    "\n",
    "latest_df = load_dataset(LATEST_DATA_URL, GEO_FILE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "IP_FILE = \"prescriptions/robojudge_test_scenario.csv\"\n",
    "countries = None\n",
    "scenario_df = generate_scenario(START_DATE, END_DATE, latest_df, countries, scenario=\"Freeze\")\n",
    "scenario_df.to_csv(IP_FILE, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generate some prescriptions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Cost weightings for each IP for each geo\n",
    "TEST_COST = \"covid_xprize/validation/data/uniform_random_costs.csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Generate some random prescriptions\n",
    "for i in range(2):\n",
    "    output_file = \"covid_xprize/examples/prescriptors/random/prescriptions/random_presc_\"+ str(i+1) +\".csv\"\n",
    "    !python covid_xprize/examples/prescriptors/random/prescribe.py -s {START_DATE} \\\n",
    "                                                                   -e {END_DATE} \\\n",
    "                                                                   -ip {IP_FILE} \\\n",
    "                                                                   -c {TEST_COST} \\\n",
    "                                                                   -o {output_file}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Generate blind_greedy prescriptions\n",
    "output_file = \"covid_xprize/examples/prescriptors/blind_greedy/prescriptions/blind_greedy.csv\"\n",
    "!python covid_xprize/examples/prescriptors/blind_greedy/prescribe.py -s {START_DATE} \\\n",
    "                                                                     -e {END_DATE} \\\n",
    "                                                                     -ip {IP_FILE} \\\n",
    "                                                                     -c {TEST_COST} \\\n",
    "                                                                     -o {output_file}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "prescription_files = {\n",
    "#     'NeatExample': 'covid_xprize/examples/prescriptors/neat/test_prescriptions/pres.csv',\n",
    "    'Random1': 'covid_xprize/examples/prescriptors/random/prescriptions/random_presc_1.csv',\n",
    "    'Random2': 'covid_xprize/examples/prescriptors/random/prescriptions/random_presc_2.csv',\n",
    "    'BlindGreedy': 'covid_xprize/examples/prescriptors/blind_greedy/prescriptions/blind_greedy.csv',\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Validate the prescription files\n",
    "for prescriptor_name, output_file in prescription_files.items():\n",
    "    errors = validate_submission(START_DATE, END_DATE, IP_FILE, output_file)\n",
    "    if errors:\n",
    "        for error in errors:\n",
    "            print(f\"{prescriptor_name}: {error}\")\n",
    "    else:\n",
    "        print(f\"{prescriptor_name}: All good!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluate them"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Collect case and stringency data for all prescriptors\n",
    "dfs = []\n",
    "for prescriptor_name, prescription_file in sorted(prescription_files.items()):\n",
    "    print(\"Generating predictions for\", prescriptor_name)\n",
    "    df, preds = generate_cases_and_stringency_for_prescriptions(START_DATE, END_DATE, prescription_file, TEST_COST, IP_FILE)\n",
    "    df['PrescriptorName'] = prescriptor_name\n",
    "    dfs.append(df)\n",
    "df = pd.concat(dfs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "df[df['CountryName'] == 'Afghanistan']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "ddf = compute_domination_df(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Get number of dominated prescriptions for each submission. This is the \"Domination Count\"\n",
    "ddf.groupby('DominatingName', group_keys=False).count().sort_values('DominatedIndex', ascending=False)['DominatedIndex']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "def plot_pareto_curve(objective1_list, objective2_list):\n",
    "    \"\"\"\n",
    "    Plot the pareto curve given the objective values for a set of solutions.\n",
    "    This curve indicates the area dominated by the solution set, i.e., \n",
    "    every point up and to the right is dominated.\n",
    "    \"\"\"\n",
    "    \n",
    "    # Compute pareto set from full solution set.\n",
    "    objective1_pareto, objective2_pareto = compute_pareto_set(objective1_list, \n",
    "                                                              objective2_list)\n",
    "    \n",
    "    # Sort by first objective.\n",
    "    objective1_pareto, objective2_pareto = list(zip(*sorted(zip(objective1_pareto,\n",
    "                                                                objective2_pareto))))\n",
    "    \n",
    "    # Compute the coordinates to plot.\n",
    "    xs = []\n",
    "    ys = []\n",
    "    \n",
    "    xs.append(objective1_pareto[0])\n",
    "    ys.append(objective2_pareto[0])\n",
    "    \n",
    "    for i in range(0, len(objective1_pareto)-1):\n",
    "        \n",
    "        # Add intermediate point between successive solutions\n",
    "        xs.append(objective1_pareto[i+1])\n",
    "        ys.append(objective2_pareto[i])\n",
    "        \n",
    "        # Add next solution on front\n",
    "        xs.append(objective1_pareto[i+1])\n",
    "        ys.append(objective2_pareto[i+1])\n",
    "        \n",
    "    plt.plot(xs, ys)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Plot overall stringency and cases of each prescription\n",
    "plt.figure(figsize=(10,8))\n",
    "for prescriptor_name in prescription_files:\n",
    "    pdf = df[df['PrescriptorName'] == prescriptor_name]\n",
    "    overall_pdf = pdf.groupby('PrescriptionIndex', group_keys=False).mean().reset_index()\n",
    "    plt.scatter(overall_pdf['Stringency'],\n",
    "                overall_pdf['PredictedDailyNewCases'], \n",
    "                label=prescriptor_name)\n",
    "    plot_pareto_curve(list(overall_pdf['Stringency']),\n",
    "                      list(overall_pdf['PredictedDailyNewCases']))\n",
    "plt.xlabel('Mean stringency')\n",
    "plt.ylabel('Mean cases per day per geo')\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Plot stacked line chart of npis over time for a prescription for a particular geo\n",
    "\n",
    "#submission_file = 'covid_xprize.examples/prescriptors/neat/test_prescriptions/pres.csv'\n",
    "submission_file = 'covid_xprize/examples/prescriptors/random/prescriptions/random_presc_1.csv'\n",
    "\n",
    "prescription_index = 2\n",
    "country_name = 'United States'\n",
    "region_name = None\n",
    "\n",
    "pdf = pd.read_csv(submission_file)\n",
    "gdf = pdf[(pdf['PrescriptionIndex'] == prescription_index) &\n",
    "          (pdf['CountryName'] == country_name) &\n",
    "          (pdf['RegionName'].isna() if region_name is None else (pdf['RegionName'] == 'region_name'))]\n",
    "gdf.plot.area(x='Date', y=NPI_COLUMNS, figsize=(10,8), ylabel='Stringency')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# Plot stringency and cases of each prescription for a particular country\n",
    "country_name = 'Albania'\n",
    "cdf = df[df.CountryName == country_name]\n",
    "\n",
    "plt.figure(figsize=(10,8))\n",
    "for prescriptor_name in prescription_files:\n",
    "    pdf = cdf[cdf['PrescriptorName'] == prescriptor_name]\n",
    "    #overall_pdf = pdf.groupby('PrescriptionIndex', group_keys=False).mean().reset_index()\n",
    "    plt.scatter(pdf['Stringency'],\n",
    "                pdf['PredictedDailyNewCases'], \n",
    "                label=prescriptor_name)\n",
    "    plot_pareto_curve(list(pdf['Stringency']),\n",
    "                      list(pdf['PredictedDailyNewCases']))\n",
    "plt.xlabel('Mean stringency')\n",
    "plt.ylabel('Mean cases per day per geo')\n",
    "plt.title(country_name)\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}