Titanic-Part1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Part 1. Local Training\n",
    "\n",
    "1. Train a sklearn locally\n",
    "2. Set up Workspace\n",
    "3. Track Runs of the Experiment\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1.1 Train a sklearn model locally"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('data/train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(891, 12)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Name</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Ticket</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Braund, Mr. Owen Harris</td>\n",
       "      <td>male</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>A/5 21171</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Cumings, Mrs. John Bradley (Florence Briggs Th...</td>\n",
       "      <td>female</td>\n",
       "      <td>38.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>PC 17599</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>C85</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>Heikkinen, Miss. Laina</td>\n",
       "      <td>female</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>STON/O2. 3101282</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Futrelle, Mrs. Jacques Heath (Lily May Peel)</td>\n",
       "      <td>female</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>113803</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>C123</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Allen, Mr. William Henry</td>\n",
       "      <td>male</td>\n",
       "      <td>35.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>373450</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   PassengerId  Survived  Pclass  \\\n",
       "0            1         0       3   \n",
       "1            2         1       1   \n",
       "2            3         1       3   \n",
       "3            4         1       1   \n",
       "4            5         0       3   \n",
       "\n",
       "                                                Name     Sex   Age  SibSp  \\\n",
       "0                            Braund, Mr. Owen Harris    male  22.0      1   \n",
       "1  Cumings, Mrs. John Bradley (Florence Briggs Th...  female  38.0      1   \n",
       "2                             Heikkinen, Miss. Laina  female  26.0      0   \n",
       "3       Futrelle, Mrs. Jacques Heath (Lily May Peel)  female  35.0      1   \n",
       "4                           Allen, Mr. William Henry    male  35.0      0   \n",
       "\n",
       "   Parch            Ticket     Fare Cabin Embarked  \n",
       "0      0         A/5 21171   7.2500   NaN        S  \n",
       "1      0          PC 17599  71.2833   C85        C  \n",
       "2      0  STON/O2. 3101282   7.9250   NaN        S  \n",
       "3      0            113803  53.1000  C123        S  \n",
       "4      0            373450   8.0500   NaN        S  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(df.shape)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We'll skip feature eng and just use the numerical columns.\n",
    "X = df[['Pclass', 'Age', 'SibSp', 'Parch', 'Fare']].copy()\n",
    "y = df['Survived'].copy()\n",
    "\n",
    "X['Age'] = X['Age'].fillna(X['Age'].median())\n",
    "assert np.any(X.isna()) == False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6564558629776022"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n",
    "m = RandomForestClassifier(n_estimators=10)\n",
    "m.fit(X_train, y_train)\n",
    "roc_auc_score(y_test, m.predict(X_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's wrap this up in a function:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6860095389507155"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def train_model(csv_fname, n_estimators):\n",
    "    df = pd.read_csv(csv_fname)\n",
    "    \n",
    "    # We'll skip feature eng and just use the numerical columns.\n",
    "    X = df[['Pclass', 'Age', 'SibSp', 'Parch', 'Fare']].copy()\n",
    "    y = df['Survived'].copy()\n",
    "    X['Age'] = X['Age'].fillna(X['Age'].median())\n",
    "    assert np.any(X.isna()) == False\n",
    "    \n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n",
    "    m = RandomForestClassifier(n_estimators=n_estimators)\n",
    "    m.fit(X_train, y_train)\n",
    "    return roc_auc_score(y_test, m.predict(X_test))\n",
    "\n",
    "train_model('data/train.csv', n_estimators=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1.2 Set up Workspace\n",
    "\n",
    "On the Azure portal:\n",
    "\n",
    "1. Create resource group\n",
    "2. Create a Machine Learning workspace\n",
    "\n",
    "This automatically creates a few other resources for you as well."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1.3 Track Runs of the Experiment\n",
    "\n",
    "Base:\n",
    "```\n",
    "pip install azureml-sdk\n",
    "```\n",
    "\n",
    "Optionals:\n",
    "\n",
    "```\n",
    "pip install azureml-sdk[notebooks,tensorboard,explain,automl]\n",
    "pip install azureml-dataprep[pandas]\n",
    "pip install azureml-contrib-explain-model\n",
    "```\n",
    "\n",
    "Note automl actually installs all the code locally on your computer... which takes a while to install."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "You are currently using version 1.0.72 of the Azure ML SDK\n"
     ]
    }
   ],
   "source": [
    "import azureml.core\n",
    "print(\"You are currently using version\", azureml.core.VERSION, \"of the Azure ML SDK\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "subscription_id = os.getenv(\"SUBSCRIPTION_ID\")\n",
    "resource_group = os.getenv(\"RESOURCE_GROUP\")\n",
    "workspace_name = os.getenv(\"WORKSPACE_NAME\")\n",
    "workspace_region = os.getenv(\"WORKSPACE_REGION\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Workspace configuration succeeded.\n"
     ]
    }
   ],
   "source": [
    "from azureml.core import Workspace\n",
    "\n",
    "try:\n",
    "    ws = Workspace(subscription_id = subscription_id, resource_group = resource_group, workspace_name = workspace_name)\n",
    "    # write the details of the workspace to a configuration file to the notebook library\n",
    "    ws.write_config() # ws = Workspace.from_config()\n",
    "    print(\"Workspace configuration succeeded.\")\n",
    "except:\n",
    "    print(\"Workspace not accessible. Change your parameters or create a new workspace below.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from azureml.core import Experiment, Run\n",
    "\n",
    "experiment = Experiment(workspace = ws, name = \"titanic-3\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This uploads the code your git repo is tracking; if your repo is too big, it will fail.\n",
    "run = experiment.start_logging()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "run.log('n_estimators', 10)\n",
    "\n",
    "roc_auc = train_model('data/train.csv', n_estimators=10)\n",
    "\n",
    "run.log('roc_auc', roc_auc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "run.complete()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You can tag and log a bunch of stuff in Run. [See documentation](https://docs.microsoft.com/en-us/python/api/azureml-core/azureml.core.run(class)?view=azure-ml-py)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now's a good time to view this on the Azure portal."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Track many experiments and compare them"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 9/9 [01:23<00:00,  9.23s/it]\n"
     ]
    }
   ],
   "source": [
    "# Note: I'm using the same experiment\n",
    "experiment = Experiment(workspace = ws, name = \"titanic-1\")\n",
    "\n",
    "for n_estimators in tqdm([1,5,10,50,100,200,300,400,500]):\n",
    "    run = experiment.start_logging()\n",
    "    run.log('n_estimators', 10)\n",
    "    roc_auc = train_model('data/train.csv', n_estimators=n_estimators)\n",
    "    run.log('roc_auc', roc_auc)\n",
    "    run.complete()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Note: **You can't actually delete any runs or experiments**. No idea why...\n",
    "\n",
    "We can fetch them all and filter.\n",
    "Perhaps a good practice is to keep each experiment small."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1.91 s, sys: 163 ms, total: 2.08 s\n",
      "Wall time: 29.5 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "run_metrics = []\n",
    "for run in experiment.get_runs():\n",
    "    if run.status == 'Completed':\n",
    "        run_metrics.append(run.get_metrics())\n",
    "run_metrics = pd.DataFrame(run_metrics)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Alternatively we can filter like so:\n",
    "\n",
    "```\n",
    "list(exp.get_runs(properties={\"author\":\"azureml-user\"},tags=\"worth another look\"))\n",
    "```\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# I called run.log('n_estimators', ...) twice in a one run, by accident. This creates a list object instead of overwriting.\n",
    "# So I'm just converting the list back into the actual value.\n",
    "run_metrics['n_estimators'] = run_metrics['n_estimators'].map(lambda x: x[0] if isinstance(x, list) else x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>roc_auc-mean</th>\n",
       "      <th>roc_auc-std</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>n_estimators</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.688588</td>\n",
       "      <td>0.047790</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.706410</td>\n",
       "      <td>0.027006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.667017</td>\n",
       "      <td>0.039594</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>0.734730</td>\n",
       "      <td>0.013213</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>0.723661</td>\n",
       "      <td>0.017606</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>200</th>\n",
       "      <td>0.720627</td>\n",
       "      <td>0.006344</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>300</th>\n",
       "      <td>0.731175</td>\n",
       "      <td>0.004351</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>400</th>\n",
       "      <td>0.733854</td>\n",
       "      <td>0.007632</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>500</th>\n",
       "      <td>0.737002</td>\n",
       "      <td>0.007094</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              roc_auc-mean  roc_auc-std\n",
       "n_estimators                           \n",
       "1                 0.688588     0.047790\n",
       "5                 0.706410     0.027006\n",
       "10                0.667017     0.039594\n",
       "50                0.734730     0.013213\n",
       "100               0.723661     0.017606\n",
       "200               0.720627     0.006344\n",
       "300               0.731175     0.004351\n",
       "400               0.733854     0.007632\n",
       "500               0.737002     0.007094"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "run_metrics.groupby('n_estimators') \\\n",
    "        .roc_auc.agg(['mean', 'std']) \\\n",
    "        .rename(columns={'mean': 'roc_auc-mean', 'std': 'roc_auc-std'})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Additional Resources"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://github.com/Azure/MachineLearningNotebooks/tree/69d4344dff6de3633773b89b818f20bd630cf40c/how-to-use-azureml"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "azure-ml-demo",
   "language": "python",
   "name": "azure-ml-demo"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}