pipelines/samples/employee-attrition/employee-attrition-pipeline.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "e801d275",
   "metadata": {},
   "source": [
    "# Employee attrition sample using ML Pipelines"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3e5a5e79",
   "metadata": {},
   "source": [
    "### OCI Data Science - Useful Tips\n",
    "<details>\n",
    "<summary><font size=\"2\">Check for Public Internet Access</font></summary>\n",
    "\n",
    "```python\n",
    "import requests\n",
    "response = requests.get(\"https://oracle.com\")\n",
    "assert response.status_code==200, \"Internet connection failed\"\n",
    "```\n",
    "</details>\n",
    "<details>\n",
    "<summary><font size=\"2\">Helpful Documentation </font></summary>\n",
    "<ul><li><a href=\"https://docs.cloud.oracle.com/en-us/iaas/data-science/using/data-science.htm\">Data Science Service Documentation</a></li>\n",
    "<li><a href=\"https://docs.cloud.oracle.com/iaas/tools/ads-sdk/latest/index.html\">ADS documentation</a></li>\n",
    "</ul>\n",
    "</details>\n",
    "<details>\n",
    "<summary><font size=\"2\">Typical Cell Imports and Settings for ADS</font></summary>\n",
    "\n",
    "```python\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "import logging\n",
    "logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.ERROR)\n",
    "\n",
    "import ads\n",
    "from ads.dataset.factory import DatasetFactory\n",
    "from ads.automl.provider import OracleAutoMLProvider\n",
    "from ads.automl.driver import AutoML\n",
    "from ads.evaluations.evaluator import ADSEvaluator\n",
    "from ads.common.data import ADSData\n",
    "from ads.explanations.explainer import ADSExplainer\n",
    "from ads.explanations.mlx_global_explainer import MLXGlobalExplainer\n",
    "from ads.explanations.mlx_local_explainer import MLXLocalExplainer\n",
    "from ads.catalog.model import ModelCatalog\n",
    "from ads.common.model_artifact import ModelArtifact\n",
    "```\n",
    "</details>\n",
    "<details>\n",
    "<summary><font size=\"2\">Useful Environment Variables</font></summary>\n",
    "\n",
    "```python\n",
    "import os\n",
    "print(os.environ[\"NB_SESSION_COMPARTMENT_OCID\"])\n",
    "print(os.environ[\"PROJECT_OCID\"])\n",
    "print(os.environ[\"USER_OCID\"])\n",
    "print(os.environ[\"TENANCY_OCID\"])\n",
    "print(os.environ[\"NB_REGION\"])\n",
    "```\n",
    "</details>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6e358b01",
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [],
   "source": [
    "import oci\n",
    "import ads\n",
    "import os\n",
    "from os import environ"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f841d08d",
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [],
   "source": [
    "if 'NB_SESSION_COMPARTMENT_OCID' in os.environ:\n",
    "    # using the notebook's compartment if running on OCI Data Science Notebook Session\n",
    "    compartment_id = os.environ['NB_SESSION_COMPARTMENT_OCID']\n",
    "else:\n",
    "    # set the compartment OCID if you are working locally\n",
    "    compartment_id = '<YOUR_COMPARTMENT_OCID>'\n",
    "print('compartment OCID = ', compartment_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ec3398d8",
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [],
   "source": [
    "# this will work whether working on a local machine (using API key) or in OCI Notebook Session (using Resource Principal)\n",
    "if \"OCI_RESOURCE_PRINCIPAL_VERSION\" in os.environ:\n",
    "    # Use resource principal\n",
    "    print(\"using Resource Principal for auth\")\n",
    "    ads.set_auth(auth=\"resource_principal\")\n",
    "else:\n",
    "    # Use api_key with config file\n",
    "    print(\"using API key for auth\")\n",
    "    ads.set_auth(auth=\"api_key\")    "
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "cc038dc6",
   "metadata": {},
   "source": [
    "Fill in your resources details:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c4e7b297",
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [],
   "source": [
    "project_id = '<YOUR_PROJECT_ID>'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "57b24379",
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [],
   "source": [
    "log_group_id = \"<YOUR_LOG_GROUP_ID>\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9a15c90c",
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [],
   "source": [
    "# create a data science client to communicate with the service\n",
    "config = oci.config.from_file()\n",
    "data_science_client = oci.data_science.DataScienceClient(config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "91fbee12",
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [],
   "source": [
    "# list all pipelines in the project\n",
    "res = data_science_client.list_pipelines(compartment_id=compartment_id)\n",
    "print(res.data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d139e5f6",
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [],
   "source": [
    "from random import randrange\n",
    "pipeline_name = f\"pipeline_sample_employee-attrition-{randrange(1000,9999)}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9d6b1b51",
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [],
   "source": [
    "# create a new pipeline (service conda pack)\n",
    "pipeline_payload = {\n",
    "    \"projectId\": project_id,\n",
    "    \"compartmentId\": compartment_id,\n",
    "    \"displayName\": pipeline_name,\n",
    "    \"infrastructureConfigurationDetails\": {\n",
    "        \"shapeName\": \"VM.Standard2.4\",\n",
    "        \"blockStorageSizeInGBs\": \"50\"\n",
    "    },\n",
    "    \"logConfigurationDetails\": {\n",
    "        \"enableLogging\": True,\n",
    "        \"logGroupId\": log_group_id,\n",
    "        \"enableAutoLogCreation\": True   # log will be automatically created\n",
    "    },\n",
    "    \"configurationDetails\": {\n",
    "        \"type\": \"DEFAULT\",\n",
    "        \"maximumRuntimeInMinutes\": 60,\n",
    "        \"environmentVariables\": {\n",
    "            \"CONDA_ENV_TYPE\": \"service\",\n",
    "            \"CONDA_ENV_SLUG\": \"generalml_p37_cpu_v1\"\n",
    "        }\n",
    "    },\n",
    "    \"stepDetails\": [\n",
    "        {\n",
    "            \"stepName\": \"data_processing\",\n",
    "            \"description\": \"Import data, feature engineering, train-test split\",\n",
    "            \"stepType\": \"CUSTOM_SCRIPT\",\n",
    "            \"stepInfrastructureConfigurationDetails\": {\n",
    "                \"shapeName\": \"VM.Standard2.4\",\n",
    "                \"blockStorageSizeInGBs\": \"50\"\n",
    "            },\n",
    "            \"stepConfigurationDetails\": {\n",
    "                \"type\": \"DEFAULT\",\n",
    "                \"maximumRuntimeInMinutes\": 30,\n",
    "                \"environmentVariables\": {\n",
    "                    \"PIPELINE_STEP_RUN_ENTRYPOINT\": \"employee-attr-dataproc.py\",\n",
    "                    \"CONDA_ENV_TYPE\": \"service\",\n",
    "                    \"CONDA_ENV_SLUG\": \"generalml_p37_cpu_v1\"\n",
    "                }\n",
    "            }\n",
    "        },\n",
    "        {\n",
    "            \"stepName\": \"train_logistic_regression\",\n",
    "            \"description\": \"Train a Logistic Regression model and save to the model catalog with its AUC score\",\n",
    "            \"stepType\": \"CUSTOM_SCRIPT\",\n",
    "            \"stepInfrastructureConfigurationDetails\": {\n",
    "                \"shapeName\": \"VM.Standard2.4\",\n",
    "                \"blockStorageSizeInGBs\": \"50\"\n",
    "            },\n",
    "            \"stepConfigurationDetails\": {\n",
    "                \"type\": \"DEFAULT\",\n",
    "                \"maximumRuntimeInMinutes\": 120,\n",
    "                \"environmentVariables\": {\n",
    "                    \"PIPELINE_STEP_RUN_ENTRYPOINT\": \"employee-attr-train-lr.py\",\n",
    "                    \"CONDA_ENV_TYPE\": \"service\",\n",
    "                    \"CONDA_ENV_SLUG\": \"generalml_p37_cpu_v1\"\n",
    "                }\n",
    "            },\n",
    "            \"dependsOn\": [\"data_processing\"]\n",
    "        },\n",
    "        {\n",
    "            \"stepName\": \"train_random_forest\",\n",
    "            \"description\": \"Train a Random Forest model and save to the model catalog with its AUC score\",\n",
    "            \"stepType\": \"CUSTOM_SCRIPT\",\n",
    "            \"stepInfrastructureConfigurationDetails\": {\n",
    "                \"shapeName\": \"VM.Standard2.4\",\n",
    "                \"blockStorageSizeInGBs\": \"50\"\n",
    "            },\n",
    "            \"stepConfigurationDetails\": {\n",
    "                \"type\": \"DEFAULT\",\n",
    "                \"maximumRuntimeInMinutes\": 120,\n",
    "                \"environmentVariables\": {\n",
    "                    \"PIPELINE_STEP_RUN_ENTRYPOINT\": \"employee-attr-train-rf.py\",\n",
    "                    \"CONDA_ENV_TYPE\": \"service\",\n",
    "                    \"CONDA_ENV_SLUG\": \"generalml_p37_cpu_v1\"\n",
    "                }\n",
    "            },\n",
    "            \"dependsOn\": [\"data_processing\"]\n",
    "        },\n",
    "        {\n",
    "            \"stepName\": \"train_xgboost\",\n",
    "            \"description\": \"Train a model with XGBoost and save to the model catalog with its AUC score\",\n",
    "            \"stepType\": \"CUSTOM_SCRIPT\",\n",
    "            \"stepInfrastructureConfigurationDetails\": {\n",
    "                \"shapeName\": \"VM.Standard2.4\",\n",
    "                \"blockStorageSizeInGBs\": \"50\"\n",
    "            },\n",
    "            \"stepConfigurationDetails\": {\n",
    "                \"type\": \"DEFAULT\",\n",
    "                \"maximumRuntimeInMinutes\": 120,\n",
    "                \"environmentVariables\": {\n",
    "                    \"PIPELINE_STEP_RUN_ENTRYPOINT\": \"employee-attr-train-xgb.py\",\n",
    "                    \"CONDA_ENV_TYPE\": \"service\",\n",
    "                    \"CONDA_ENV_SLUG\": \"generalml_p37_cpu_v1\"\n",
    "                }\n",
    "            },\n",
    "            \"dependsOn\": [\"data_processing\"]\n",
    "        },\n",
    "        {\n",
    "            \"stepName\": \"evaluate_and_deploy\",\n",
    "            \"description\": \"find the best model by their AUC score and deploy\",\n",
    "            \"stepType\": \"CUSTOM_SCRIPT\",\n",
    "            \"stepInfrastructureConfigurationDetails\": {\n",
    "                \"shapeName\": \"VM.Standard2.4\",\n",
    "                \"blockStorageSizeInGBs\": \"50\"\n",
    "            },\n",
    "            \"stepConfigurationDetails\": {\n",
    "                \"type\": \"DEFAULT\",\n",
    "                \"maximumRuntimeInMinutes\": 30,\n",
    "                \"environmentVariables\": {\n",
    "                    \"PIPELINE_STEP_RUN_ENTRYPOINT\": \"employee-attr-eval-deploy.py\",\n",
    "                    \"CONDA_ENV_TYPE\": \"service\",\n",
    "                    \"CONDA_ENV_SLUG\": \"generalml_p37_cpu_v1\"\n",
    "                }\n",
    "            },\n",
    "            \"dependsOn\": [\"train_logistic_regression\", \"train_random_forest\", \"train_xgboost\"]\n",
    "        }\n",
    "    ],\n",
    "    \"freeformTags\": {\n",
    "        \"freeTags\": \"employee-attrition-sample\"\n",
    "    }\n",
    "}\n",
    "pipeline_res = data_science_client.create_pipeline(pipeline_payload)\n",
    "pipeline_id = pipeline_res.data.id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d805a1eb",
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [],
   "source": [
    "print(pipeline_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ccd72f03",
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [],
   "source": [
    "# the pipeline will be in CREATING state until all steps have their artifacts uploaded\n",
    "print(pipeline_res.data.lifecycle_state)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "de8843b4",
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [],
   "source": [
    "steps_names_and_artifacts = {\"data_processing\":\"employee-attr-dataproc.zip\",\n",
    "                             \"train_logistic_regression\":\"employee-attr-train-lr.zip\",\n",
    "                             \"train_random_forest\":\"employee-attr-train-rf.zip\",\n",
    "                             \"train_xgboost\":\"employee-attr-train-xgb.zip\",\n",
    "                             \"evaluate_and_deploy\":\"employee-attr-eval-deploy.zip\"}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "54751185",
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [],
   "source": [
    "# upload steps artifacts\n",
    "for name in steps_names_and_artifacts:\n",
    "    fl = open(steps_names_and_artifacts[name], \"rb\")\n",
    "    ret = data_science_client.create_step_artifact(pipeline_id, name, fl, content_disposition=f\"attachment; filename={steps_names_and_artifacts[name]}\")\n",
    "    print(f\"{steps_names_and_artifacts[name]}: OK\" if ret.status==204 else f\"{steps_names_and_artifacts[name]}: {ret.status}\")  # 204 is ok"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c2e6582c",
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [],
   "source": [
    "# pipeline should be in ACTIVE state now\n",
    "res = data_science_client.get_pipeline(pipeline_id)\n",
    "print(res.data.lifecycle_state)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "50e76349",
   "metadata": {},
   "source": [
    "## Run the pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e1108690",
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [],
   "source": [
    "# set the configuration and the environment variables for the run\n",
    "pipeline_run_name = f\"pipeline-run-{randrange(1000,9999)}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ac586f6b",
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [],
   "source": [
    "# uncomment the next line and set to the OCI Object Storage location to use for passing data. Make sure you have proper permissions.\n",
    "#data_location = \"<YOUR_OBJECT_STORAGE_BUCKET>\"  # use: 'oci://<bucket>@<workspace>/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "39df7664",
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [],
   "source": [
    "pipeline_run_payload = {\n",
    "    \"projectId\": project_id,\n",
    "    \"displayName\": pipeline_run_name,\n",
    "    \"pipelineId\": pipeline_id,\n",
    "    \"compartmentId\": compartment_id,\n",
    "    \"configurationOverrideDetails\": {\n",
    "        \"type\": \"DEFAULT\",\n",
    "        \"environmentVariables\": {\n",
    "            \"DATA_LOCATION\": data_location,   # provide the data location to the run\n",
    "            \"CLEANUP_RESOURCES\": \"True\"       # delete the models from the model catalog and delete the model deployment\n",
    "        }\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "503e4cf7",
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [],
   "source": [
    "pipelinerun_res = data_science_client.create_pipeline_run(pipeline_run_payload)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "04db5f17",
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [],
   "source": [
    "# check pipeline run status\n",
    "run_status = data_science_client.get_pipeline_run(pipelinerun_res.data.id)\n",
    "print(run_status.data.lifecycle_state)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "27b093bd",
   "metadata": {},
   "source": [
    "## Run the pipeline from the console UI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bcfa279a",
   "metadata": {
    "vscode": {
     "languageId": "python"
    }
   },
   "outputs": [],
   "source": [
    "print(\"Ctrl-Click the hyperlink to open the pipeline run page in the OCI console UI\")\n",
    "print(\"https://cloud.oracle.com/data-science/pipelines/{}/pipeline-runs\".format(pipeline_id))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ab60fb18",
   "metadata": {},
   "source": [
    "#### Don't forget to set environment varaibles when running the pipeline: DATA_LOCATION"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}