template.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Шаблон решения\n",
    "* Простой способ разработать своё первое решение и сгенерировать файл на проверку\n",
    "* Дублирует функции из baseline\n",
    "* Следовать ему не обязательно"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from matplotlib import pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Прочитаем данные "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import datanight\n",
    "datanight.download_datasets()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data = pd.DataFrame.from_csv(\"./train.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "x_col = [ 'dist','due', 'lat','lon','f_class','s_class','t_class',]\n",
    "X_raw = data[x_col]\n",
    "y = data['burned'].values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Предобработка данных"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction import DictVectorizer\n",
    "\n",
    "#вытаскиватель категориальных фичей\n",
    "vectorizer = DictVectorizer(sparse=False,dtype=np.bool)\n",
    "\n",
    "def preprocess_data(X_raw):\n",
    "    \n",
    "    <Как нибудь предобработать данные>\n",
    "    \n",
    "    return <предобработанные данные>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X = preprocess_data(X_raw)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Поделим на обучение и валидацию\n",
    "* Важно, что мы делим не случайно, а по времени, т.к. тестовые данные по времени дальше обучающих"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Как-нибудь порезать данные\n",
    "Xtr = X[:1000000]\n",
    "Xval = X[1000000:]\n",
    "Ytr = y[:1000000]\n",
    "Yval = y[1000000:]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Обучим модель"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#обучение и тест\n",
    "pivot = int(len(X) * 0.75)\n",
    "Xtr = X[:pivot]\n",
    "Xval = X[pivot:]\n",
    "Ytr = y[:pivot]\n",
    "Yval = y[pivot:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Оценим качество"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_auc_score,accuracy_score,precision_score,recall_score\n",
    "from sklearn.metrics import roc_curve\n",
    "\n",
    "\n",
    "for source_i, Xi,Yi in [\n",
    "                            [\"train\",Xtr,Ytr],\n",
    "                            [\"val\",Xval,Yval]\n",
    "                                ]:\n",
    "    \n",
    "    # Предскажем вероятность сгорания\n",
    "    Yi_pred_proba = model.predict_proba(Xi)[:,1]\n",
    "    \n",
    "    #Поделим предсказание на сгоревшие и не сгоревшие\n",
    "    Yi_pred_class = np.argsort(Yi_pred_proba) < 10000\n",
    "    \n",
    "    auc = roc_auc_score(Yi,Yi_pred_proba)\n",
    "    acc = accuracy_score(Yi,Yi_pred_class)\n",
    "    \n",
    "    print '%s: \\t AUC = %.5f \\t Accuracy = %.5f'%(source_i, auc, acc)\n",
    "    \n",
    "    fpr,tpr,_ = roc_curve(Yi,Yi_pred_proba)\n",
    "    \n",
    "    plt.plot(fpr,tpr,label = source_i)\n",
    "    \n",
    "plt.legend()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "datanight.plot_forest_feature_importances(model,X.columns)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Отгрузим решение в файл"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "datanight.generate_submission(preprocess_data,model,\"my_submission.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "!head my_submission.csv"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}