fetcher.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "fetcher.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": 1
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/RedevLtd/ml-colab-test-1/blob/master/fetcher.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "soexiboTGV3W",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import datetime\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AErRS7PmEtlh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "dateparselambda  = lambda x : (read_csv_date(x))\n",
        "\n",
        "def read_csv_date( x ):\n",
        "    try:\n",
        "        if len(x) > 8:\n",
        "            return pd.datetime.strptime(x, \"%d/%m/%Y\")\n",
        "        else:\n",
        "            return pd.datetime.strptime(x, \"%d/%m/%y\")\n",
        "    except:\n",
        "        return datetime.datetime(1970,1,1,0,0)\n",
        "\n",
        "def get_raw_dataframe():\n",
        "  url_base='https://raw.githubusercontent.com/RedevLtd/ml-colab-test-1/master/football_data/' # Read from github repository \n",
        "  start_year=1\n",
        "  end_year=16\n",
        "  leagues = [\"E0\", \"E1\", \"E2\", \"E3\"]\n",
        "  #name_list=['Div','Date','HomeTeam','AwayTeam','FTHG','FTAG','FTR','HTHG','HTAG','HTR','Attendance',\n",
        "  #            'Referee','HS','AS','HST','AST','HHW','AHW','HC','AC','HF','AF','HO','AO','HY','AY','HR',\n",
        "  #            'AR','HBP','ABP','GBH','GBD','GBA','IWH','IWD','IWA','LBH','LBD','LBA','SBH','SBD','SBA',\n",
        " #             'SYH',\t'SYD',\t'SYA','WHH','WHD','WHA']\n",
        "  fields=[\"Div\",\"Date\",\"HomeTeam\",\"AwayTeam\",\"FTHG\",\"FTAG\",\"FTR\",\"HTHG\",\"HTAG\",\"HTR\",\"WHH\",\"WHD\",\"WHA\"]\n",
        "  df=pd.DataFrame()\n",
        "  for year in range(start_year,end_year):\n",
        "    season= format( year, '02d') + format( year+1, '02d')\n",
        "    for league in leagues:\n",
        "      url=url_base+ league +'_'+str(season)+'.csv'\n",
        "      temp_df=pd.read_csv(url,encoding = \"ISO-8859-1\",date_parser=dateparselambda,\n",
        "                          keep_default_na=True,na_filter=True,na_values='',usecols=fields)\n",
        "      df=pd.concat([df,temp_df],axis=0)\n",
        "  df=df[1:df.shape[0]] # Delete the first row of header titles\n",
        "  return df\n"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}