data_cleaning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# **Wildfire Risk Assesment** \n",
    "# Data Cleaning and general pre-processing\n",
    "\n",
    "This code pre-processes the forest fire data, performing tasks as dropping unwanted columns, filtering down the scope of data in terms of years and the fire reporting agencies to only provinces and also imputing missing/unknown fire causes. The cleaned data is then exported to a CSV file for further problem modelling.\n",
    "\n",
    "### Input Required:\n",
    "- **forest_fire.txt**: A txt file containing the data about all the forest fires in Canada for different provinces and national parks.\n",
    "\n",
    "### Output Generated:\n",
    "- **fire_data_cleaned.csv**: A cleaned and processed version of the input data, which is used for the problem in the next part."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/_1/slbty4054pvb722zx16dmj7w0000gn/T/ipykernel_84927/1322022443.py:1: DtypeWarning: Columns (10,11,24) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  data = pd.read_csv('forest_fires.txt', sep=\",\")\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>FID</th>\n",
       "      <th>SRC_AGENCY</th>\n",
       "      <th>FIRE_ID</th>\n",
       "      <th>FIRENAME</th>\n",
       "      <th>LATITUDE</th>\n",
       "      <th>LONGITUDE</th>\n",
       "      <th>YEAR</th>\n",
       "      <th>MONTH</th>\n",
       "      <th>DAY</th>\n",
       "      <th>REP_DATE</th>\n",
       "      <th>...</th>\n",
       "      <th>MORE_INFO</th>\n",
       "      <th>CFS_REF_ID</th>\n",
       "      <th>CFS_NOTE1</th>\n",
       "      <th>CFS_NOTE2</th>\n",
       "      <th>ACQ_DATE</th>\n",
       "      <th>SRC_AGY2</th>\n",
       "      <th>ECOZONE</th>\n",
       "      <th>ECOZ_REF</th>\n",
       "      <th>ECOZ_NAME</th>\n",
       "      <th>ECOZ_NOM</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>BC</td>\n",
       "      <td>1953-G00041</td>\n",
       "      <td></td>\n",
       "      <td>59.963</td>\n",
       "      <td>-128.172</td>\n",
       "      <td>1953</td>\n",
       "      <td>5</td>\n",
       "      <td>26</td>\n",
       "      <td>1953-05-26 00:00:00</td>\n",
       "      <td>...</td>\n",
       "      <td></td>\n",
       "      <td>BC-1953-1953-G00041</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>2020-05-05 00:00:00</td>\n",
       "      <td>BC</td>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "      <td>Boreal Cordillera</td>\n",
       "      <td>CordillCre boreale</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>BC</td>\n",
       "      <td>1950-R00028</td>\n",
       "      <td></td>\n",
       "      <td>59.318</td>\n",
       "      <td>-132.172</td>\n",
       "      <td>1950</td>\n",
       "      <td>6</td>\n",
       "      <td>22</td>\n",
       "      <td>1950-06-22 00:00:00</td>\n",
       "      <td>...</td>\n",
       "      <td></td>\n",
       "      <td>BC-1950-1950-R00028</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>2020-05-05 00:00:00</td>\n",
       "      <td>BC</td>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "      <td>Boreal Cordillera</td>\n",
       "      <td>CordillCre boreale</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>BC</td>\n",
       "      <td>1950-G00026</td>\n",
       "      <td></td>\n",
       "      <td>59.876</td>\n",
       "      <td>-131.922</td>\n",
       "      <td>1950</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "      <td>1950-06-04 00:00:00</td>\n",
       "      <td>...</td>\n",
       "      <td></td>\n",
       "      <td>BC-1950-1950-G00026</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>2020-05-05 00:00:00</td>\n",
       "      <td>BC</td>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "      <td>Boreal Cordillera</td>\n",
       "      <td>CordillCre boreale</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>BC</td>\n",
       "      <td>1951-R00097</td>\n",
       "      <td></td>\n",
       "      <td>59.760</td>\n",
       "      <td>-132.808</td>\n",
       "      <td>1951</td>\n",
       "      <td>7</td>\n",
       "      <td>15</td>\n",
       "      <td>1951-07-15 00:00:00</td>\n",
       "      <td>...</td>\n",
       "      <td></td>\n",
       "      <td>BC-1951-1951-R00097</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>2020-05-05 00:00:00</td>\n",
       "      <td>BC</td>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "      <td>Boreal Cordillera</td>\n",
       "      <td>CordillCre boreale</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>BC</td>\n",
       "      <td>1952-G00116</td>\n",
       "      <td></td>\n",
       "      <td>59.434</td>\n",
       "      <td>-126.172</td>\n",
       "      <td>1952</td>\n",
       "      <td>6</td>\n",
       "      <td>12</td>\n",
       "      <td>1952-06-12 00:00:00</td>\n",
       "      <td>...</td>\n",
       "      <td></td>\n",
       "      <td>BC-1952-1952-G00116</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>2020-05-05 00:00:00</td>\n",
       "      <td>BC</td>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "      <td>Boreal Cordillera</td>\n",
       "      <td>CordillCre boreale</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 27 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   FID SRC_AGENCY      FIRE_ID FIRENAME  LATITUDE  LONGITUDE  YEAR  MONTH  \\\n",
       "0    0         BC  1953-G00041             59.963   -128.172  1953      5   \n",
       "1    1         BC  1950-R00028             59.318   -132.172  1950      6   \n",
       "2    2         BC  1950-G00026             59.876   -131.922  1950      6   \n",
       "3    3         BC  1951-R00097             59.760   -132.808  1951      7   \n",
       "4    4         BC  1952-G00116             59.434   -126.172  1952      6   \n",
       "\n",
       "   DAY             REP_DATE  ... MORE_INFO           CFS_REF_ID CFS_NOTE1  \\\n",
       "0   26  1953-05-26 00:00:00  ...            BC-1953-1953-G00041             \n",
       "1   22  1950-06-22 00:00:00  ...            BC-1950-1950-R00028             \n",
       "2    4  1950-06-04 00:00:00  ...            BC-1950-1950-G00026             \n",
       "3   15  1951-07-15 00:00:00  ...            BC-1951-1951-R00097             \n",
       "4   12  1952-06-12 00:00:00  ...            BC-1952-1952-G00116             \n",
       "\n",
       "   CFS_NOTE2             ACQ_DATE SRC_AGY2 ECOZONE ECOZ_REF  \\\n",
       "0             2020-05-05 00:00:00       BC      12       12   \n",
       "1             2020-05-05 00:00:00       BC      12       12   \n",
       "2             2020-05-05 00:00:00       BC      12       12   \n",
       "3             2020-05-05 00:00:00       BC      12       12   \n",
       "4             2020-05-05 00:00:00       BC      12       12   \n",
       "\n",
       "           ECOZ_NAME            ECOZ_NOM  \n",
       "0  Boreal Cordillera  CordillCre boreale  \n",
       "1  Boreal Cordillera  CordillCre boreale  \n",
       "2  Boreal Cordillera  CordillCre boreale  \n",
       "3  Boreal Cordillera  CordillCre boreale  \n",
       "4  Boreal Cordillera  CordillCre boreale  \n",
       "\n",
       "[5 rows x 27 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv('forest_fires.txt', sep=\",\")\n",
    "\n",
    "data.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Removing unwanted columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "remove_cols = ['FIRENAME', 'ATTK_DATE', 'DECADE', 'PROTZONE', 'MORE_INFO', 'CFS_REF_ID', 'CFS_NOTE1', 'CFS_NOTE2', 'ACQ_DATE', 'SRC_AGY2', 'FIRE_TYPE']\n",
    "\n",
    "fire_data = data.drop(columns=remove_cols)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Using data from year 2015 onwards"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "fire_data = fire_data[fire_data['YEAR'] >= 2015]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SRC_AGENCY\n",
       "BC          9520\n",
       "AB          8983\n",
       "ON          5772\n",
       "QC          3583\n",
       "SK          3001\n",
       "MB          2617\n",
       "NB          1859\n",
       "NS          1313\n",
       "NT          1108\n",
       "YT           657\n",
       "NL           532\n",
       "PC-BA        193\n",
       "PC-WB        160\n",
       "PC-JA         78\n",
       "PC-GL         50\n",
       "PC-KO         43\n",
       "PC-NA         37\n",
       "PC-GR         34\n",
       "PC-WL         33\n",
       "PC-RM         30\n",
       "PC-PA         29\n",
       "PC-LM         28\n",
       "PC-YO         24\n",
       "PC-PP         21\n",
       "PC-RE         17\n",
       "PC-CB         10\n",
       "PC-TI         10\n",
       "PC-TN         10\n",
       "PC-WP          8\n",
       "PC-EI          7\n",
       "PC-BP          6\n",
       "PC-GI          6\n",
       "PC-PU          4\n",
       "PC-FO          4\n",
       "PC-RE-GL       3\n",
       "PC-FU          3\n",
       "PC-KG          3\n",
       "PC-KE          3\n",
       "PC-MM          2\n",
       "PC-RO          2\n",
       "PC-TH          2\n",
       "PC-KL          2\n",
       "PC-VU          2\n",
       "PC-GF          1\n",
       "PC-LO          1\n",
       "PC-PR          1\n",
       "PC-SE          1\n",
       "PC-NC          1\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fire_data['SRC_AGENCY'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Keeping the SRC_AGENCY restricted to the Provinces and removing the National Parks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(38945, 16)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mask = fire_data['SRC_AGENCY'].str.startswith('PC')\n",
    "\n",
    "fire_data = fire_data[~mask]\n",
    "\n",
    "fire_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CAUSE\n",
       "H       20252\n",
       "L       17762\n",
       "U         815\n",
       "H-PB        7\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fire_data['CAUSE'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Dropping observations with H-PB as fire CAUSE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "fire_data = fire_data[fire_data['CAUSE'] != 'H-PB']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CAUSE\n",
       "H    20252\n",
       "L    17762\n",
       "U      815\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fire_data['CAUSE'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Checking if there are any null values in the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "FID               0\n",
       "SRC_AGENCY        0\n",
       "FIRE_ID           0\n",
       "LATITUDE          0\n",
       "LONGITUDE         0\n",
       "YEAR              0\n",
       "MONTH             0\n",
       "DAY               0\n",
       "REP_DATE          2\n",
       "OUT_DATE      14342\n",
       "SIZE_HA           0\n",
       "CAUSE           109\n",
       "ECOZONE           0\n",
       "ECOZ_REF          0\n",
       "ECOZ_NAME         0\n",
       "ECOZ_NOM          0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fire_data.isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Dropping observations wherer the REP_DATE (Reporting Date) column is null"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "fire_data = fire_data.dropna(subset=['REP_DATE'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "FID               0\n",
       "SRC_AGENCY        0\n",
       "FIRE_ID           0\n",
       "LATITUDE          0\n",
       "LONGITUDE         0\n",
       "YEAR              0\n",
       "MONTH             0\n",
       "DAY               0\n",
       "REP_DATE          0\n",
       "OUT_DATE      14340\n",
       "SIZE_HA           0\n",
       "CAUSE           109\n",
       "ECOZONE           0\n",
       "ECOZ_REF          0\n",
       "ECOZ_NAME         0\n",
       "ECOZ_NOM          0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fire_data.isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Filling the null values in the CAUSE column to 'U'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# fill null values with U for column CAUSE\n",
    "fire_data['CAUSE'] = fire_data['CAUSE'].fillna('U')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SRC_AGENCY\n",
       "BC    9520\n",
       "AB    8983\n",
       "ON    5765\n",
       "QC    3583\n",
       "SK    3001\n",
       "MB    2617\n",
       "NB    1859\n",
       "NS    1313\n",
       "NT    1107\n",
       "YT     657\n",
       "NL     531\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fire_data['SRC_AGENCY'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Export fire_data to new csv file\n",
    "fire_data.to_csv('fire_data_cleaned.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}