house_reallocation_under_criteria.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c25e675b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "0106f670",
   "metadata": {},
   "outputs": [],
   "source": [
    "s = \"\"\"House, Occupancy, Construction Year, Appliances Owned, Type, Size\n",
    "1\t,\t2\t,\t1975-1980\t\t\t\t, 35 , Detached\t\t\t, 4 bed\n",
    "2\t,\t4\t,\t-\t\t\t\t\t\t, 15 , Semi-detached\t, 3 bed\n",
    "3\t,\t2\t,\t1988\t\t\t\t\t, 27 , Detached\t\t\t, 3 bed\n",
    "4\t,\t2\t,\t1850-1899 \t\t\t\t, 33 , Detached\t\t\t, 4 bed\n",
    "5\t,\t4\t,\t1878\t\t\t\t\t, 44 , Mid-terrace\t\t, 4 bed\n",
    "6\t,\t2\t,\t2005\t\t\t\t\t, 49 , Detached\t\t\t, 4 bed\n",
    "7\t,\t4\t,\t1965-1974\t\t\t\t, 25 , Detached\t\t\t, 3 bed\n",
    "8\t,\t2\t,\t1966\t\t\t\t\t, 35 , Detached\t\t\t, 2 bed\n",
    "9\t,\t2\t,\t1919-1944\t\t\t\t, 24 , Detached\t\t\t, 3 bed\n",
    "10\t,\t4\t,\t1919-1944\t\t\t\t, 31 , Detached\t\t\t, 3 bed\n",
    "11\t,\t1\t,\t1945-1964\t\t\t\t, 25 , Detached\t\t\t, 3 bed\n",
    "12\t,\t3\t,\t1991-1995\t\t\t\t, 26 , Detached\t\t\t, 3 bed\n",
    "13\t,\t4\t,\tpost 2002\t\t\t\t, 28 , Detached\t\t\t, 4 bed\n",
    "15\t,\t1\t,\t1965-1974\t\t\t\t, 19 , Semi-detached\t, 3 bed\n",
    "16\t,\t6\t,\t1981-1990\t\t\t\t, 48 , Detached\t\t\t, 5 bed\n",
    "17\t,\t3\t,\tmid 60s\t\t\t\t\t, 22 , Detached\t\t\t, 3 bed\n",
    "18\t,\t2\t,\t1965-1974\t\t\t\t, 34 , Detached\t\t\t, 3 bed\n",
    "19\t,\t4\t,\t1945-1964\t\t\t\t, 26 , Semi-detached\t, 3 bed\n",
    "20\t,\t2\t,\t1965-1974\t\t\t\t, 39 , Detached\t\t\t, 3 bed\n",
    "21\t,\t4\t,\t1981-1990\t\t\t\t, 23 , Detached\t\t\t, 3 bed\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b801ad9a",
   "metadata": {},
   "outputs": [],
   "source": [
    "l = [x.replace('\\t', '').replace(' ', '').split(',') for x in s.split('\\n')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e7ec10a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(l[1:], columns = l[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "95949dd5",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[~df['House'].isin({'12', '13'})]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "28f6aa3c",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['AppliancesOwned'] = df['AppliancesOwned'].astype(int)\n",
    "df['Occupancy'] = df['Occupancy'].astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "e11cdc5d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>House</th>\n",
       "      <th>Occupancy</th>\n",
       "      <th>ConstructionYear</th>\n",
       "      <th>AppliancesOwned</th>\n",
       "      <th>Type</th>\n",
       "      <th>Size</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1975-1980</td>\n",
       "      <td>35</td>\n",
       "      <td>Detached</td>\n",
       "      <td>4bed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>-</td>\n",
       "      <td>15</td>\n",
       "      <td>Semi-detached</td>\n",
       "      <td>3bed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>1988</td>\n",
       "      <td>27</td>\n",
       "      <td>Detached</td>\n",
       "      <td>3bed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>1850-1899</td>\n",
       "      <td>33</td>\n",
       "      <td>Detached</td>\n",
       "      <td>4bed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>1878</td>\n",
       "      <td>44</td>\n",
       "      <td>Mid-terrace</td>\n",
       "      <td>4bed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>2005</td>\n",
       "      <td>49</td>\n",
       "      <td>Detached</td>\n",
       "      <td>4bed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7</td>\n",
       "      <td>4</td>\n",
       "      <td>1965-1974</td>\n",
       "      <td>25</td>\n",
       "      <td>Detached</td>\n",
       "      <td>3bed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>1966</td>\n",
       "      <td>35</td>\n",
       "      <td>Detached</td>\n",
       "      <td>2bed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>1919-1944</td>\n",
       "      <td>24</td>\n",
       "      <td>Detached</td>\n",
       "      <td>3bed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>10</td>\n",
       "      <td>4</td>\n",
       "      <td>1919-1944</td>\n",
       "      <td>31</td>\n",
       "      <td>Detached</td>\n",
       "      <td>3bed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "      <td>1945-1964</td>\n",
       "      <td>25</td>\n",
       "      <td>Detached</td>\n",
       "      <td>3bed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>1965-1974</td>\n",
       "      <td>19</td>\n",
       "      <td>Semi-detached</td>\n",
       "      <td>3bed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>16</td>\n",
       "      <td>6</td>\n",
       "      <td>1981-1990</td>\n",
       "      <td>48</td>\n",
       "      <td>Detached</td>\n",
       "      <td>5bed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>17</td>\n",
       "      <td>3</td>\n",
       "      <td>mid60s</td>\n",
       "      <td>22</td>\n",
       "      <td>Detached</td>\n",
       "      <td>3bed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>18</td>\n",
       "      <td>2</td>\n",
       "      <td>1965-1974</td>\n",
       "      <td>34</td>\n",
       "      <td>Detached</td>\n",
       "      <td>3bed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>19</td>\n",
       "      <td>4</td>\n",
       "      <td>1945-1964</td>\n",
       "      <td>26</td>\n",
       "      <td>Semi-detached</td>\n",
       "      <td>3bed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>20</td>\n",
       "      <td>2</td>\n",
       "      <td>1965-1974</td>\n",
       "      <td>39</td>\n",
       "      <td>Detached</td>\n",
       "      <td>3bed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>21</td>\n",
       "      <td>4</td>\n",
       "      <td>1981-1990</td>\n",
       "      <td>23</td>\n",
       "      <td>Detached</td>\n",
       "      <td>3bed</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   House  Occupancy ConstructionYear  AppliancesOwned           Type  Size\n",
       "0      1          2        1975-1980               35       Detached  4bed\n",
       "1      2          4                -               15  Semi-detached  3bed\n",
       "2      3          2             1988               27       Detached  3bed\n",
       "3      4          2        1850-1899               33       Detached  4bed\n",
       "4      5          4             1878               44    Mid-terrace  4bed\n",
       "5      6          2             2005               49       Detached  4bed\n",
       "6      7          4        1965-1974               25       Detached  3bed\n",
       "7      8          2             1966               35       Detached  2bed\n",
       "8      9          2        1919-1944               24       Detached  3bed\n",
       "9     10          4        1919-1944               31       Detached  3bed\n",
       "10    11          1        1945-1964               25       Detached  3bed\n",
       "13    15          1        1965-1974               19  Semi-detached  3bed\n",
       "14    16          6        1981-1990               48       Detached  5bed\n",
       "15    17          3           mid60s               22       Detached  3bed\n",
       "16    18          2        1965-1974               34       Detached  3bed\n",
       "17    19          4        1945-1964               26  Semi-detached  3bed\n",
       "18    20          2        1965-1974               39       Detached  3bed\n",
       "19    21          4        1981-1990               23       Detached  3bed"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1387fb9d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>House</th>\n",
       "      <th>Occupancy</th>\n",
       "      <th>ConstructionYear</th>\n",
       "      <th>AppliancesOwned</th>\n",
       "      <th>Type</th>\n",
       "      <th>Size</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1975-1980</td>\n",
       "      <td>35</td>\n",
       "      <td>Detached</td>\n",
       "      <td>4bed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>1850-1899</td>\n",
       "      <td>33</td>\n",
       "      <td>Detached</td>\n",
       "      <td>4bed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>1878</td>\n",
       "      <td>44</td>\n",
       "      <td>Mid-terrace</td>\n",
       "      <td>4bed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>2005</td>\n",
       "      <td>49</td>\n",
       "      <td>Detached</td>\n",
       "      <td>4bed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>1966</td>\n",
       "      <td>35</td>\n",
       "      <td>Detached</td>\n",
       "      <td>2bed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>10</td>\n",
       "      <td>4</td>\n",
       "      <td>1919-1944</td>\n",
       "      <td>31</td>\n",
       "      <td>Detached</td>\n",
       "      <td>3bed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>16</td>\n",
       "      <td>6</td>\n",
       "      <td>1981-1990</td>\n",
       "      <td>48</td>\n",
       "      <td>Detached</td>\n",
       "      <td>5bed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>18</td>\n",
       "      <td>2</td>\n",
       "      <td>1965-1974</td>\n",
       "      <td>34</td>\n",
       "      <td>Detached</td>\n",
       "      <td>3bed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>20</td>\n",
       "      <td>2</td>\n",
       "      <td>1965-1974</td>\n",
       "      <td>39</td>\n",
       "      <td>Detached</td>\n",
       "      <td>3bed</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   House  Occupancy ConstructionYear  AppliancesOwned         Type  Size\n",
       "0      1          2        1975-1980               35     Detached  4bed\n",
       "3      4          2        1850-1899               33     Detached  4bed\n",
       "4      5          4             1878               44  Mid-terrace  4bed\n",
       "5      6          2             2005               49     Detached  4bed\n",
       "7      8          2             1966               35     Detached  2bed\n",
       "9     10          4        1919-1944               31     Detached  3bed\n",
       "14    16          6        1981-1990               48     Detached  5bed\n",
       "16    18          2        1965-1974               34     Detached  3bed\n",
       "18    20          2        1965-1974               39     Detached  3bed"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df['AppliancesOwned'] > 30]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "67c161fa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>House</th>\n",
       "      <th>Occupancy</th>\n",
       "      <th>ConstructionYear</th>\n",
       "      <th>AppliancesOwned</th>\n",
       "      <th>Type</th>\n",
       "      <th>Size</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>-</td>\n",
       "      <td>15</td>\n",
       "      <td>Semi-detached</td>\n",
       "      <td>3bed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>1988</td>\n",
       "      <td>27</td>\n",
       "      <td>Detached</td>\n",
       "      <td>3bed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7</td>\n",
       "      <td>4</td>\n",
       "      <td>1965-1974</td>\n",
       "      <td>25</td>\n",
       "      <td>Detached</td>\n",
       "      <td>3bed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9</td>\n",
       "      <td>2</td>\n",
       "      <td>1919-1944</td>\n",
       "      <td>24</td>\n",
       "      <td>Detached</td>\n",
       "      <td>3bed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "      <td>1945-1964</td>\n",
       "      <td>25</td>\n",
       "      <td>Detached</td>\n",
       "      <td>3bed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>1965-1974</td>\n",
       "      <td>19</td>\n",
       "      <td>Semi-detached</td>\n",
       "      <td>3bed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>17</td>\n",
       "      <td>3</td>\n",
       "      <td>mid60s</td>\n",
       "      <td>22</td>\n",
       "      <td>Detached</td>\n",
       "      <td>3bed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>19</td>\n",
       "      <td>4</td>\n",
       "      <td>1945-1964</td>\n",
       "      <td>26</td>\n",
       "      <td>Semi-detached</td>\n",
       "      <td>3bed</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>21</td>\n",
       "      <td>4</td>\n",
       "      <td>1981-1990</td>\n",
       "      <td>23</td>\n",
       "      <td>Detached</td>\n",
       "      <td>3bed</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   House  Occupancy ConstructionYear  AppliancesOwned           Type  Size\n",
       "1      2          4                -               15  Semi-detached  3bed\n",
       "2      3          2             1988               27       Detached  3bed\n",
       "6      7          4        1965-1974               25       Detached  3bed\n",
       "8      9          2        1919-1944               24       Detached  3bed\n",
       "10    11          1        1945-1964               25       Detached  3bed\n",
       "13    15          1        1965-1974               19  Semi-detached  3bed\n",
       "15    17          3           mid60s               22       Detached  3bed\n",
       "17    19          4        1945-1964               26  Semi-detached  3bed\n",
       "19    21          4        1981-1990               23       Detached  3bed"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df['AppliancesOwned'] < 30]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "21160c0f",
   "metadata": {},
   "source": [
    "# Data permutations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "17f31fd2",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('data/refit_data_by_category.pk', 'rb') as f:\n",
    "    data = pickle.load(f)\n",
    "    \n",
    "train, test = data['train'], data['test']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "2405dc92",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(168, 13), (168, 13), (168, 13), (168, 13)]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[x.shape for x in list(train[0]['X'].values())]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "5911cb2a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def concatenate_data(train, test):\n",
    "    data = []\n",
    "    for week in range(len(train)):\n",
    "        d = dict()\n",
    "        d['X_bar'] = pd.concat([train[week]['X_bar'], test[week]['X_bar']], axis=1)\n",
    "        d['X'] = {cat: pd.concat([train[week]['X'][cat], test[week]['X'][cat]], axis=1) for cat in train[week]['X'].keys()}\n",
    "        data.append(d)\n",
    "        \n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ef190c5d",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = concatenate_data(train, test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "f428dc7a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def permute_concatenated_data(data, source_houses, target_houses_in_train, test_houses):\n",
    "    train, test = [], []\n",
    "    for week in range(len(data)):\n",
    "        d = dict()\n",
    "        d['X_bar'] = data[week]['X_bar'][source_houses + target_houses_in_train]\n",
    "        for house in source_houses:\n",
    "            if '_target' in house:\n",
    "                d['X_bar'] = d['X_bar'].rename(columns = {house: house.replace('_target', '')})\n",
    "        for house in target_houses_in_train:\n",
    "            if '_target' not in house:\n",
    "                d['X_bar'] = d['X_bar'].rename(columns = {house: house + '_target'})\n",
    "        \n",
    "        \n",
    "        d['X'] = {cat: data[week]['X'][cat][source_houses + target_houses_in_train] for cat in data[week]['X'].keys()}\n",
    "        for cat in d.keys():\n",
    "            for house in source_houses:\n",
    "                if '_target' in house:\n",
    "                    for cat in data[week]['X'].keys():\n",
    "                        d['X'][cat] = d['X'][cat].rename(columns = {house: house.replace('_target', '')})\n",
    "\n",
    "            for house in target_houses_in_train:\n",
    "                if '_target' not in house:\n",
    "                    for cat in data[week]['X'].keys():\n",
    "                        d['X'][cat] = d['X'][cat].rename(columns = {house: house + '_target'})\n",
    "\n",
    "        train.append(d)\n",
    "\n",
    "        d = dict()\n",
    "        d['X_bar'] = data[week]['X_bar'][test_houses]\n",
    "        for house in test_houses:\n",
    "            if '_target' in house:\n",
    "                d['X_bar'] = d['X_bar'].rename(columns = {house: house.replace('_target', '')})\n",
    "        \n",
    "        \n",
    "        d['X'] = {cat: data[week]['X'][cat][test_houses] for cat in data[week]['X'].keys()}\n",
    "        for cat in d.keys():\n",
    "            for house in test_houses:\n",
    "                if '_target' in house:\n",
    "                    for cat in data[week]['X'].keys():\n",
    "                        d['X'][cat] = d['X'][cat].rename(columns = {house: house.replace('_target', '')})\n",
    "\n",
    "\n",
    "        test.append(d)\n",
    "        \n",
    "            \n",
    "    return train, test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "98f7a526",
   "metadata": {},
   "outputs": [],
   "source": [
    "#len(source_houses + target_houses_in_train + test_houses), len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "6f396fbd",
   "metadata": {},
   "outputs": [],
   "source": [
    "#sorted(source_houses + target_houses_in_train + test_houses)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "55ff2d3b",
   "metadata": {},
   "source": [
    "### Less target train houses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "e3bcc78a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>house_21</th>\n",
       "      <th>house_3</th>\n",
       "      <th>house_10</th>\n",
       "      <th>house_5</th>\n",
       "      <th>house_7</th>\n",
       "      <th>house_16</th>\n",
       "      <th>house_8</th>\n",
       "      <th>house_9</th>\n",
       "      <th>house_2_target</th>\n",
       "      <th>house_15_target</th>\n",
       "      <th>house_18_target</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Unix</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1401793200</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1401796800</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1401800400</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1401804000</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1401807600</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            house_21  house_3  house_10  house_5  house_7  house_16  house_8  \\\n",
       "Unix                                                                           \n",
       "1401793200         0        0         0        2        0         1        0   \n",
       "1401796800         0        0         0        2        0         1        0   \n",
       "1401800400         0        0         0        2        0         1        0   \n",
       "1401804000         0        0         0        2        0         1        0   \n",
       "1401807600         0        0         0        2        0         0        0   \n",
       "\n",
       "            house_9  house_2_target  house_15_target  house_18_target  \n",
       "Unix                                                                   \n",
       "1401793200        4               2                0                1  \n",
       "1401796800        4               2                0                1  \n",
       "1401800400        4               2                0                1  \n",
       "1401804000        4               2                0                1  \n",
       "1401807600        4               1                0                1  "
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "source_houses = ['house_21', 'house_3', 'house_10', 'house_5', 'house_7', 'house_16', 'house_8', 'house_9']\n",
    "target_houses_in_train = ['house_2_target', 'house_15_target', 'house_18_target']\n",
    "test_houses = ['house_19_target', 'house_4_target', 'house_20', 'house_1', 'house_11', 'house_6', 'house_17']\n",
    "\n",
    "train, test = permute_concatenated_data(data, source_houses, target_houses_in_train, test_houses)\n",
    "\n",
    "train[0]['X']['heating'].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "4f1af5bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('data/refit_data_by_category_less_target.pk', 'wb') as f:\n",
    "    pickle.dump({'train': train, 'test': test}, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e0005368",
   "metadata": {},
   "source": [
    "### Based on total owned appliances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "db996e3d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>house_2</th>\n",
       "      <th>house_3</th>\n",
       "      <th>house_7</th>\n",
       "      <th>house_9</th>\n",
       "      <th>house_11</th>\n",
       "      <th>house_15</th>\n",
       "      <th>house_17</th>\n",
       "      <th>house_19</th>\n",
       "      <th>house_21</th>\n",
       "      <th>house_1_target</th>\n",
       "      <th>house_4_target</th>\n",
       "      <th>house_5_target</th>\n",
       "      <th>house_6_target</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Unix</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1401793200</th>\n",
       "      <td>212</td>\n",
       "      <td>4418</td>\n",
       "      <td>270</td>\n",
       "      <td>245</td>\n",
       "      <td>1777</td>\n",
       "      <td>171</td>\n",
       "      <td>145</td>\n",
       "      <td>383</td>\n",
       "      <td>632</td>\n",
       "      <td>211</td>\n",
       "      <td>229</td>\n",
       "      <td>400</td>\n",
       "      <td>378</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1401796800</th>\n",
       "      <td>202</td>\n",
       "      <td>2446</td>\n",
       "      <td>277</td>\n",
       "      <td>292</td>\n",
       "      <td>1351</td>\n",
       "      <td>198</td>\n",
       "      <td>336</td>\n",
       "      <td>181</td>\n",
       "      <td>1256</td>\n",
       "      <td>151</td>\n",
       "      <td>3094</td>\n",
       "      <td>268</td>\n",
       "      <td>398</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1401800400</th>\n",
       "      <td>181</td>\n",
       "      <td>590</td>\n",
       "      <td>280</td>\n",
       "      <td>228</td>\n",
       "      <td>2302</td>\n",
       "      <td>171</td>\n",
       "      <td>289</td>\n",
       "      <td>148</td>\n",
       "      <td>1532</td>\n",
       "      <td>223</td>\n",
       "      <td>171</td>\n",
       "      <td>516</td>\n",
       "      <td>408</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1401804000</th>\n",
       "      <td>181</td>\n",
       "      <td>346</td>\n",
       "      <td>213</td>\n",
       "      <td>205</td>\n",
       "      <td>920</td>\n",
       "      <td>199</td>\n",
       "      <td>351</td>\n",
       "      <td>542</td>\n",
       "      <td>465</td>\n",
       "      <td>221</td>\n",
       "      <td>95</td>\n",
       "      <td>534</td>\n",
       "      <td>529</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1401807600</th>\n",
       "      <td>2395</td>\n",
       "      <td>213</td>\n",
       "      <td>288</td>\n",
       "      <td>346</td>\n",
       "      <td>1039</td>\n",
       "      <td>173</td>\n",
       "      <td>174</td>\n",
       "      <td>136</td>\n",
       "      <td>830</td>\n",
       "      <td>175</td>\n",
       "      <td>159</td>\n",
       "      <td>458</td>\n",
       "      <td>563</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            house_2  house_3  house_7  house_9  house_11  house_15  house_17  \\\n",
       "Unix                                                                           \n",
       "1401793200      212     4418      270      245      1777       171       145   \n",
       "1401796800      202     2446      277      292      1351       198       336   \n",
       "1401800400      181      590      280      228      2302       171       289   \n",
       "1401804000      181      346      213      205       920       199       351   \n",
       "1401807600     2395      213      288      346      1039       173       174   \n",
       "\n",
       "            house_19  house_21  house_1_target  house_4_target  \\\n",
       "Unix                                                             \n",
       "1401793200       383       632             211             229   \n",
       "1401796800       181      1256             151            3094   \n",
       "1401800400       148      1532             223             171   \n",
       "1401804000       542       465             221              95   \n",
       "1401807600       136       830             175             159   \n",
       "\n",
       "            house_5_target  house_6_target  \n",
       "Unix                                        \n",
       "1401793200             400             378  \n",
       "1401796800             268             398  \n",
       "1401800400             516             408  \n",
       "1401804000             534             529  \n",
       "1401807600             458             563  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#source_houses = df[df['AppliancesOwned'] < 30]['House'].tolist()\n",
    "source_houses = ['house_2_target',\n",
    " 'house_3',\n",
    " 'house_7',\n",
    " 'house_9',\n",
    " 'house_11',\n",
    " 'house_15_target',\n",
    " 'house_17',\n",
    " 'house_19_target',\n",
    " 'house_21']\n",
    "target_houses_in_train = ['house_1',\n",
    " 'house_4_target',\n",
    " 'house_5',\n",
    " 'house_6']\n",
    "\n",
    "test_houses = ['house_8',\n",
    " 'house_10',\n",
    " 'house_16',\n",
    " 'house_18_target',\n",
    " 'house_20']\n",
    "\n",
    "\n",
    "train, test = permute_concatenated_data(data, source_houses, target_houses_in_train, test_houses)\n",
    "\n",
    "train[0]['X_bar'].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "816d11e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('data/refit_data_by_category_owned_appliances.pk', 'wb') as f:\n",
    "    pickle.dump({'train': train, 'test': test}, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e0b71eab",
   "metadata": {},
   "source": [
    "#### Less Target in Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "027b45b1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>house_2</th>\n",
       "      <th>house_3</th>\n",
       "      <th>house_7</th>\n",
       "      <th>house_9</th>\n",
       "      <th>house_11</th>\n",
       "      <th>house_15</th>\n",
       "      <th>house_17</th>\n",
       "      <th>house_19</th>\n",
       "      <th>house_21</th>\n",
       "      <th>house_1_target</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Unix</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1401793200</th>\n",
       "      <td>212</td>\n",
       "      <td>4418</td>\n",
       "      <td>270</td>\n",
       "      <td>245</td>\n",
       "      <td>1777</td>\n",
       "      <td>171</td>\n",
       "      <td>145</td>\n",
       "      <td>383</td>\n",
       "      <td>632</td>\n",
       "      <td>211</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1401796800</th>\n",
       "      <td>202</td>\n",
       "      <td>2446</td>\n",
       "      <td>277</td>\n",
       "      <td>292</td>\n",
       "      <td>1351</td>\n",
       "      <td>198</td>\n",
       "      <td>336</td>\n",
       "      <td>181</td>\n",
       "      <td>1256</td>\n",
       "      <td>151</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1401800400</th>\n",
       "      <td>181</td>\n",
       "      <td>590</td>\n",
       "      <td>280</td>\n",
       "      <td>228</td>\n",
       "      <td>2302</td>\n",
       "      <td>171</td>\n",
       "      <td>289</td>\n",
       "      <td>148</td>\n",
       "      <td>1532</td>\n",
       "      <td>223</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1401804000</th>\n",
       "      <td>181</td>\n",
       "      <td>346</td>\n",
       "      <td>213</td>\n",
       "      <td>205</td>\n",
       "      <td>920</td>\n",
       "      <td>199</td>\n",
       "      <td>351</td>\n",
       "      <td>542</td>\n",
       "      <td>465</td>\n",
       "      <td>221</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1401807600</th>\n",
       "      <td>2395</td>\n",
       "      <td>213</td>\n",
       "      <td>288</td>\n",
       "      <td>346</td>\n",
       "      <td>1039</td>\n",
       "      <td>173</td>\n",
       "      <td>174</td>\n",
       "      <td>136</td>\n",
       "      <td>830</td>\n",
       "      <td>175</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            house_2  house_3  house_7  house_9  house_11  house_15  house_17  \\\n",
       "Unix                                                                           \n",
       "1401793200      212     4418      270      245      1777       171       145   \n",
       "1401796800      202     2446      277      292      1351       198       336   \n",
       "1401800400      181      590      280      228      2302       171       289   \n",
       "1401804000      181      346      213      205       920       199       351   \n",
       "1401807600     2395      213      288      346      1039       173       174   \n",
       "\n",
       "            house_19  house_21  house_1_target  \n",
       "Unix                                            \n",
       "1401793200       383       632             211  \n",
       "1401796800       181      1256             151  \n",
       "1401800400       148      1532             223  \n",
       "1401804000       542       465             221  \n",
       "1401807600       136       830             175  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#source_houses = df[df['AppliancesOwned'] < 30]['House'].tolist()\n",
    "source_houses = ['house_2_target',\n",
    " 'house_3',\n",
    " 'house_7',\n",
    " 'house_9',\n",
    " 'house_11',\n",
    " 'house_15_target',\n",
    " 'house_17',\n",
    " 'house_19_target',\n",
    " 'house_21']\n",
    "target_houses_in_train = ['house_1',\n",
    " #'house_4_target'\n",
    "]\n",
    "\n",
    "test_houses = [\n",
    "#'house_5',\n",
    " #'house_6',\n",
    " 'house_8',\n",
    " 'house_10',\n",
    " 'house_16',\n",
    " 'house_18_target',\n",
    " 'house_20']\n",
    "\n",
    "\n",
    "train, test = permute_concatenated_data(data, source_houses, target_houses_in_train, test_houses)\n",
    "\n",
    "train[0]['X_bar'].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "bd958216",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('data/refit_data_by_category_owned_appliances_less_target_2.pk', 'wb') as f:\n",
    "    pickle.dump({'train': train, 'test': test}, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a5afaa6c",
   "metadata": {},
   "source": [
    "### Based on Occupancy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "f48af82a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(['1', '3', '4', '6', '8', '9', '11', '15', '18', '20'],\n",
       " ['2', '5', '7', '10', '16', '17', '19', '21'])"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df['Occupancy'] < 3]['House'].tolist(), df[df['Occupancy'] >= 3]['House'].tolist(), "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "4a55b34a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>house_1</th>\n",
       "      <th>house_3</th>\n",
       "      <th>house_4</th>\n",
       "      <th>house_6</th>\n",
       "      <th>house_8</th>\n",
       "      <th>house_9</th>\n",
       "      <th>house_11</th>\n",
       "      <th>house_15</th>\n",
       "      <th>house_18</th>\n",
       "      <th>house_20</th>\n",
       "      <th>house_2_target</th>\n",
       "      <th>house_5_target</th>\n",
       "      <th>house_7_target</th>\n",
       "      <th>house_10_target</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Unix</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1401793200</th>\n",
       "      <td>211</td>\n",
       "      <td>4418</td>\n",
       "      <td>229</td>\n",
       "      <td>378</td>\n",
       "      <td>108</td>\n",
       "      <td>245</td>\n",
       "      <td>1777</td>\n",
       "      <td>171</td>\n",
       "      <td>435</td>\n",
       "      <td>365</td>\n",
       "      <td>212</td>\n",
       "      <td>400</td>\n",
       "      <td>270</td>\n",
       "      <td>1120</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1401796800</th>\n",
       "      <td>151</td>\n",
       "      <td>2446</td>\n",
       "      <td>3094</td>\n",
       "      <td>398</td>\n",
       "      <td>197</td>\n",
       "      <td>292</td>\n",
       "      <td>1351</td>\n",
       "      <td>198</td>\n",
       "      <td>677</td>\n",
       "      <td>219</td>\n",
       "      <td>202</td>\n",
       "      <td>268</td>\n",
       "      <td>277</td>\n",
       "      <td>367</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1401800400</th>\n",
       "      <td>223</td>\n",
       "      <td>590</td>\n",
       "      <td>171</td>\n",
       "      <td>408</td>\n",
       "      <td>256</td>\n",
       "      <td>228</td>\n",
       "      <td>2302</td>\n",
       "      <td>171</td>\n",
       "      <td>378</td>\n",
       "      <td>206</td>\n",
       "      <td>181</td>\n",
       "      <td>516</td>\n",
       "      <td>280</td>\n",
       "      <td>623</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1401804000</th>\n",
       "      <td>221</td>\n",
       "      <td>346</td>\n",
       "      <td>95</td>\n",
       "      <td>529</td>\n",
       "      <td>224</td>\n",
       "      <td>205</td>\n",
       "      <td>920</td>\n",
       "      <td>199</td>\n",
       "      <td>539</td>\n",
       "      <td>261</td>\n",
       "      <td>181</td>\n",
       "      <td>534</td>\n",
       "      <td>213</td>\n",
       "      <td>605</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1401807600</th>\n",
       "      <td>175</td>\n",
       "      <td>213</td>\n",
       "      <td>159</td>\n",
       "      <td>563</td>\n",
       "      <td>127</td>\n",
       "      <td>346</td>\n",
       "      <td>1039</td>\n",
       "      <td>173</td>\n",
       "      <td>581</td>\n",
       "      <td>230</td>\n",
       "      <td>2395</td>\n",
       "      <td>458</td>\n",
       "      <td>288</td>\n",
       "      <td>479</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            house_1  house_3  house_4  house_6  house_8  house_9  house_11  \\\n",
       "Unix                                                                         \n",
       "1401793200      211     4418      229      378      108      245      1777   \n",
       "1401796800      151     2446     3094      398      197      292      1351   \n",
       "1401800400      223      590      171      408      256      228      2302   \n",
       "1401804000      221      346       95      529      224      205       920   \n",
       "1401807600      175      213      159      563      127      346      1039   \n",
       "\n",
       "            house_15  house_18  house_20  house_2_target  house_5_target  \\\n",
       "Unix                                                                       \n",
       "1401793200       171       435       365             212             400   \n",
       "1401796800       198       677       219             202             268   \n",
       "1401800400       171       378       206             181             516   \n",
       "1401804000       199       539       261             181             534   \n",
       "1401807600       173       581       230            2395             458   \n",
       "\n",
       "            house_7_target  house_10_target  \n",
       "Unix                                         \n",
       "1401793200             270             1120  \n",
       "1401796800             277              367  \n",
       "1401800400             280              623  \n",
       "1401804000             213              605  \n",
       "1401807600             288              479  "
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "source_houses = ['house_1',\n",
    " 'house_3',\n",
    " 'house_4_target',\n",
    " 'house_6',\n",
    " 'house_8',\n",
    " 'house_9',\n",
    " 'house_11',\n",
    " 'house_15_target',\n",
    " 'house_18_target',\n",
    " 'house_20']\n",
    "target_houses_in_train = ['house_2_target',\n",
    " 'house_5',\n",
    " 'house_7',\n",
    " 'house_10']\n",
    "\n",
    "test_houses = ['house_16',\n",
    " 'house_17',\n",
    " 'house_19_target',\n",
    " 'house_21']\n",
    "\n",
    "\n",
    "train, test = permute_concatenated_data(data, source_houses, target_houses_in_train, test_houses)\n",
    "\n",
    "train[0]['X_bar'].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "93fd9607",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('data/refit_data_by_category_occupancy.pk', 'wb') as f:\n",
    "    pickle.dump({'train': train, 'test': test}, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "14187377",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}