LocalDatasetPreprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "43bac8c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from PIL import Image\n",
    "import time\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6c93aa23",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_root = '.\\\\..\\\\data_raw'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "07a250ca",
   "metadata": {},
   "source": [
    "# STATS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "23baea10",
   "metadata": {},
   "outputs": [],
   "source": [
    "class_paths = [f.path for f in os.scandir(data_root) if f.is_dir()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6292b68d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['amazon', 'blogs', 'facebook', 'food', 'games', 'instagram', 'maps', 'news', 'other', 'pinterest', 'reddit', 'shopping', 'sports', 'tiktok', 'travel', 'twitch', 'twitter', 'youtube']\n",
      "['Algorithms', 'Art', 'Astronomy', 'Biology', 'Business', 'Chemistry', 'ClinicalPractium', 'ComputerScience', 'Data Analytics', 'DrugBiology', 'English', 'Ethics', 'extra', 'GenderSexuality', 'GeneralPhysics', 'Genetics', 'InternationalSocialJustice', 'IntroductionToEthics', 'IntroductionToPsychology', 'IntroGenderSexuality', 'LatinAmericanGovPolitics', 'LinearAlgebra', 'Literature', 'Management', 'Marketing', 'Neuroscience', 'neutral', 'other', 'Pharmacology', 'Philosophy', 'Phisiology', 'PhysicalChemistry', 'Physics', 'Politics', 'Principles&Practice', 'ProfessionalPractices', 'QuantitativeAnalysis', 'Race&Racism', 'ReadingFilm', 'ReadingLiterature', 'Social Politics', 'VRdevelopment']\n"
     ]
    }
   ],
   "source": [
    "for c in class_paths:\n",
    "    subcategory_paths = [f.path for f in os.scandir(c) if f.is_dir()]\n",
    "    subcategories = [os.path.basename(os.path.normpath(x)) for x in subcategory_paths]\n",
    "    print(subcategories)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "000aacd4",
   "metadata": {},
   "outputs": [],
   "source": [
    "d = []\n",
    "for c in class_paths:\n",
    "    cat = os.path.basename(os.path.normpath(c))\n",
    "    subcategory_paths = [f.path for f in os.scandir(c) if f.is_dir()]\n",
    "    for subc in subcategory_paths:\n",
    "        subcat = os.path.basename(os.path.normpath(subc))\n",
    "        division_paths = [f.path for f in os.scandir(subc) if f.is_dir()]\n",
    "        for div in division_paths:\n",
    "            division = os.path.basename(os.path.normpath(div))\n",
    "            _, _, files = next(os.walk(div))\n",
    "            d.append({\"class\": cat, \"subcategory\":subcat, \"window\":division, \"samples\":len(files)})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ffc17a18",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "70c69d5f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>class</th>\n",
       "      <th>subcategory</th>\n",
       "      <th>window</th>\n",
       "      <th>samples</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>distracted</td>\n",
       "      <td>amazon</td>\n",
       "      <td>one</td>\n",
       "      <td>182</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>distracted</td>\n",
       "      <td>amazon</td>\n",
       "      <td>partial</td>\n",
       "      <td>74</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>distracted</td>\n",
       "      <td>blogs</td>\n",
       "      <td>one</td>\n",
       "      <td>288</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>distracted</td>\n",
       "      <td>blogs</td>\n",
       "      <td>partial</td>\n",
       "      <td>90</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>distracted</td>\n",
       "      <td>facebook</td>\n",
       "      <td>one</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>115</th>\n",
       "      <td>focused</td>\n",
       "      <td>ReadingLiterature</td>\n",
       "      <td>partial</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>116</th>\n",
       "      <td>focused</td>\n",
       "      <td>Social Politics</td>\n",
       "      <td>one</td>\n",
       "      <td>157</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117</th>\n",
       "      <td>focused</td>\n",
       "      <td>Social Politics</td>\n",
       "      <td>partial</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118</th>\n",
       "      <td>focused</td>\n",
       "      <td>VRdevelopment</td>\n",
       "      <td>one</td>\n",
       "      <td>65</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>119</th>\n",
       "      <td>focused</td>\n",
       "      <td>VRdevelopment</td>\n",
       "      <td>partial</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>120 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          class        subcategory   window  samples\n",
       "0    distracted             amazon      one      182\n",
       "1    distracted             amazon  partial       74\n",
       "2    distracted              blogs      one      288\n",
       "3    distracted              blogs  partial       90\n",
       "4    distracted           facebook      one       21\n",
       "..          ...                ...      ...      ...\n",
       "115     focused  ReadingLiterature  partial        0\n",
       "116     focused    Social Politics      one      157\n",
       "117     focused    Social Politics  partial        0\n",
       "118     focused      VRdevelopment      one       65\n",
       "119     focused      VRdevelopment  partial        0\n",
       "\n",
       "[120 rows x 4 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "a380bbad",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\ADHD Project\\AppData\\Local\\Temp\\ipykernel_18944\\4182530805.py:1: FutureWarning: The default value of numeric_only in DataFrameGroupBy.sum is deprecated. In a future version, numeric_only will default to False. Either specify numeric_only or select only columns which should be valid for the function.\n",
      "  df.groupby(['window']).sum()\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>samples</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>window</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>one</th>\n",
       "      <td>9879</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>partial</th>\n",
       "      <td>4312</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         samples\n",
       "window          \n",
       "one         9879\n",
       "partial     4312"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.groupby(['window']).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "cfb2e580",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\ADHD Project\\AppData\\Local\\Temp\\ipykernel_18944\\624196825.py:1: FutureWarning: The default value of numeric_only in DataFrameGroupBy.sum is deprecated. In a future version, numeric_only will default to False. Either specify numeric_only or select only columns which should be valid for the function.\n",
      "  df.groupby(['class']).sum()\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>samples</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>class</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>distracted</th>\n",
       "      <td>5013</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>focused</th>\n",
       "      <td>9178</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            samples\n",
       "class              \n",
       "distracted     5013\n",
       "focused        9178"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.groupby(['class']).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "ce71d296",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\ADHD Project\\AppData\\Local\\Temp\\ipykernel_18944\\1982953666.py:1: FutureWarning: The default value of numeric_only in DataFrameGroupBy.sum is deprecated. In a future version, numeric_only will default to False. Either specify numeric_only or select only columns which should be valid for the function.\n",
      "  df.groupby(['class', 'window']).sum()\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>samples</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>class</th>\n",
       "      <th>window</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">distracted</th>\n",
       "      <th>one</th>\n",
       "      <td>3354</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>partial</th>\n",
       "      <td>1659</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">focused</th>\n",
       "      <th>one</th>\n",
       "      <td>6525</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>partial</th>\n",
       "      <td>2653</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    samples\n",
       "class      window          \n",
       "distracted one         3354\n",
       "           partial     1659\n",
       "focused    one         6525\n",
       "           partial     2653"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.groupby(['class', 'window']).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "f0665585",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\ADHD Project\\AppData\\Local\\Temp\\ipykernel_18944\\1882247642.py:1: FutureWarning: The default value of numeric_only in DataFrameGroupBy.sum is deprecated. In a future version, numeric_only will default to False. Either specify numeric_only or select only columns which should be valid for the function.\n",
      "  df.groupby(['class', 'subcategory']).sum()\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>samples</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>class</th>\n",
       "      <th>subcategory</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"18\" valign=\"top\">distracted</th>\n",
       "      <th>amazon</th>\n",
       "      <td>256</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>blogs</th>\n",
       "      <td>378</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>facebook</th>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>food</th>\n",
       "      <td>35</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>games</th>\n",
       "      <td>861</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>instagram</th>\n",
       "      <td>107</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>maps</th>\n",
       "      <td>29</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>news</th>\n",
       "      <td>168</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>other</th>\n",
       "      <td>416</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>pinterest</th>\n",
       "      <td>161</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>reddit</th>\n",
       "      <td>201</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>shopping</th>\n",
       "      <td>499</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sports</th>\n",
       "      <td>68</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>tiktok</th>\n",
       "      <td>61</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>travel</th>\n",
       "      <td>71</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>twitch</th>\n",
       "      <td>120</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>twitter</th>\n",
       "      <td>785</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>youtube</th>\n",
       "      <td>773</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"42\" valign=\"top\">focused</th>\n",
       "      <th>Algorithms</th>\n",
       "      <td>325</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Art</th>\n",
       "      <td>93</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Astronomy</th>\n",
       "      <td>269</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Biology</th>\n",
       "      <td>164</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Business</th>\n",
       "      <td>120</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Chemistry</th>\n",
       "      <td>799</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ClinicalPractium</th>\n",
       "      <td>127</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ComputerScience</th>\n",
       "      <td>476</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Data Analytics</th>\n",
       "      <td>145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DrugBiology</th>\n",
       "      <td>223</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>English</th>\n",
       "      <td>114</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Ethics</th>\n",
       "      <td>53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GenderSexuality</th>\n",
       "      <td>126</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GeneralPhysics</th>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Genetics</th>\n",
       "      <td>122</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>InternationalSocialJustice</th>\n",
       "      <td>445</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>IntroGenderSexuality</th>\n",
       "      <td>65</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>IntroductionToEthics</th>\n",
       "      <td>59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>IntroductionToPsychology</th>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LatinAmericanGovPolitics</th>\n",
       "      <td>310</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LinearAlgebra</th>\n",
       "      <td>133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Literature</th>\n",
       "      <td>213</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Management</th>\n",
       "      <td>348</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Marketing</th>\n",
       "      <td>139</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Neuroscience</th>\n",
       "      <td>41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Pharmacology</th>\n",
       "      <td>141</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Philosophy</th>\n",
       "      <td>372</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Phisiology</th>\n",
       "      <td>146</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PhysicalChemistry</th>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Physics</th>\n",
       "      <td>70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Politics</th>\n",
       "      <td>329</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Principles&amp;Practice</th>\n",
       "      <td>81</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ProfessionalPractices</th>\n",
       "      <td>186</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>QuantitativeAnalysis</th>\n",
       "      <td>232</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Race&amp;Racism</th>\n",
       "      <td>178</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ReadingFilm</th>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ReadingLiterature</th>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Social Politics</th>\n",
       "      <td>157</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>VRdevelopment</th>\n",
       "      <td>65</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>extra</th>\n",
       "      <td>283</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>neutral</th>\n",
       "      <td>170</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>other</th>\n",
       "      <td>1749</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       samples\n",
       "class      subcategory                        \n",
       "distracted amazon                          256\n",
       "           blogs                           378\n",
       "           facebook                         24\n",
       "           food                             35\n",
       "           games                           861\n",
       "           instagram                       107\n",
       "           maps                             29\n",
       "           news                            168\n",
       "           other                           416\n",
       "           pinterest                       161\n",
       "           reddit                          201\n",
       "           shopping                        499\n",
       "           sports                           68\n",
       "           tiktok                           61\n",
       "           travel                           71\n",
       "           twitch                          120\n",
       "           twitter                         785\n",
       "           youtube                         773\n",
       "focused    Algorithms                      325\n",
       "           Art                              93\n",
       "           Astronomy                       269\n",
       "           Biology                         164\n",
       "           Business                        120\n",
       "           Chemistry                       799\n",
       "           ClinicalPractium                127\n",
       "           ComputerScience                 476\n",
       "           Data Analytics                  145\n",
       "           DrugBiology                     223\n",
       "           English                         114\n",
       "           Ethics                           53\n",
       "           GenderSexuality                 126\n",
       "           GeneralPhysics                   10\n",
       "           Genetics                        122\n",
       "           InternationalSocialJustice      445\n",
       "           IntroGenderSexuality             65\n",
       "           IntroductionToEthics             59\n",
       "           IntroductionToPsychology         37\n",
       "           LatinAmericanGovPolitics        310\n",
       "           LinearAlgebra                   133\n",
       "           Literature                      213\n",
       "           Management                      348\n",
       "           Marketing                       139\n",
       "           Neuroscience                     41\n",
       "           Pharmacology                    141\n",
       "           Philosophy                      372\n",
       "           Phisiology                      146\n",
       "           PhysicalChemistry                11\n",
       "           Physics                          70\n",
       "           Politics                        329\n",
       "           Principles&Practice              81\n",
       "           ProfessionalPractices           186\n",
       "           QuantitativeAnalysis            232\n",
       "           Race&Racism                     178\n",
       "           ReadingFilm                      16\n",
       "           ReadingLiterature                36\n",
       "           Social Politics                 157\n",
       "           VRdevelopment                    65\n",
       "           extra                           283\n",
       "           neutral                         170\n",
       "           other                          1749"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.groupby(['class', 'subcategory']).sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "36474bb3",
   "metadata": {},
   "source": [
    "# Creating new clean folder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "7aff0767",
   "metadata": {},
   "outputs": [],
   "source": [
    "clean_root = '.\\\\..\\\\datasets\\\\data_clean_xlarge'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "8fd71c2f",
   "metadata": {},
   "outputs": [],
   "source": [
    "counter = dict()\n",
    "for c in class_paths:\n",
    "    cat = os.path.basename(os.path.normpath(c))\n",
    "    counter[cat] = 0\n",
    "    os.makedirs(os.path.join(clean_root, cat))\n",
    "    subcategory_paths = [f.path for f in os.scandir(c) if f.is_dir()]\n",
    "    for subc in subcategory_paths:\n",
    "        subcat = os.path.basename(os.path.normpath(subc))\n",
    "        division_paths = [f.path for f in os.scandir(subc) if f.is_dir()]\n",
    "        for div in division_paths:\n",
    "            division = os.path.basename(os.path.normpath(div))\n",
    "            _, _, files = next(os.walk(div))\n",
    "            for file in files:\n",
    "                im = Image.open(os.path.join(div, file))\n",
    "                filemame = cat + '_' + subcat + '_' + division + '_' + str(counter[cat]) + '.jpg'\n",
    "                if 'png' in file:\n",
    "                    im = im.convert('RGB')\n",
    "                im.save(os.path.join(clean_root, cat, filemame))\n",
    "                counter[cat] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "24978d01",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:env_mladhd]",
   "language": "python",
   "name": "conda-env-env_mladhd-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}