Grup36-5_dataset_preprocessing.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "O_ejb9R8eKJ3"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "import glob \n",
        "import cv2 as cv\n",
        "import numpy as np"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "cd drive/MyDrive/Global\\ AI\\ Hub\\ -\\ 36.\\ Grup"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "slehKhttIdRJ",
        "outputId": "565eb602-30bb-4031-ea5f-d8bbc3b86fe1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/drive/MyDrive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "count=0\n",
        "\n",
        "def load_img():\n",
        "    global count\n",
        "    data=[]\n",
        "    main_data = []\n",
        "    labels = []\n",
        "    im_folders=os.listdir()\n",
        "    im_folders=im_folders[1:]\n",
        "    im_folders=sorted(im_folders)\n",
        "    print(im_folders)\n",
        "    \n",
        "    for im_folder in im_folders:\n",
        "      if (im_folder.find(\"zip\") ==-1) and (im_folder.find(\"py\") ==-1)  :\n",
        "          temp=0\n",
        "          for img in os.listdir(im_folder):\n",
        "              path_new = im_folder+\"/\"+img\n",
        "              img = cv.imread(path_new,0)\n",
        "              scale_percent =40\n",
        "\n",
        "              width = int(img.shape[1] * scale_percent / 100)\n",
        "              height = int(img.shape[0] * scale_percent / 100)\n",
        "              dim = (width, height)\n",
        "\n",
        "                \n",
        "              resized = cv.resize(img, dim, interpolation = cv.INTER_AREA)\n",
        "              data.append([resized,int(im_folder)])\n",
        "              main_data.append(resized)\n",
        "              labels.append(int(im_folder))\n",
        "              cv.imwrite(\"path_to_save\"+str(count)+\"/\"+\"img_{}.png\".format(temp),resized)\n",
        "              temp+=1\n",
        "          count+=1\n",
        "    return data, dim, main_data, labels\n",
        "\n",
        "datas, dim, maindata, labels = load_img()\n",
        "\n"
      ],
      "metadata": {
        "id": "MnrjngNeehB3",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "fd58d4df-3407-4c1f-a32c-35d3bc0a217d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['0', '0.zip', '1', '2', '2.zip', '3', '3.zip', '4', '4.zip', '5', '5.zip', '6', '6.zip', '7', '7.zip', '8', '8.zip', '9', '9.zip', 'bootcamp.py']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "len(datas)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "26uek8V4NMLs",
        "outputId": "abe851d1-1209-4b36-8bec-5fb249990e42"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "8732"
            ]
          },
          "metadata": {},
          "execution_count": 176
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "dim"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AR85uu0XPUDk",
        "outputId": "4bb42b29-b5a0-442d-f2cd-9ee9b631d313"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(80, 59)"
            ]
          },
          "metadata": {},
          "execution_count": 177
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "datas[1000]"
      ],
      "metadata": {
        "id": "HFxzRNh8itof",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "73bda230-18c6-450e-a9fe-c22b0b9979ab"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[array([[201,   0,   0, ...,   0,   0,   0],\n",
              "        [202,   0,   0, ...,   0,   0,   0],\n",
              "        [202,   0,   0, ...,   0,   0,   0],\n",
              "        ...,\n",
              "        [214,  88,  99, ..., 145, 139, 121],\n",
              "        [210,  70,  92, ..., 147, 172, 155],\n",
              "        [236, 168, 170, ..., 177, 180, 175]], dtype=uint8), 1]"
            ]
          },
          "metadata": {},
          "execution_count": 178
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#%70 train %20 val %10 test\n",
        "#train\n",
        "print(\"train size = \", int(8732*70/100))\n",
        "#val\n",
        "print(\"val size = \", int(8732*20/100))\n",
        "#test\n",
        "print(\"test size = \", int(8732*10/100)+1)"
      ],
      "metadata": {
        "id": "bEY98--W0V77",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "34c4f7b2-9ca7-4f9c-9cbf-b422f7bb4024"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "train size =  6112\n",
            "val size =  1746\n",
            "test size =  874\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import random\n",
        "test_indexleri = []\n",
        "val_indexleri = []\n",
        "i = 0\n",
        "while i < 874:\n",
        "  rand_num = random.randint(0,len(datas)-1)\n",
        "  if rand_num not in test_indexleri:\n",
        "    test_indexleri.append(rand_num)\n",
        "    i+=1\n",
        "\n",
        "\n",
        "i = 0 \n",
        "while i < 1746:\n",
        "  rand_num = random.randint(0,len(datas)-1)\n",
        "  if (rand_num not in test_indexleri) and (rand_num not in val_indexleri):\n",
        "    val_indexleri.append(rand_num)\n",
        "    i+=1"
      ],
      "metadata": {
        "id": "1ilWZJE6Q_kA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# B şıkkı\n",
        "trainlist = []\n",
        "testlist = []\n",
        "vallist = []\n",
        "\n",
        "data2 = datas.copy()\n",
        "\n",
        "for idx in test_indexleri:\n",
        "  testlist.append(data2[idx-1])\n",
        "\n",
        "for idx in val_indexleri:\n",
        "  vallist.append(data2[idx-1])\n",
        "\n",
        "\n",
        "removed_idx=0\n",
        "for idx in test_indexleri:\n",
        "  data2.pop(idx-removed_idx-1)\n",
        "  removed_idx+=1\n",
        "\n",
        "\n",
        "for idx in val_indexleri:\n",
        "  data2.pop(idx-removed_idx-1)\n",
        "  removed_idx +=1\n",
        "\n",
        "for data in data2:\n",
        "  trainlist.append(data)"
      ],
      "metadata": {
        "id": "9TaeBfuZSZxA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "w_NzoPqZXSRV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# C şıkkı\n",
        "X_train, y_train, X_test, y_test, X_val, y_val = [], [], [], [], [], []\n",
        "data2 = maindata.copy()\n",
        "labels2 = labels.copy()\n",
        "\n",
        "for idx in test_indexleri:\n",
        "  X_test.append(data2[idx-1])\n",
        "  y_test.append(labels2[idx-1])\n",
        "\n",
        "for idx in val_indexleri:\n",
        "  X_val.append(data2[idx-1])\n",
        "  y_val.append(labels2[idx-1])\n",
        "\n",
        "\n",
        "removed_idx=0\n",
        "for idx in test_indexleri:\n",
        "  data2.pop(idx-removed_idx-1)\n",
        "  labels2.pop(idx-removed_idx-1)\n",
        "\n",
        "  removed_idx+=1\n",
        "\n",
        "\n",
        "for idx in val_indexleri:\n",
        "  data2.pop(idx-removed_idx-1)\n",
        "  labels2.pop(idx-removed_idx-1)\n",
        "  removed_idx +=1\n",
        "\n",
        "for data in data2:\n",
        "  X_train.append(data)\n",
        "\n",
        "for lab in labels2:\n",
        "  y_train.append(lab)"
      ],
      "metadata": {
        "id": "AcUFy2VrUTdk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(len(X_train), len(y_train), len(X_val), len(y_val), len(X_test), len(y_test))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yGsoYssDYOjM",
        "outputId": "9367b6c9-2caf-40de-a3ca-f5ba3a90eb94"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "6112 6112 1746 1746 874 874\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd"
      ],
      "metadata": {
        "id": "0LXynYILY8tO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "val_data = pd.DataFrame(vallist, columns=[\"img\", \"label\"])\n",
        "test_data = pd.DataFrame(testlist, columns=[\"img\", \"label\"])\n",
        "train_data = pd.DataFrame(trainlist, columns=[\"img\", \"label\"])"
      ],
      "metadata": {
        "id": "S5NYAFRLYXQO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "val_data.to_csv(\"val_data.csv\", index=False)"
      ],
      "metadata": {
        "id": "lVMRnsr-Yx3n"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "test_data.to_csv(\"test_data.csv\", index=False)\n",
        "train_data.to_csv(\"train_data.csv\", index=False)"
      ],
      "metadata": {
        "id": "C_LwpAdsZh7t"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "val_data"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "bBVmoTpTZrJb",
        "outputId": "fd9cfe59-9ae5-4ce4-8776-3a1b8d08973b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                    img  label\n",
              "0     [[201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 5, ...      6\n",
              "1     [[201, 1, 0, 3, 0, 1, 1, 2, 4, 0, 6, 0, 3, 1, ...      7\n",
              "2     [[201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...      9\n",
              "3     [[201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...      0\n",
              "4     [[201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...      9\n",
              "...                                                 ...    ...\n",
              "1741  [[201, 0, 0, 0, 0, 0, 0, 1, 1, 1, 6, 8, 5, 4, ...      7\n",
              "1742  [[201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...      3\n",
              "1743  [[201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...      8\n",
              "1744  [[201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...      8\n",
              "1745  [[201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...      9\n",
              "\n",
              "[1746 rows x 2 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-22af5a3d-3f84-4da2-85e4-7fd0c82abb32\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>img</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>[[201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 5, 5, ...</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>[[201, 1, 0, 3, 0, 1, 1, 2, 4, 0, 6, 0, 3, 1, ...</td>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>[[201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
              "      <td>9</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>[[201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>[[201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
              "      <td>9</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1741</th>\n",
              "      <td>[[201, 0, 0, 0, 0, 0, 0, 1, 1, 1, 6, 8, 5, 4, ...</td>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1742</th>\n",
              "      <td>[[201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1743</th>\n",
              "      <td>[[201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1744</th>\n",
              "      <td>[[201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1745</th>\n",
              "      <td>[[201, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
              "      <td>9</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>1746 rows × 2 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-22af5a3d-3f84-4da2-85e4-7fd0c82abb32')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-22af5a3d-3f84-4da2-85e4-7fd0c82abb32 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-22af5a3d-3f84-4da2-85e4-7fd0c82abb32');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 202
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "Gr2iu3_naTq9"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}