5_3_Tree_Ensemble.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zp6fW8MP-mrO"
      },
      "source": [
        "# 트리의 앙상블"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dIaIAizcRSG-"
      },
      "source": [
        "## 랜덤포레스트"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "ioJUlZ0M_uSZ"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "wine = pd.read_csv('https://bit.ly/wine_csv_data')\n",
        "\n",
        "data = wine[['alcohol', 'sugar', 'pH']].to_numpy()\n",
        "target = wine['class'].to_numpy()\n",
        "\n",
        "train_input, test_input, train_target, test_target = train_test_split(data, target, test_size=0.2, random_state=42)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.model_selection import cross_validate\n",
        "# 랜덤 포레스트 분류기\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "\n",
        "rf = RandomForestClassifier(n_jobs=-1, random_state=42)\n",
        "# 교차 검증 (5-fold cross validation)\n",
        "# return_train_score: True로 지정하면 validation set의 score 뿐만 아니라 train set의 score를 함께 반환 - 과대적합 파악 가능\n",
        "scores = cross_validate(rf, train_input, train_target, n_jobs=-1, return_train_score=True)\n",
        "\n",
        "# 평균 계산 (train score까지 출력 가능)\n",
        "print(np.mean(scores['train_score']), np.mean(scores['test_score']))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "__JXeWakdy9Q",
        "outputId": "c5659fb1-8947-45df-a78a-e903d30c2bb1"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0.9973541965122431 0.8905151032797809\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XYDbzXNLG8fK",
        "outputId": "caf9031e-f170-4171-9d73-7f87c1a34717"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[0.23167441 0.50039841 0.26792718]\n"
          ]
        }
      ],
      "source": [
        "# train\n",
        "rf.fit(train_input, train_target)\n",
        "# feature 중요도 출력\n",
        "print(rf.feature_importances_)\n",
        "\n",
        "# features = ['alcohol', 'sugar', 'pH']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oMc06S1Fa_A-",
        "outputId": "78d2c123-5b56-492b-9695-bfaa34dc515d"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0.8934000384837406\n"
          ]
        }
      ],
      "source": [
        "# oob_score: True로 지정하면 각 결정 트리의 OOB 샘플로 평가한 score의 평균값 반환\n",
        "rf = RandomForestClassifier(oob_score=True, n_jobs=-1, random_state=42)\n",
        "\n",
        "rf.fit(train_input, train_target)\n",
        "# OOB score 출력\n",
        "print(rf.oob_score_)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KdrVoeQZRU14"
      },
      "source": [
        "## 엑스트라트리"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from sklearn.ensemble import ExtraTreesClassifier\n",
        "# 엑스트라 트리 분류기\n",
        "et = ExtraTreesClassifier(n_jobs=-1, random_state=42)\n",
        "\n",
        "# 교차 검증 (5-fold cross validation)\n",
        "scores = cross_validate(et, train_input, train_target, n_jobs=-1, return_train_score=True)\n",
        "\n",
        "# 평균 계산\n",
        "print(np.mean(scores['train_score']), np.mean(scores['test_score']))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sVt_wjk1m1Jy",
        "outputId": "b0d0d370-ff53-42f0-cdbf-34f42f1c858d"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0.9974503966084433 0.8887848893166506\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "csKxnaxeRX8s"
      },
      "source": [
        "## 그레이디언트 부스팅"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_IlNEFkaNsoG",
        "outputId": "4b12d37c-073a-4dd4-9211-89eae370b070"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0.8881086892152563 0.8720430147331015\n"
          ]
        }
      ],
      "source": [
        "from sklearn.ensemble import GradientBoostingClassifier\n",
        "\n",
        "# 그레이디언트 부스팅 분류기\n",
        "gb = GradientBoostingClassifier(random_state=42)\n",
        "# 교차 검증\n",
        "scores = cross_validate(gb, train_input, train_target, return_train_score=True, n_jobs=-1)\n",
        "\n",
        "# 평균 계산\n",
        "print(np.mean(scores['train_score']), np.mean(scores['test_score']))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pNpeS8EWpeEi",
        "outputId": "a307ade4-1fd1-4551-b125-5bbbb73da1a5"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0.9464595437171814 0.8780082549788999\n"
          ]
        }
      ],
      "source": [
        "# n_estimators: 결정 트리 개수\n",
        "# learning_rate: 학습률 (기본값은 0.1)\n",
        "# 결정 트리 개수와 학습률을 늘려 성능 향상\n",
        "gb = GradientBoostingClassifier(n_estimators=500, learning_rate=0.2, random_state=42)\n",
        "scores = cross_validate(gb, train_input, train_target, return_train_score=True, n_jobs=-1)\n",
        "\n",
        "print(np.mean(scores['train_score']), np.mean(scores['test_score']))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BthW_II9RbLa"
      },
      "source": [
        "## 히스토그램 기반 부스팅"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_3Ct_NNWQbdA",
        "outputId": "b9a85883-85ca-460f-c142-428d36088a33"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0.9321723946453317 0.8801241948619236\n"
          ]
        }
      ],
      "source": [
        "from sklearn.ensemble import HistGradientBoostingClassifier\n",
        "\n",
        "# 히스토그램 기반 그레이디언트 부스팅\n",
        "hgb = HistGradientBoostingClassifier(random_state=42)\n",
        "# 교차 검증\n",
        "scores = cross_validate(hgb, train_input, train_target, return_train_score=True, n_jobs=-1)\n",
        "\n",
        "# 평균 출력\n",
        "print(np.mean(scores['train_score']), np.mean(scores['test_score']))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TvlB0GMTS3hn",
        "outputId": "8678f04d-94ed-48ae-89db-9e3f6cf16db5"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[0.08876275 0.23438522 0.08027708]\n"
          ]
        }
      ],
      "source": [
        "from sklearn.inspection import permutation_importance\n",
        "\n",
        "# train\n",
        "hgb.fit(train_input, train_target)\n",
        "# permutation_importance() 함수: feature 중요도 계산\n",
        "# n_repeats 매개변수: 랜덤하게 섞을 횟수 지정(기본값은 5)\n",
        "result = permutation_importance(hgb, train_input, train_target, n_repeats=10,\n",
        "                                random_state=42, n_jobs=-1)\n",
        "# 반복해서 계산한 중요도의 평균 출력\n",
        "print(result.importances_mean)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "S8FfxInn-xBQ",
        "outputId": "436d5891-1931-4f96-e74c-9407ab247d26"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[0.05969231 0.20238462 0.049     ]\n"
          ]
        }
      ],
      "source": [
        "# test set에서 feature 중요도 계산\n",
        "result = permutation_importance(hgb, test_input, test_target, n_repeats=10,\n",
        "                                random_state=42, n_jobs=-1)\n",
        "print(result.importances_mean)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pqplZjh0j2nw",
        "outputId": "35e2a583-4281-4a32-c431-5b22341a49fe"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.8723076923076923"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ],
      "source": [
        "# 최종 성능\n",
        "hgb.score(test_input, test_target)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8fz_FrezBezR"
      },
      "source": [
        "#### XGBoost"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YBYLvOiV6rga",
        "outputId": "678de662-a437-4377-dc9f-9832c2037cd0"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0.9558403027491312 0.8782000074035686\n"
          ]
        }
      ],
      "source": [
        "from xgboost import XGBClassifier\n",
        "\n",
        "# XGBoost를 사용하여 HGB 모델 훈련(tree_method='hist')\n",
        "xgb = XGBClassifier(tree_method='hist', random_state=42)\n",
        "# 교차 검증\n",
        "scores = cross_validate(xgb, train_input, train_target, return_train_score=True, n_jobs=-1)\n",
        "\n",
        "# 평균 출력\n",
        "print(np.mean(scores['train_score']), np.mean(scores['test_score']))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zl6nh6DOBd-B"
      },
      "source": [
        "#### LightGBM"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "maihlDMP7lmY",
        "outputId": "c0abecdf-5b37-4da9-e179-57da5a57fdd3"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0.935828414851749 0.8801251203079884\n"
          ]
        }
      ],
      "source": [
        "from lightgbm import LGBMClassifier\n",
        "\n",
        "# LightGBM을 사용하여 HGB 모델 훈련\n",
        "lgb = LGBMClassifier(random_state=42)\n",
        "# 교차 검증\n",
        "scores = cross_validate(lgb, train_input, train_target, return_train_score=True, n_jobs=-1)\n",
        "\n",
        "# 평균 출력\n",
        "print(np.mean(scores['train_score']), np.mean(scores['test_score']))"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": [
        "dIaIAizcRSG-",
        "KdrVoeQZRU14",
        "csKxnaxeRX8s"
      ]
    },
    "kernelspec": {
      "display_name": "default:Python",
      "language": "python",
      "name": "conda-env-default-py"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.10"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}