model_bow.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model based on Bags of Words feature extraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import essay\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loaded count of essays: 2467\n"
     ]
    }
   ],
   "source": [
    "# load the preprocessed data which we saved\n",
    "# choose how much data you want to load (2467, 11142 or 89364)\n",
    "\n",
    "essays = pickle.load(open( \"data/essays/essays2467.p\", \"rb\"))\n",
    "\n",
    "#essays = pickle.load(open( \"data/essays/essays11142.p\", \"rb\"))\n",
    "\n",
    "#essays = pickle.load(open( \"data/essays/essays89364.p\", \"rb\"))\n",
    "\n",
    "print(\"loaded count of essays:\", len(essays))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Split data in train & test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "training, test = train_test_split(essays, test_size=0.20, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x = [x.clean_text for x in training]\n",
    "\n",
    "train_y_cEXT = [x.cEXT for x in training]\n",
    "train_y_cNEU = [x.cNEU for x in training]\n",
    "train_y_cAGR = [x.cAGR for x in training]\n",
    "train_y_cCON = [x.cCON for x in training]\n",
    "train_y_cOPN = [x.cOPN for x in training]\n",
    "\n",
    "\n",
    "test_x = [x.clean_text for x in test]\n",
    "\n",
    "test_y_cEXT = [x.cEXT for x in test]\n",
    "test_y_cNEU = [x.cNEU for x in test]\n",
    "test_y_cAGR = [x.cAGR for x in test]\n",
    "test_y_cCON = [x.cCON for x in test]\n",
    "test_y_cOPN = [x.cOPN for x in test]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# bags of words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "bow_vectorizer = CountVectorizer()\n",
    "\n",
    "# create vectors from our words\n",
    "train_x_vectors = bow_vectorizer.fit_transform(train_x)\n",
    "test_x_vectors = bow_vectorizer.transform(test_x)\n",
    "# # now that's a big thing :-O"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for evaluation save some data for later:\n",
    "evaluation = []\n",
    "data = len(essays)\n",
    "vec_name = \"BoW\"\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SVM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training Extraversion cEXT using SVM...\n",
      "cEXT score:  0.7344100493494841\n",
      "training Neuroticism cNEU using SVM...\n",
      "with this data not available (MBTI only 4 dimensions)\n",
      "training Agreeableness cAGR using using SVM...\n",
      "cAGR score:  0.7128757290264692\n",
      "training Conscientiousness cCON using SVM...\n",
      "cCON score:  0.6810228802153432\n",
      "training Openness to Experience cOPN using SVM...\n",
      "cOPN score:  0.7707492148945716\n"
     ]
    }
   ],
   "source": [
    "from sklearn import svm\n",
    "name = \"svm\"\n",
    "\n",
    "print(\"training Extraversion cEXT using SVM...\")\n",
    "clf_svm_cEXT = svm.SVC(kernel='linear')\n",
    "clf_svm_cEXT.fit(train_x_vectors, train_y_cEXT)\n",
    "evaluation.append([data, vec_name, name, \"cEXT\", clf_svm_cEXT.score(test_x_vectors, test_y_cEXT)])\n",
    "print(\"cEXT score: \", clf_svm_cEXT.score(test_x_vectors, test_y_cEXT))\n",
    "\n",
    "try:\n",
    "    print(\"training Neuroticism cNEU using SVM...\")\n",
    "    clf_svm_cNEU = svm.SVC(kernel='linear')\n",
    "    clf_svm_cNEU.fit(train_x_vectors, train_y_cNEU)\n",
    "    evaluation.append([data, vec_name, name, \"cNEU\", clf_svm_cNEU.score(test_x_vectors, test_y_cNEU)])\n",
    "    print(\"cNEU score: \", clf_svm_cNEU.score(test_x_vectors, test_y_cNEU))\n",
    "except:\n",
    "    print(\"with this data not available (MBTI only 4 dimensions)\")\n",
    "    \n",
    "print(\"training Agreeableness cAGR using using SVM...\")\n",
    "clf_svm_cAGR = svm.SVC(kernel='linear')\n",
    "clf_svm_cAGR.fit(train_x_vectors, train_y_cAGR)\n",
    "evaluation.append([data, vec_name, name, \"cAGR\", clf_svm_cAGR.score(test_x_vectors, test_y_cAGR)])\n",
    "\n",
    "print(\"cAGR score: \", clf_svm_cAGR.score(test_x_vectors, test_y_cAGR))\n",
    "\n",
    "print(\"training Conscientiousness cCON using SVM...\")\n",
    "clf_svm_cCON = svm.SVC(kernel='linear')\n",
    "clf_svm_cCON.fit(train_x_vectors, train_y_cCON)\n",
    "evaluation.append([data, vec_name, name, \"cCON\", clf_svm_cCON.score(test_x_vectors, test_y_cCON)])\n",
    "print(\"cCON score: \", clf_svm_cCON.score(test_x_vectors, test_y_cCON))\n",
    "\n",
    "print(\"training Openness to Experience cOPN using SVM...\")\n",
    "clf_svm_cOPN = svm.SVC(kernel='linear')\n",
    "clf_svm_cOPN.fit(train_x_vectors, train_y_cOPN)\n",
    "evaluation.append([data, vec_name, name, \"cOPN\", clf_svm_cOPN.score(test_x_vectors, test_y_cOPN)])\n",
    "print(\"cOPN score: \", clf_svm_cOPN.score(test_x_vectors, test_y_cOPN))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Decision Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training Extraversion cEXT using dec...\n",
      "cEXT score:  0.7240915208613729\n",
      "training Neuroticism cNEU using dec...\n",
      "with this data not available (MBTI only 4 dimensions)\n",
      "training Agreeableness cAGR using using dec...\n",
      "cAGR score:  0.6747420367877972\n",
      "training Conscientiousness cCON using dec...\n",
      "cCON score:  0.6514131897711979\n",
      "training Openness to Experience cOPN using dec...\n",
      "cOPN score:  0.7738896366083445\n"
     ]
    }
   ],
   "source": [
    "from sklearn import tree\n",
    "name = \"tree\"\n",
    "\n",
    "print(\"training Extraversion cEXT using dec...\")\n",
    "clf_dec_cEXT = tree.DecisionTreeClassifier()\n",
    "clf_dec_cEXT.fit(train_x_vectors, train_y_cEXT)\n",
    "evaluation.append([data, vec_name, name, \"cEXT\", clf_dec_cEXT.score(test_x_vectors, test_y_cEXT)])\n",
    "\n",
    "print(\"cEXT score: \", clf_dec_cEXT.score(test_x_vectors, test_y_cEXT))\n",
    "\n",
    "try:\n",
    "    print(\"training Neuroticism cNEU using dec...\")\n",
    "    clf_dec_cNEU = tree.DecisionTreeClassifier()\n",
    "    clf_dec_cNEU.fit(train_x_vectors, train_y_cNEU)\n",
    "    evaluation.append([data, vec_name, name, \"cNEU\", clf_dec_cNEU.score(test_x_vectors, test_y_cNEU)])\n",
    "    print(\"cNEU score: \", clf_dec_cNEU.score(test_x_vectors, test_y_cNEU))\n",
    "except:\n",
    "    print(\"with this data not available (MBTI only 4 dimensions)\")\n",
    "\n",
    "print(\"training Agreeableness cAGR using using dec...\")\n",
    "clf_dec_cAGR = tree.DecisionTreeClassifier()\n",
    "clf_dec_cAGR.fit(train_x_vectors, train_y_cAGR)\n",
    "evaluation.append([data, vec_name, name, \"cAGR\", clf_dec_cAGR.score(test_x_vectors, test_y_cAGR)])\n",
    "print(\"cAGR score: \", clf_dec_cAGR.score(test_x_vectors, test_y_cAGR))\n",
    "\n",
    "print(\"training Conscientiousness cCON using dec...\")\n",
    "clf_dec_cCON = tree.DecisionTreeClassifier()\n",
    "clf_dec_cCON.fit(train_x_vectors, train_y_cCON)\n",
    "evaluation.append([data, vec_name, name, \"cCON\", clf_dec_cCON.score(test_x_vectors, test_y_cCON)])\n",
    "print(\"cCON score: \", clf_dec_cCON.score(test_x_vectors, test_y_cCON))\n",
    "\n",
    "print(\"training Openness to Experience cOPN using dec...\")\n",
    "clf_dec_cOPN = tree.DecisionTreeClassifier()\n",
    "clf_dec_cOPN.fit(train_x_vectors, train_y_cOPN)\n",
    "evaluation.append([data, vec_name, name, \"cOPN\", clf_dec_cOPN.score(test_x_vectors, test_y_cOPN)])\n",
    "print(\"cOPN score: \", clf_dec_cOPN.score(test_x_vectors, test_y_cOPN))\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Naive Bayes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training Extraversion cEXT using GaussianNaiveBayes...\n",
      "cEXT score:  0.7079407806191117\n",
      "training Neuroticism cNEU using GaussianNaiveBayes...\n",
      "with this data not available (MBTI only 4 dimensions)\n",
      "training Agreeableness cAGR using using GaussianNaiveBayes...\n",
      "cAGR score:  0.5612382234185733\n",
      "training Conscientiousness cCON using GaussianNaiveBayes...\n",
      "cCON score:  0.5697622252131\n",
      "training Openness to Experience cOPN using GaussianNaiveBayes...\n",
      "cOPN score:  0.7837595334230597\n"
     ]
    }
   ],
   "source": [
    "from sklearn.naive_bayes import GaussianNB\n",
    "name = \"gNB\"\n",
    "# clf_gnb = GaussianNB()\n",
    "# clf_gnb.fit(train_x_vectors.toarray(), train_y)\n",
    "\n",
    "\n",
    "print(\"training Extraversion cEXT using GaussianNaiveBayes...\")\n",
    "clf_gnb_cEXT = GaussianNB()\n",
    "clf_gnb_cEXT.fit(train_x_vectors.toarray(), train_y_cEXT)\n",
    "evaluation.append([data, vec_name, name, \"cEXT\", clf_gnb_cEXT.score(test_x_vectors.toarray(), test_y_cEXT)])\n",
    "print(\"cEXT score: \", clf_gnb_cEXT.score(test_x_vectors.toarray(), test_y_cEXT))\n",
    "\n",
    "try:\n",
    "    print(\"training Neuroticism cNEU using GaussianNaiveBayes...\")\n",
    "    clf_gnb_cNEU = GaussianNB()\n",
    "    clf_gnb_cNEU.fit(train_x_vectors.toarray(), train_y_cNEU)\n",
    "    evaluation.append([data, vec_name, name, \"cNEU\", clf_gnb_cNEU.score(test_x_vectors.toarray(), test_y_cNEU)])\n",
    "    print(\"cNEU score: \", clf_gnb_cNEU.score(test_x_vectors.toarray(), test_y_cNEU))\n",
    "except:\n",
    "    print(\"with this data not available (MBTI only 4 dimensions)\")\n",
    "\n",
    "    \n",
    "print(\"training Agreeableness cAGR using using GaussianNaiveBayes...\")\n",
    "clf_gnb_cAGR = GaussianNB()\n",
    "clf_gnb_cAGR.fit(train_x_vectors.toarray(), train_y_cAGR)\n",
    "evaluation.append([data, vec_name, name, \"cAGR\", clf_gnb_cAGR.score(test_x_vectors.toarray(), test_y_cAGR)])\n",
    "print(\"cAGR score: \", clf_gnb_cAGR.score(test_x_vectors.toarray(), test_y_cAGR))\n",
    "\n",
    "print(\"training Conscientiousness cCON using GaussianNaiveBayes...\")\n",
    "clf_gnb_cCON = GaussianNB()\n",
    "clf_gnb_cCON.fit(train_x_vectors.toarray(), train_y_cCON)\n",
    "evaluation.append([data, vec_name, name, \"cCON\", clf_gnb_cCON.score(test_x_vectors.toarray(), test_y_cCON)])\n",
    "print(\"cCON score: \", clf_gnb_cCON.score(test_x_vectors.toarray(), test_y_cCON))\n",
    "\n",
    "print(\"training Openness to Experience cOPN using GaussianNaiveBayes...\")\n",
    "clf_gnb_cOPN = GaussianNB()\n",
    "clf_gnb_cOPN.fit(train_x_vectors.toarray(), train_y_cOPN)\n",
    "evaluation.append([data, vec_name, name, \"cOPN\", clf_gnb_cOPN.score(test_x_vectors.toarray(), test_y_cOPN)])\n",
    "print(\"cOPN score: \", clf_gnb_cOPN.score(test_x_vectors.toarray(), test_y_cOPN))\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Logisic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training Extraversion cEXT using Logistic Regression...\n",
      "cEXT score:  0.7590847913862718\n",
      "training Neuroticism cNEU using Logistic Regression...\n",
      "with this data not available (MBTI only 4 dimensions)\n",
      "training Agreeableness cAGR using using Logistic Regression...\n",
      "cAGR score:  0.7357559443696725\n",
      "training Conscientiousness cCON using Logistic Regression...\n",
      "cCON score:  0.7034544638851503\n",
      "training Openness to Experience cOPN using Logistic Regression...\n",
      "cOPN score:  0.7900403768506057\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "name=\"logR\"\n",
    "print(\"training Extraversion cEXT using Logistic Regression...\")\n",
    "clf_log_cEXT = LogisticRegression(solver=\"newton-cg\")\n",
    "clf_log_cEXT.fit(train_x_vectors, train_y_cEXT)\n",
    "evaluation.append([data, vec_name, name, \"cEXT\", clf_log_cEXT.score(test_x_vectors, test_y_cEXT)])\n",
    "print(\"cEXT score: \", clf_log_cEXT.score(test_x_vectors, test_y_cEXT))\n",
    "\n",
    "try:\n",
    "    print(\"training Neuroticism cNEU using Logistic Regression...\")\n",
    "    clf_log_cNEU = LogisticRegression(solver=\"newton-cg\")\n",
    "    clf_log_cNEU.fit(train_x_vectors, train_y_cNEU)\n",
    "    evaluation.append([data, vec_name, name, \"cNEU\", clf_log_cNEU.score(test_x_vectors, test_y_cNEU)])\n",
    "    print(\"cNEU score: \", clf_log_cNEU.score(test_x_vectors, test_y_cNEU))\n",
    "except:\n",
    "    print(\"with this data not available (MBTI only 4 dimensions)\")\n",
    "    \n",
    "print(\"training Agreeableness cAGR using using Logistic Regression...\")\n",
    "clf_log_cAGR = LogisticRegression(solver=\"newton-cg\")\n",
    "clf_log_cAGR.fit(train_x_vectors, train_y_cAGR)\n",
    "evaluation.append([data, vec_name, name, \"cAGR\", clf_log_cAGR.score(test_x_vectors, test_y_cAGR)])\n",
    "print(\"cAGR score: \", clf_log_cAGR.score(test_x_vectors, test_y_cAGR))\n",
    "\n",
    "print(\"training Conscientiousness cCON using Logistic Regression...\")\n",
    "clf_log_cCON = LogisticRegression(solver=\"newton-cg\")\n",
    "clf_log_cCON.fit(train_x_vectors, train_y_cCON)\n",
    "evaluation.append([data, vec_name, name, \"cCON\", clf_log_cCON.score(test_x_vectors, test_y_cCON)])\n",
    "print(\"cCON score: \", clf_log_cCON.score(test_x_vectors, test_y_cCON))\n",
    "\n",
    "print(\"training Openness to Experience cOPN using Logistic Regression...\")\n",
    "clf_log_cOPN = LogisticRegression(solver=\"newton-cg\")\n",
    "clf_log_cOPN.fit(train_x_vectors, train_y_cOPN)\n",
    "evaluation.append([data, vec_name, name, \"cOPN\", clf_log_cOPN.score(test_x_vectors, test_y_cOPN)])\n",
    "print(\"cOPN score: \", clf_log_cOPN.score(test_x_vectors, test_y_cOPN))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training Extraversion cEXT using Random Forest...\n",
      "cEXT score:  0.7128757290264692\n",
      "training Neuroticism cNEU using Random Forest...\n",
      "with this data not available (MBTI only 4 dimensions)\n",
      "training Agreeableness cAGR using using Random Forest...\n",
      "cAGR score:  0.7115298340062809\n",
      "training Conscientiousness cCON using Random Forest...\n",
      "cCON score:  0.5895020188425303\n",
      "training Openness to Experience cOPN using Random Forest...\n",
      "cOPN score:  0.7864513234634365\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "name=\"RF\"\n",
    "\n",
    "\n",
    "print(\"training Extraversion cEXT using Random Forest...\")\n",
    "clf_rf_cEXT = RandomForestClassifier(n_estimators=100)\n",
    "clf_rf_cEXT.fit(train_x_vectors, train_y_cEXT)\n",
    "evaluation.append([data, vec_name, name, \"cEXT\", clf_rf_cEXT.score(test_x_vectors, test_y_cEXT)])\n",
    "print(\"cEXT score: \", clf_rf_cEXT.score(test_x_vectors, test_y_cEXT))\n",
    "\n",
    "try:\n",
    "    print(\"training Neuroticism cNEU using Random Forest...\")\n",
    "    clf_rf_cNEU = RandomForestClassifier(n_estimators=100)\n",
    "    clf_rf_cNEU.fit(train_x_vectors, train_y_cNEU)\n",
    "    evaluation.append([data, vec_name, name, \"cNEU\", clf_rf_cNEU.score(test_x_vectors, test_y_cNEU)])\n",
    "    print(\"cNEU score: \", clf_rf_cNEU.score(test_x_vectors, test_y_cNEU))\n",
    "except:\n",
    "    print(\"with this data not available (MBTI only 4 dimensions)\")\n",
    "\n",
    "print(\"training Agreeableness cAGR using using Random Forest...\")\n",
    "clf_rf_cAGR = RandomForestClassifier(n_estimators=100)\n",
    "clf_rf_cAGR.fit(train_x_vectors, train_y_cAGR)\n",
    "evaluation.append([data, vec_name, name, \"cAGR\", clf_rf_cAGR.score(test_x_vectors, test_y_cAGR)])\n",
    "print(\"cAGR score: \", clf_rf_cAGR.score(test_x_vectors, test_y_cAGR))\n",
    "\n",
    "print(\"training Conscientiousness cCON using Random Forest...\")\n",
    "clf_rf_cCON = RandomForestClassifier(n_estimators=100)\n",
    "clf_rf_cCON.fit(train_x_vectors, train_y_cCON)\n",
    "evaluation.append([data, vec_name, name, \"cCON\", clf_rf_cCON.score(test_x_vectors, test_y_cCON)])\n",
    "print(\"cCON score: \", clf_rf_cCON.score(test_x_vectors, test_y_cCON))\n",
    "\n",
    "print(\"training Openness to Experience cOPN using Random Forest...\")\n",
    "clf_rf_cOPN = RandomForestClassifier(n_estimators=100)\n",
    "clf_rf_cOPN.fit(train_x_vectors, train_y_cOPN)\n",
    "evaluation.append([data, vec_name, name, \"cOPN\", clf_rf_cOPN.score(test_x_vectors, test_y_cOPN)])\n",
    "print(\"cOPN score: \", clf_rf_cOPN.score(test_x_vectors, test_y_cOPN))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = \"data/evaluation/evaluation\" + str(data) + vec_name + \".p\"\n",
    "pickle.dump(evaluation, open(filename, \"wb\"))\n",
    "print(\"evaluation saved as\", filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}