Extract Factors From Model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\u001b[32mimport \u001b[39m\u001b[36m$ivy.$                                     // Or use any other 2.x version here\n",
       "\n",
       "\u001b[39m"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import $ivy.`org.apache.spark::spark-mllib:2.4.4` // Or use any other 2.x version here\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\u001b[32mimport \u001b[39m\u001b[36morg.apache.spark.ml.recommendation.ALSModel\n",
       "\u001b[39m\n",
       "\u001b[32mimport \u001b[39m\u001b[36morg.apache.log4j.{Level, Logger}\n",
       "\u001b[39m"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import org.apache.spark.ml.recommendation.ALSModel\n",
    "import org.apache.log4j.{Level, Logger}\n",
    "Logger.getLogger(\"org\").setLevel(Level.OFF)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading spark-stubs\n",
      "Creating SparkSession\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<a href=\"http://mboyanov:4041\">Spark UI</a>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32mimport \u001b[39m\u001b[36morg.apache.spark.sql._\n",
       "\n",
       "\u001b[39m\n",
       "\u001b[36mspark\u001b[39m: \u001b[32mSparkSession\u001b[39m = org.apache.spark.sql.SparkSession@4d444874"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import org.apache.spark.sql._\n",
    "\n",
    "val spark = {\n",
    "  NotebookSparkSession.builder()\n",
    "    .master(\"local[*]\")\n",
    "    .getOrCreate()\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\u001b[36mmodel_path\u001b[39m: \u001b[32mString\u001b[39m = \u001b[32m\"data_dir/rank_500_it_10_lambda_0.1_alpha_1.0\"\u001b[39m"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val model_path = \"data_dir/rank_500_it_10_lambda_0.1_alpha_1.0\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<script>\n",
       "var comm = Jupyter.notebook.kernel.comm_manager.new_comm('cancel-stage-f537e106-34e3-4220-8f7c-c1f48c2a61af', {});\n",
       "\n",
       "function cancelStage(stageId) {\n",
       "  console.log('Cancelling stage ' + stageId);\n",
       "  comm.send({ 'stageId': stageId });\n",
       "}\n",
       "</script>\n",
       "          "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "  <span style=\"float: left;\">first at ReadWrite.scala:615</span>\n",
       "</div>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div class=\"progress\">\n",
       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: blue; width: 100%; word-wrap: normal; white-space: nowrap; text-align: center; color: white\" aria-valuenow=\"100\" aria-valuemin=\"0\" aria-valuemax=\"100\">\n",
       "    1 / 1\n",
       "  </div>\n",
       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: red; width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\"></div>\n",
       "</div>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "  <span style=\"float: left;\">load at ALS.scala:527</span>\n",
       "</div>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div class=\"progress\">\n",
       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: blue; width: 100%; word-wrap: normal; white-space: nowrap; text-align: center; color: white\" aria-valuenow=\"100\" aria-valuemin=\"0\" aria-valuemax=\"100\">\n",
       "    1 / 1\n",
       "  </div>\n",
       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: red; width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\"></div>\n",
       "</div>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "  <span style=\"float: left;\">load at ALS.scala:529</span>\n",
       "</div>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div class=\"progress\">\n",
       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: blue; width: 100%; word-wrap: normal; white-space: nowrap; text-align: center; color: white\" aria-valuenow=\"100\" aria-valuemin=\"0\" aria-valuemax=\"100\">\n",
       "    1 / 1\n",
       "  </div>\n",
       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: red; width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\"></div>\n",
       "</div>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[36mmodel\u001b[39m: \u001b[32mALSModel\u001b[39m = als_a0c6620a7d35"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val model = ALSModel.load(s\"${model_path}/model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "  <span style=\"float: left;\">collect at cmd5.sc:1</span>\n",
       "</div>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div class=\"progress\">\n",
       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: blue; width: 100%; word-wrap: normal; white-space: nowrap; text-align: center; color: white\" aria-valuenow=\"100\" aria-valuemin=\"0\" aria-valuemax=\"100\">\n",
       "    10 / 10\n",
       "  </div>\n",
       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: red; width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\"></div>\n",
       "</div>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[36muFactors\u001b[39m: \u001b[32mArray\u001b[39m[\u001b[32mRow\u001b[39m] = \u001b[33mArray\u001b[39m(\n",
       "  [0,WrappedArray(-0.037335385, -0.008733383, 0.024352103, -0.0041547096, 0.032182526, 0.014274008, -0.038711745, 0.0044250437, -0.061623797, 0.012769637, 0.06440961, -0.042560827, -0.030923888, -0.013964174, 0.032346394, 0.02694291, -0.055387188, -0.021201564, 0.0031795627, 0.016078277, -0.0225782, -0.024403851, -0.095074296, 0.03209017, -0.0458251, 0.02693864, 0.014098344, -0.0027378493, -0.037523795, -0.005256525, 0.009141889, -0.009772336, -0.0038924874, 0.03563304, 0.020321155, -0.07970114, 0.0325638, -0.014674704, 0.03623155, 0.018706108, 0.056124996, -0.009984402, 0.06657998, -0.08080357, -0.026738599, 0.011224127, -0.1028453, 0.007832226, -0.047058403, 0.043576017, -0.05755836, -0.01138259, -0.03373226, -0.026041199, 0.032833308, 0.0406594, -4.7975953E-4, -0.0019218441, 0.0071669645, 0.012721727, 0.04898993, -0.012035682, 0.066750795, 0.022443317, -0.04937081, 0.01668032, 0.0031412523, -0.08062505, -0.038194623, 0.03315475, -0.001098953, -0.046631105, -0.0125239575, 0.037338596, 0.033102114, 0.019146118, -0.05295058, -0.006623996, 0.016422939, -0.026641123, 0.013359, 0.024964359, 0.0036562656, 0.0359808, 0.002894848, 0.006522968, 0.039603233, -0.010266425, -0.01415203, -0.013581696, -0.05313097, -0.030390302, 0.0051244865, -0.021488477, 0.0081220055, -0.021704376, -0.023756457, -0.016389033, -0.02756662, -0.022650603, 0.010236058, -0.019490585, -0.03760005, 0.04150946, 0.025576923, -0.042545088, -0.040217273, 0.020301994, 0.0018157649, -0.023596384, 0.02232842, 0.036166985, 0.03926218, 0.014105963, -0.011414022, -0.010533553, -0.0020845903, 0.024395425, -0.052650712, -0.05501048, 0.037559308, -0.014192862, -0.0049006795, -0.024709398, 0.03443293, -0.0110125635, 0.06763879, 0.04676938, -0.0014624397, 0.062456228, 0.0720098, -0.004818086, 0.03477422, 0.02341294, -0.04816011, 0.03845193, -0.035161413, 0.057253193, -0.01167871, 0.03138463, 0.023687258, 0.015847392, 0.0073789177, 0.041559726, -0.022144582, 0.010862852, 0.051311374, -0.013756715, 0.070423126, 0.0027484149, 0.02988998, -0.08980274, 0.018874157, -0.013019497, 0.012582485, 0.0136006335, 0.0034507255, 0.025652431, 0.040071797, 0.016571742, -0.034697663, -0.009561478, -0.0014368859, 0.046124134, 0.027925888, 0.0032203873, -0.02735427, 0.04585878, 0.034735613, 0.052151907, 0.014283457, 0.05032223, 0.06608651, -0.036670618, -0.030681556, -0.07317128, 0.019442804, -0.007229736, 0.021314343, -0.0054300297, -0.029344823, -0.03171738, -0.040794775, -0.028847534, -0.036198262, 0.0022128175, -0.008238222, 0.022494117, 0.022222355, 0.03774045, -0.019818444, -0.015786383, -0.008007311, -0.05848404, -0.015436559, -8.495574E-4, -0.019750139, 0.04468871, 0.01841788, 0.046795547, 0.02641853, -0.048054677, -0.056188174, 0.019815153, -0.024721231, -0.084114075, 0.017381858, 0.012891655, -0.081582375, -0.018351343, -0.036993574, 0.05464615, -0.006783785, -8.325992E-4, -0.046780065, -0.03422388, 0.012797083, -0.002681464, -0.050126564, 0.052968565, 0.031757865, 0.010827615, 0.0019362539, 0.041591175, 0.04441633, -0.009245033, 0.010530808, ..."
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val uFactors = model.userFactors.collect"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "  <span style=\"float: left;\">collect at cmd6.sc:1</span>\n",
       "</div>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div class=\"progress\">\n",
       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: blue; width: 100%; word-wrap: normal; white-space: nowrap; text-align: center; color: white\" aria-valuenow=\"100\" aria-valuemin=\"0\" aria-valuemax=\"100\">\n",
       "    5 / 5\n",
       "  </div>\n",
       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: red; width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\"></div>\n",
       "</div>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[36miFactors\u001b[39m: \u001b[32mArray\u001b[39m[\u001b[32mRow\u001b[39m] = \u001b[33mArray\u001b[39m(\n",
       "  [5,WrappedArray(-0.0033890076, -0.0183697, -0.07358357, -0.056528028, -8.634869E-4, -4.665708E-4, -0.008711772, 0.0022401747, -0.048226774, 0.03837093, 0.017697545, -0.089427695, -0.07024473, -0.02006952, 0.031444, 0.043658633, -0.045298938, -0.012100678, -0.044062935, 0.081118554, 0.051584456, -0.07682077, 0.02776613, 0.092098124, -0.060605828, 0.033082582, 0.10760073, -0.072611734, -0.0032410438, -0.093005314, 0.018406015, 0.040117357, -0.04335531, 0.052118376, 0.004615605, -0.03310666, 0.06517782, 0.031704623, 0.050520387, -0.048969544, 0.07915561, -0.081822135, 0.077515535, -0.12779436, 0.025762249, 0.047122423, -0.08713052, -0.001206585, -0.045517612, 0.020266576, -0.13630173, 0.012876112, 0.02311005, 0.08051183, 0.041162014, 0.09222604, 0.0036695374, -0.045187276, -0.024773045, 0.012751852, -0.011284296, -0.0092498055, 0.16199766, -0.030298814, -0.07298994, 0.018984852, -0.043937664, -0.030564398, -0.04369474, 0.075715914, -0.0015815308, -0.08362986, -0.048056908, 0.020371152, 0.030798465, -0.06518974, 0.009890549, -0.15312065, 0.08047196, -0.059188098, -0.010077895, -0.031923994, 0.023525424, -0.040793393, -0.030496866, 0.041046377, -0.011607531, 0.04484228, -0.00581624, 0.0033507515, 0.03158372, -0.0404447, 0.04547132, -0.023491504, 0.038657073, 0.032218244, -0.08471914, 0.02227912, 0.022379702, -0.047082953, 0.06453248, 0.023728617, 0.030933585, 0.045718998, -0.029526042, -0.03487092, 0.012317448, -0.02979765, 0.01842435, 0.00457788, -0.03932717, 0.044275276, 0.04720283, 0.015879087, 0.013472971, 0.021518523, -0.07068604, 0.0017652202, -0.036841474, -0.10890125, 0.036661014, 0.04275491, -0.064881615, -0.035589047, -0.016183466, -0.103722624, 0.14033565, -0.057821352, 0.03357356, 0.014127042, 0.038468674, 0.04861189, 0.016572628, -0.0071277455, -0.03062007, 0.024195328, -0.009996692, 0.02521539, -0.062113747, 6.48001E-4, 0.01662146, -0.010124622, 0.03232906, 0.010943723, 0.048453763, -0.023408163, -0.01239137, 0.048559375, 0.0804415, -0.121619634, 0.037000597, -0.06498804, 0.021395378, -0.057454143, -0.09268905, -0.013475306, 0.0144404, 0.02783538, 0.045760866, -0.029316384, -0.04840355, -3.2225085E-4, -0.003963608, -0.020049354, 0.0918842, -0.08578963, -0.10206996, 0.011545259, -0.02970338, 0.09692536, -0.047001295, 0.0075962376, -0.0027852717, -0.062109347, 0.037181, -0.06391684, 0.057492975, 0.02449821, -0.020553356, -0.060326096, -0.0687971, -0.021063138, -0.032194726, -0.111233726, -0.01405916, 0.022889063, -0.05974384, -0.0047451016, 0.07999591, -7.8247365E-4, 0.06309156, -0.04767456, 0.022967098, -0.04546943, 0.00160397, -0.022771668, 0.02036045, 0.0443068, -0.025240343, 0.008313448, 0.09681253, -0.012119567, -0.011126318, -0.062171306, 0.019172736, -0.057128694, 0.08669089, 0.014717249, -0.10123154, 0.016134959, -0.014852902, 0.02800171, 0.09485099, -0.019594202, -0.06535874, -0.0019326813, -0.012151987, -0.050176788, -0.04569228, 0.030491192, 0.045655, -0.011350033, -0.04794073, 0.058848638, 0.030160857, -0.018738868, 0.04700824, 0.04259064, -0.06825557, -0.10145607,..."
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val iFactors = model.itemFactors.collect"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "  <span style=\"float: left;\">json at cmd7.sc:1</span>\n",
       "</div>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div class=\"progress\">\n",
       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: blue; width: 100%; word-wrap: normal; white-space: nowrap; text-align: center; color: white\" aria-valuenow=\"100\" aria-valuemin=\"0\" aria-valuemax=\"100\">\n",
       "    10 / 10\n",
       "  </div>\n",
       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: red; width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\"></div>\n",
       "</div>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "  <span style=\"float: left;\">json at cmd7.sc:1</span>\n",
       "</div>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div class=\"progress\">\n",
       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: blue; width: 100%; word-wrap: normal; white-space: nowrap; text-align: center; color: white\" aria-valuenow=\"100\" aria-valuemin=\"0\" aria-valuemax=\"100\">\n",
       "    1 / 1\n",
       "  </div>\n",
       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: red; width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\"></div>\n",
       "</div>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "model.userFactors.repartition(1).write.json(s\"${model_path}/user_factors\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "  <span style=\"float: left;\">json at cmd8.sc:1</span>\n",
       "</div>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div class=\"progress\">\n",
       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: blue; width: 100%; word-wrap: normal; white-space: nowrap; text-align: center; color: white\" aria-valuenow=\"100\" aria-valuemin=\"0\" aria-valuemax=\"100\">\n",
       "    5 / 5\n",
       "  </div>\n",
       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: red; width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\"></div>\n",
       "</div>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "  <span style=\"float: left;\">json at cmd8.sc:1</span>\n",
       "</div>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div class=\"progress\">\n",
       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: blue; width: 100%; word-wrap: normal; white-space: nowrap; text-align: center; color: white\" aria-valuenow=\"100\" aria-valuemin=\"0\" aria-valuemax=\"100\">\n",
       "    1 / 1\n",
       "  </div>\n",
       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: red; width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\"></div>\n",
       "</div>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "model.itemFactors.repartition(1).write.json(s\"${model_path}/item_factors\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\u001b[32mimport \u001b[39m\u001b[36mspark.implicits._\u001b[39m"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import spark.implicits._"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\u001b[36musers\u001b[39m: \u001b[32mDataFrame\u001b[39m = [user: int]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "val users = spark.createDataset(Seq(0)).toDF(\"user\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\u001b[36mrecs\u001b[39m: \u001b[32mDataFrame\u001b[39m = [user: int, recommendations: array<struct<item:int,rating:float>>]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val recs = model.recommendForUserSubset(users, 1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "  <span style=\"float: left;\">run at ThreadPoolExecutor.java:1149</span>\n",
       "</div>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div class=\"progress\">\n",
       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: blue; width: 100%; word-wrap: normal; white-space: nowrap; text-align: center; color: white\" aria-valuenow=\"100\" aria-valuemin=\"0\" aria-valuemax=\"100\">\n",
       "    1 / 1\n",
       "  </div>\n",
       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: red; width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\"></div>\n",
       "</div>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "  <span style=\"float: left;\">run at ThreadPoolExecutor.java:1149</span>\n",
       "</div>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div class=\"progress\">\n",
       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: blue; width: 100%; word-wrap: normal; white-space: nowrap; text-align: center; color: white\" aria-valuenow=\"100\" aria-valuemin=\"0\" aria-valuemax=\"100\">\n",
       "    5 / 5\n",
       "  </div>\n",
       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: red; width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\"></div>\n",
       "</div>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "  <span style=\"float: left;\">head at cmd16.sc:1</span>\n",
       "</div>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div class=\"progress\">\n",
       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: blue; width: 100%; word-wrap: normal; white-space: nowrap; text-align: center; color: white\" aria-valuenow=\"100\" aria-valuemin=\"0\" aria-valuemax=\"100\">\n",
       "    5 / 5\n",
       "  </div>\n",
       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: red; width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\"></div>\n",
       "</div>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "  <span style=\"float: left;\">head at cmd16.sc:1</span>\n",
       "</div>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div class=\"progress\">\n",
       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: blue; width: 100%; word-wrap: normal; white-space: nowrap; text-align: center; color: white\" aria-valuenow=\"100\" aria-valuemin=\"0\" aria-valuemax=\"100\">\n",
       "    1 / 1\n",
       "  </div>\n",
       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: red; width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\"></div>\n",
       "</div>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "  <span style=\"float: left;\">head at cmd16.sc:1</span>\n",
       "</div>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div class=\"progress\">\n",
       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: blue; width: 100%; word-wrap: normal; white-space: nowrap; text-align: center; color: white\" aria-valuenow=\"100\" aria-valuemin=\"0\" aria-valuemax=\"100\">\n",
       "    4 / 4\n",
       "  </div>\n",
       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: red; width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\"></div>\n",
       "</div>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "  <span style=\"float: left;\">head at cmd16.sc:1</span>\n",
       "</div>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div class=\"progress\">\n",
       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: blue; width: 100%; word-wrap: normal; white-space: nowrap; text-align: center; color: white\" aria-valuenow=\"100\" aria-valuemin=\"0\" aria-valuemax=\"100\">\n",
       "    20 / 20\n",
       "  </div>\n",
       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: red; width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\"></div>\n",
       "</div>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "  <span style=\"float: left;\">head at cmd16.sc:1</span>\n",
       "</div>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div class=\"progress\">\n",
       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: blue; width: 100%; word-wrap: normal; white-space: nowrap; text-align: center; color: white\" aria-valuenow=\"100\" aria-valuemin=\"0\" aria-valuemax=\"100\">\n",
       "    100 / 100\n",
       "  </div>\n",
       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: red; width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\"></div>\n",
       "</div>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "  <span style=\"float: left;\">head at cmd16.sc:1</span>\n",
       "</div>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div class=\"progress\">\n",
       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: blue; width: 100%; word-wrap: normal; white-space: nowrap; text-align: center; color: white\" aria-valuenow=\"100\" aria-valuemin=\"0\" aria-valuemax=\"100\">\n",
       "    75 / 75\n",
       "  </div>\n",
       "  <div class=\"progress-bar\" role=\"progressbar\" style=\"background-color: red; width: 0%\" aria-valuenow=\"0\" aria-valuemin=\"0\" aria-valuemax=\"100\"></div>\n",
       "</div>\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "\u001b[36mres16\u001b[39m: \u001b[32mArray\u001b[39m[\u001b[32mRow\u001b[39m] = \u001b[33mArray\u001b[39m(\n",
       "  [0,WrappedArray([9,0.9998198], [6,0.9994253], [3,0.9896057], [8,0.95701975], [4,0.9504272], [0,0.6889168], [164,0.64174086], [270,0.58688635], [321,0.5568616], [305,0.5554546], [7,0.54067075], [299,0.5247053], [319,0.5201434], [131,0.51783526], [281,0.49550053], [297,0.48222396], [1589,0.48043996], [324,0.4660332], [66,0.462397], [328,0.46189767], [1163,0.46016228], [796,0.45715895], [262,0.45160472], [1385,0.45033094], [296,0.4120285], [279,0.41009313], [42,0.408076], [2,0.389066], [61,0.38220748], [714,0.37462252], [80,0.36884978], [819,0.3649351], [170,0.35815194], [224,0.35450587], [249,0.33256194], [772,0.33116844], [362,0.32680166], [329,0.32517686], [106,0.3221301], [438,0.32121447], [285,0.32096958], [1108,0.3195253], [650,0.31649455], [5,0.31537923], [1002,0.30981508], [471,0.3032417], [259,0.3010294], [558,0.29749772], [1940,0.2972986], [234,0.29682094], [996,0.29600406], [768,0.29519916], [141,0.29483795], [37,0.2914761], [1258,0.28993866], [186,0.28962144], [280,0.2836928], [2867,0.2803334], [2583,0.2787224], [2687,0.27828377], [27,0.2748715], [898,0.27481347], [2938,0.27203962], [945,0.26752093], [317,0.2652719], [2516,0.2618133], [513,0.26179075], [1169,0.261371], [14,0.25761166], [60,0.24452569], [2027,0.24257608], [1392,0.23913297], [915,0.23902807], [50,0.23743781], [1151,0.23618576], [1821,0.23494677], [4622,0.23323077], [304,0.23315504], [123,0.23026234], [1065,0.22946663], [1243,0.22943187], [589,0.22589378], [797,0.22509322], [450,0.2221728], [1135,0.22107501], [929,0.21828762], [311,0.21374914], [264,0.21334253], [641,0.21216418], [32,0.21037428], [16,0.21010458], [1140,0.20959654], [250,0.20908491], [1794,0.20833544], [220,0.205939], [1993,0.2049496], [217,0.20466578], [820,0.19959183], [331,0.19949302], [1871,0.1990388], [1230,0.19730802], [586,0.19704494], [1277,0.19544132], [3407,0.19262671], [33,0.19188446], [1797,0.19002278], [1580,0.18953654], [954,0.18838148], [3885,0.18820256], [221,0.18731034], [13,0.18552041], [26,0.18330222], [1013,0.18308902], [1124,0.18212625], [21,0.18119416], [951,0.18075813], [1289,0.17981702], [3116,0.17893745], [1222,0.17426504], [1417,0.17418155], [1939,0.17393215], [75,0.17355247], [1005,0.17309628], [1936,0.17196643], [1177,0.17169881], [2926,0.16941611], [322,0.16923901], [2079,0.16795963], [1220,0.1671846], [604,0.1657677], [40,0.1656208], [1021,0.1655938], [1868,0.16426156], [4646,0.16377021], [793,0.16355193], [858,0.16023897], [1305,0.15998177], [2751,0.15740821], [973,0.15595993], [505,0.15539056], [364,0.15538101], [3082,0.15507089], [672,0.15380043], [330,0.15271187], [4714,0.15093924], [1118,0.1505287], [303,0.15027586], [1771,0.14905584], [1521,0.1490061], [959,0.14713989], [2434,0.14681855], [1883,0.14599107], [627,0.14546378], [3679,0.14387256], [499,0.14368421], [1048,0.14364237], [256,0.14257966], [778,0.14235844], [3255,0.14230141], [76,0.14223325], [1143,0.14188527], [41,0.14180815], [1862,0.14155354], [985,0.14121838], [1147,0.14032632], [1881,0.13991217], [677,0.13965268], [1084,0.13940875], [48,0.13852751..."
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recs.head(1000)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "recs.head(1000)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Scala",
   "language": "scala",
   "name": "scala"
  },
  "language_info": {
   "codemirror_mode": "text/x-scala",
   "file_extension": ".scala",
   "mimetype": "text/x-scala",
   "name": "scala",
   "nbconvert_exporter": "script",
   "version": "2.12.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}