Unifiedcodebase.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Untitled5.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyMvlEsb3ciFfmDAnr1yVSP4",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/pjsandwich/bda-718-group-1/blob/master/Unifiedcodebase.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1RHV6KlU_6dj",
        "outputId": "131600ad-9c67-4800-c09c-1fe67dca98db"
      },
      "source": [
        "%%bash\n",
        "# Need to install pyspark\n",
        "# if pyspark is already installed, will print a message indicating pyspark already isntalled\n",
        "pip install pyspark\n",
        "pip install seaborn"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting pyspark\n",
            "  Downloading https://files.pythonhosted.org/packages/f0/26/198fc8c0b98580f617cb03cb298c6056587b8f0447e20fa40c5b634ced77/pyspark-3.0.1.tar.gz (204.2MB)\n",
            "Collecting py4j==0.10.9\n",
            "  Downloading https://files.pythonhosted.org/packages/9e/b6/6a4fb90cd235dc8e265a6a2067f2a2c99f0d91787f06aca4bcf7c23f3f80/py4j-0.10.9-py2.py3-none-any.whl (198kB)\n",
            "Building wheels for collected packages: pyspark\n",
            "  Building wheel for pyspark (setup.py): started\n",
            "  Building wheel for pyspark (setup.py): finished with status 'done'\n",
            "  Created wheel for pyspark: filename=pyspark-3.0.1-py2.py3-none-any.whl size=204612243 sha256=a65bd487b4223d7ea56575d3527aab61d86ecf9bf4e70bc5252c3e6d25afc185\n",
            "  Stored in directory: /root/.cache/pip/wheels/5e/bd/07/031766ca628adec8435bb40f0bd83bb676ce65ff4007f8e73f\n",
            "Successfully built pyspark\n",
            "Installing collected packages: py4j, pyspark\n",
            "Successfully installed py4j-0.10.9 pyspark-3.0.1\n",
            "Requirement already satisfied: seaborn in /usr/local/lib/python3.6/dist-packages (0.11.0)\n",
            "Requirement already satisfied: numpy>=1.15 in /usr/local/lib/python3.6/dist-packages (from seaborn) (1.18.5)\n",
            "Requirement already satisfied: scipy>=1.0 in /usr/local/lib/python3.6/dist-packages (from seaborn) (1.4.1)\n",
            "Requirement already satisfied: pandas>=0.23 in /usr/local/lib/python3.6/dist-packages (from seaborn) (1.1.4)\n",
            "Requirement already satisfied: matplotlib>=2.2 in /usr/local/lib/python3.6/dist-packages (from seaborn) (3.2.2)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.6/dist-packages (from pandas>=0.23->seaborn) (2.8.1)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.6/dist-packages (from pandas>=0.23->seaborn) (2018.9)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=2.2->seaborn) (2.4.7)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=2.2->seaborn) (1.3.1)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=2.2->seaborn) (0.10.0)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.6/dist-packages (from python-dateutil>=2.7.3->pandas>=0.23->seaborn) (1.15.0)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bmu-n-IX_7jx",
        "outputId": "1447f368-6b36-41d4-b134-be54f788a421"
      },
      "source": [
        "%matplotlib inline\n",
        "from pyspark.sql import SparkSession\n",
        "from pyspark.sql import SQLContext\n",
        "from pyspark.ml import feature\n",
        "import re\n",
        "from pyspark.sql import functions as fn\n",
        "from pyspark.sql.functions import col, regexp_replace, split\n",
        "from pyspark.sql.utils import AnalysisException\n",
        "from pyspark.sql import Row\n",
        "from pyspark.sql.types import IntegerType, DateType, FloatType, BooleanType\n",
        "import os\n",
        "import pandas as pd\n",
        "import seaborn as sns\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "spark = SparkSession \\\n",
        "  .builder \\\n",
        "  .master(\"local[*]\")\\\n",
        "  .config(\"spark.memory.fraction\", 0.8) \\\n",
        "  .config(\"spark.executor.memory\", \"12g\") \\\n",
        "  .config(\"spark.driver.memory\", \"12g\")\\\n",
        "  .config(\"spark.memory.offHeap.enabled\",'true')\\\n",
        "  .config(\"spark.memory.offHeap.size\",\"12g\")\\\n",
        "  .getOrCreate()\n",
        "sc = spark.sparkContext\n",
        "sqlContext = SQLContext(sc)\n",
        "\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "\n",
        "from pyspark.sql.functions import col, udf\n",
        "from pyspark.ml.stat import Correlation\n",
        "from pyspark.ml.feature import VectorAssembler, StringIndexer\n",
        "from pyspark.sql.functions import create_map, lit\n",
        "from itertools import chain\n",
        "from pyspark.mllib import numpy as np\n",
        "from pyspark.ml.feature import PCA\n",
        "from pyspark.ml.clustering import KMeans\n",
        "from pyspark.ml.evaluation import ClusteringEvaluator\n",
        "from pyspark.ml.tuning import ParamGridBuilder"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "El-yjdShAI8a"
      },
      "source": [
        "new_df = spark.read.format(\"csv\").option(\"header\", \"true\").option('inferSchema','true').load(\"drive/Shared drives/IST 718 Group1/spaghetti.csv\")"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EOp5mBoFAamY",
        "outputId": "230686c8-babe-4124-e95a-81a0ba0db016"
      },
      "source": [
        "new_df.show(5)"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "+------------+----+---------------+-----+-------------+-----------------+------------+----------+----------------+-------------------+-----------+-----+-------------+----------------+--------------+-------------+----------------+---------+-------------+------+--------------------+----------+--------------+-----+------+------+---------+--------+------+-----------+-------------+---------+--------------------+----------+---------------+-------+-----------------+-----------+-------+-------+--------------+-------------+------+--------------------+-----------+------------+--------------------+------------+---------+-----+----+-----------+----------+------------+-----------+-------------+-----------+--------------+--------------+------+-----------+------------+----------------+---------+----------------+------------+---------------+------+----+----+-----+---+------+---+---+---+----+----+----+----+----+-----+----+----+----+----+-------+-----+--------+----+--------+----+-------+----+------+---------+----+----+-----+----+------+-----+------+-----------+----+----+----+----+----+------+----+------+----+-------+----+-----+----+\n",
            "|back_legroom| bed|      body_type|cabin|         city|city_fuel_economy|daysonmarket|dealer_zip|engine_cylinders|engine_displacement|engine_type|fleet|frame_damaged|franchise_dealer|franchise_make|front_legroom|fuel_tank_volume|fuel_type|has_accidents|height|highway_fuel_economy|horsepower|interior_color|isCab|is_cpo|is_new|is_oemcpo|latitude|length|listed_date|listing_color|longitude|       major_options| make_name|maximum_seating|mileage|       model_name|owner_count|  price|salvage|savings_amount|seller_rating| sp_id|             sp_name|theft_title|transmission|transmission_display|wheel_system|wheelbase|width|year|torque_ftlb|torque_rpm|torque_grade|n_body_type|n_engine_type|n_fuel_type|n_transmission|n_wheel_system|n_city|n_make_name|n_dealer_zip|n_franchise_make|n_sp_name|n_interior_color|n_model_name|n_listing_color|kmeans| pc1| pc2|  pc3|pc4|   pc5|pc6|pc7|pc8| pc9|pc10|pc11|pc12|pc13| pc14|pc15|pc16|pc17|pc18|   pc19| pc20|    pc21|pc22|    pc23|pc24|   pc25|pc26|  pc27|     pc28|pc29|pc30| pc31|pc32|  pc33| pc34|  pc35|       pc36|pc37|pc38|pc39|pc40|pc41|  pc42|pc43|  pc44|pc45|   pc46|pc47| pc48|pc49|\n",
            "+------------+----+---------------+-----+-------------+-----------------+------------+----------+----------------+-------------------+-----------+-----+-------------+----------------+--------------+-------------+----------------+---------+-------------+------+--------------------+----------+--------------+-----+------+------+---------+--------+------+-----------+-------------+---------+--------------------+----------+---------------+-------+-----------------+-----------+-------+-------+--------------+-------------+------+--------------------+-----------+------------+--------------------+------------+---------+-----+----+-----------+----------+------------+-----------+-------------+-----------+--------------+--------------+------+-----------+------------+----------------+---------+----------------+------------+---------------+------+----+----+-----+---+------+---+---+---+----+----+----+----+----+-----+----+----+----+----+-------+-----+--------+----+--------+----+-------+----+------+---------+----+----+-----+----+------+-----+------+-----------+----+----+----+----+----+------+----+------+----+-------+----+-----+----+\n",
            "|        38.4|null|SUV / Crossover| null| Little Ferry|             28.0|         104|      7643|               6|               3500|          V|false|        false|           false|          null|         43.2|            17.2|   Hybrid|         true|  69.3|                28.0|       280|          Gray|false| false| false|    false| 40.8558| 188.4|       null|        BLACK| -74.0493|Navigation System...|    Toyota|              7| 109851|Highlander Hybrid|          1|14999.0|  false|          3160|    3.5777779|351457|    Magic Auto Sales|      false|         CVT|                   6|         AWD|    109.8| 75.2|2013|        258|      4000|     0.06875|          6|            4|          7|             2|             3|1616.0|        2.0|      2271.0|            47.0|   5118.0|             2.0|       114.0|            1.0|     0|38.4|28.0|104.0|6.0|3500.0|0.0|0.0|0.0|43.2|17.2| 1.0|69.3|28.0|280.0| 0.0| 0.0| 0.0| 0.0|40.8558|188.4|-74.0493| 7.0|109851.0| 1.0|14999.0| 0.0|3160.0|3.5777779| 0.0| 6.0|109.8|75.2|2013.0|258.0|4000.0|    0.06875| 6.0| 4.0| 7.0| 2.0| 3.0|1616.0| 2.0|2271.0|47.0| 5118.0| 2.0|114.0| 1.0|\n",
            "|        35.1|null|          Sedan| null|  Los Angeles|             32.0|          23|     90045|               4|               2000|          I|false|        false|           false|          null|         42.0|            15.8|   Diesel|        false|  56.3|                45.0|       180|          null|false| false| false|    false| 33.9592| 182.5|       null|        WHITE| -118.381|Sport Package, Le...|       BMW|              5|  97500|         3 Series|          2|13600.0|  false|          1036|          5.0|402955|   Cartag Auto Sales|      false|           A|                   6|         RWD|    110.6| 80.0|2014|        280|      1750|        0.16|          4|            1|          6|             3|             2|  64.0|       15.0|       574.0|            47.0|  19114.0|             1.0|        63.0|            0.0|     0|35.1|32.0| 23.0|4.0|2000.0|0.0|0.0|0.0|42.0|15.8| 0.0|56.3|45.0|180.0| 0.0| 0.0| 0.0| 0.0|33.9592|182.5|-118.381| 5.0| 97500.0| 2.0|13600.0| 0.0|1036.0|      5.0| 0.0| 6.0|110.6|80.0|2014.0|280.0|1750.0|       0.16| 4.0| 1.0| 6.0| 3.0| 2.0|  64.0|15.0| 574.0|47.0|19114.0| 1.0| 63.0| 0.0|\n",
            "|        38.6|null|SUV / Crossover| null|Saint Charles|             18.0|          11|     55972|               6|               3600|          V|false|        false|            true|          Jeep|          4.3|            24.6| Gasoline|        false|  69.3|                25.0|       295|         Black|false| false|  true|    false| 43.8252| 189.8|       null|        WHITE| -91.7629|Sunroof/Moonroof,...|      Jeep|              5|     12|   Grand Cherokee|          1|37789.0|  false|             0|    4.7777777|435509|Whitewater Chrysl...|      false|           A|                   6|         RWD|    114.7| 84.8|2020|        260|      4000|       0.065|          6|            4|          3|             3|             2| 342.0|        5.0|      2981.0|             4.0|   3247.0|             0.0|         7.0|            0.0|     1|38.6|18.0| 11.0|6.0|3600.0|0.0|0.0|1.0| 4.3|24.6| 0.0|69.3|25.0|295.0| 0.0| 0.0| 1.0| 0.0|43.8252|189.8|-91.7629| 5.0|    12.0| 1.0|37789.0| 0.0|   0.0|4.7777777| 0.0| 6.0|114.7|84.8|2020.0|260.0|4000.0|      0.065| 6.0| 4.0| 3.0| 3.0| 2.0| 342.0| 5.0|2981.0| 4.0| 3247.0| 0.0|  7.0| 0.0|\n",
            "|        35.8|null|SUV / Crossover| null|  East Meadow|             20.0|          48|     11554|               4|               2000|          I|false|        false|            true|           Kia|          4.1|            16.9| Gasoline|        false|  67.1|                23.0|       200|          null|false| false| false|    false| 40.7604| 174.5|       null|        BLACK| -73.5319|Sunroof/Moonroof,...|Volkswagen|              5|  40351|           Tiguan|          1|18879.0|  false|           824|    3.5555556|293986|       Autoworld Kia|      false|           A|                   6|         AWD|    102.5| 71.2|2017|        207|      1700| 0.121764705|          6|            1|          3|             3|             3|2771.0|       11.0|      4472.0|             7.0|  10563.0|             1.0|        53.0|            1.0|     1|35.8|20.0| 48.0|4.0|2000.0|0.0|0.0|1.0| 4.1|16.9| 0.0|67.1|23.0|200.0| 0.0| 0.0| 0.0| 0.0|40.7604|174.5|-73.5319| 5.0| 40351.0| 1.0|18879.0| 0.0| 824.0|3.5555556| 0.0| 6.0|102.5|71.2|2017.0|207.0|1700.0|0.121764705| 6.0| 1.0| 3.0| 3.0| 3.0|2771.0|11.0|4472.0| 7.0|10563.0| 1.0| 53.0| 1.0|\n",
            "|        35.7|null|SUV / Crossover| null|      Madison|             18.0|           1|     37115|               6|               3600|          V|false|        false|            true|          Jeep|         41.2|            17.5| Gasoline|        false|  73.6|                23.0|       285|         Black|false| false|  true|    false| 36.3062| 166.8|       null|       SILVER| -86.6868|Backup Camera, Qu...|      Jeep|              4|      3|         Wrangler|          1|32190.0|  false|             0|     4.642857|276219|Rhythm Chrysler D...|      false|           A|                   8|         RWD|     96.8| 73.8|2020|        295|      3000|  0.09833334|          6|            4|          3|             3|             2|  22.0|        5.0|        67.0|             4.0|   1597.0|             0.0|       134.0|            3.0|     1|35.7|18.0|  1.0|6.0|3600.0|0.0|0.0|1.0|41.2|17.5| 0.0|73.6|23.0|285.0| 0.0| 0.0| 1.0| 0.0|36.3062|166.8|-86.6868| 4.0|     3.0| 1.0|32190.0| 0.0|   0.0| 4.642857| 0.0| 8.0| 96.8|73.8|2020.0|295.0|3000.0| 0.09833334| 6.0| 4.0| 3.0| 3.0| 2.0|  22.0| 5.0|  67.0| 4.0| 1597.0| 0.0|134.0| 3.0|\n",
            "+------------+----+---------------+-----+-------------+-----------------+------------+----------+----------------+-------------------+-----------+-----+-------------+----------------+--------------+-------------+----------------+---------+-------------+------+--------------------+----------+--------------+-----+------+------+---------+--------+------+-----------+-------------+---------+--------------------+----------+---------------+-------+-----------------+-----------+-------+-------+--------------+-------------+------+--------------------+-----------+------------+--------------------+------------+---------+-----+----+-----------+----------+------------+-----------+-------------+-----------+--------------+--------------+------+-----------+------------+----------------+---------+----------------+------------+---------------+------+----+----+-----+---+------+---+---+---+----+----+----+----+----+-----+----+----+----+----+-------+-----+--------+----+--------+----+-------+----+------+---------+----+----+-----+----+------+-----+------+-----------+----+----+----+----+----+------+----+------+----+-------+----+-----+----+\n",
            "only showing top 5 rows\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9i47JnBVAjeD",
        "outputId": "fc361181-1dd9-49f1-b82f-1fb06ca917d9"
      },
      "source": [
        "new_df.describe().show()"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "+-------+------------------+-----+-----------+-----------+-----------+------------------+------------------+------------------+------------------+-------------------+-----------+--------------+------------------+------------------+---------+-----------------+--------------------+------------------+--------------------+------------------+------------------+-----------+-------------+------------------+--------------------+----------+------------------+-----------------+-----------------+------------------+------------------+-----------------+------------------+------------------+--------------------+------------+--------------------+------------+------------------+-----------------+------------------+------------------+------------------+-------------------+------------------+------------------+------------------+-------------------+------------------+-----------------+-----------------+------------------+------------------+-----------------+------------------+------------------+------------------+-------------------+------------------+------------------+------------------+------------------+------------------+-------------------+--------------------+------------------+------------------+------------------+-------------------+-----------------+------------------+------------------+-------------------+-------------------+-------------------+-------------------+------------------+------------------+------------------+------------------+-----------------+------------------+------------------+--------------------+-----------------+------------------+--------------------+------------------+------------------+-----------------+------------------+------------------+------------------+-------------------+------------------+------------------+------------------+-------------------+------------------+-----------------+-----------------+------------------+------------------+-----------------+------------------+------------------+------------------+\n",
            "|summary|      back_legroom|  bed|  body_type|      cabin|       city| city_fuel_economy|      daysonmarket|        dealer_zip|  engine_cylinders|engine_displacement|engine_type|franchise_make|     front_legroom|  fuel_tank_volume|fuel_type|           height|highway_fuel_economy|        horsepower|      interior_color|          latitude|            length|listed_date|listing_color|         longitude|       major_options| make_name|   maximum_seating|          mileage|       model_name|       owner_count|             price|   savings_amount|     seller_rating|             sp_id|             sp_name|transmission|transmission_display|wheel_system|         wheelbase|            width|              year|       torque_ftlb|        torque_rpm|       torque_grade|       n_body_type|     n_engine_type|       n_fuel_type|     n_transmission|    n_wheel_system|           n_city|      n_make_name|      n_dealer_zip|  n_franchise_make|        n_sp_name|  n_interior_color|      n_model_name|   n_listing_color|             kmeans|               pc1|               pc2|               pc3|               pc4|               pc5|                pc6|                 pc7|               pc8|               pc9|              pc10|               pc11|             pc12|              pc13|              pc14|               pc15|               pc16|               pc17|               pc18|              pc19|              pc20|              pc21|              pc22|             pc23|              pc24|              pc25|                pc26|             pc27|              pc28|                pc29|              pc30|              pc31|             pc32|              pc33|              pc34|              pc35|               pc36|              pc37|              pc38|              pc39|               pc40|              pc41|             pc42|             pc43|              pc44|              pc45|             pc46|              pc47|              pc48|              pc49|\n",
            "+-------+------------------+-----+-----------+-----------+-----------+------------------+------------------+------------------+------------------+-------------------+-----------+--------------+------------------+------------------+---------+-----------------+--------------------+------------------+--------------------+------------------+------------------+-----------+-------------+------------------+--------------------+----------+------------------+-----------------+-----------------+------------------+------------------+-----------------+------------------+------------------+--------------------+------------+--------------------+------------+------------------+-----------------+------------------+------------------+------------------+-------------------+------------------+------------------+------------------+-------------------+------------------+-----------------+-----------------+------------------+------------------+-----------------+------------------+------------------+------------------+-------------------+------------------+------------------+------------------+------------------+------------------+-------------------+--------------------+------------------+------------------+------------------+-------------------+-----------------+------------------+------------------+-------------------+-------------------+-------------------+-------------------+------------------+------------------+------------------+------------------+-----------------+------------------+------------------+--------------------+-----------------+------------------+--------------------+------------------+------------------+-----------------+------------------+------------------+------------------+-------------------+------------------+------------------+------------------+-------------------+------------------+-----------------+-----------------+------------------+------------------+-----------------+------------------+------------------+------------------+\n",
            "|  count|            595365| 3871|     595365|      12684|     595365|            595365|            595365|            595365|            595365|             595365|     595365|        482257|            595365|            595365|   595365|           595365|              595365|            595365|              519048|            595365|            595365|          0|       595365|            595365|              556158|    595365|            595365|           595365|           595365|            595365|            595365|           595365|            595365|            595344|              595365|      595365|              595365|      595365|            595365|           595365|            595365|            595365|            595365|             595365|            595365|            595365|            595365|             595365|            595365|           595365|           595365|            595365|            595365|           595365|            595365|            595365|            595365|             595365|            595365|            595365|            595365|            595365|            595365|             595365|              595365|            595365|            595365|            595365|             595365|           595365|            595365|            595365|             595365|             595365|             595365|             595365|            595365|            595365|            595365|            595365|           595365|            595365|            595365|              595365|           595365|            595365|              595365|            595365|            595365|           595365|            595365|            595365|            595365|             595365|            595365|            595365|            595365|             595365|            595365|           595365|           595365|            595365|            595365|           595365|            595365|            595365|            595365|\n",
            "|   mean| 35.01224727688242| null|       null|       null|       null|22.736442057847317| 75.60854937727277| 50656.61339794898| 5.071114358418785| 2930.4633292182107|       null|          null|  35.3886006483426|18.571329254828928|     null|65.89472213112003|  29.522490984366364|245.81981305585649|            21481.02|36.990371745230654|193.64100233803723|       null|         null|-90.61292403651152|                null|      null|  5.45510569146658|30091.31102097033|1609.530466282742|1.2616344595332276|29122.640618459267|542.9157558808462|4.2699556692070315|233134.60238954285|                null|        null|  6.3325254255792665|        null|115.30140786024347|78.32899866640432|2017.7391348164572| 262.5168392498719|3595.6089642488223|0.09930819156954586| 5.795375945848345|2.2444399653993767|3.2405549536838745|  2.813519437655892|1.7568886313437975|627.3688090499105|6.888864814021651|1384.2782847496915|14.168782175640153|5473.464499928616| 450.3719818934603| 78.89573286975217|2.5057418558363356| 0.8123621643865527| 35.01224727688242|22.736442057847317| 75.60854937727277| 5.071114358418785|2930.4633292182107|  0.112740923635081|0.005003653221133254|0.8112519210904235|  35.3886006483426|18.571329254828928|0.08091338926540861|65.89472213112003|29.522490984366364|245.81981305585649|0.09656261285094019| 0.0608987763808756|0.48972814995842884|0.04520084318023397|36.990371745230654|193.64100233803723|-90.61292403651152|  5.45510569146658|30091.31102097033|1.2616344595332276|29122.640618459267|0.003493655152721...|542.9157558808462|4.2699556692070315|0.001766983279164...|6.3325254255792665|115.30140786024347|78.32899866640432|2017.7391348164572| 262.5168392498719|3595.6089642488223|0.09930819156954586| 5.795375945848345|2.2444399653993767|3.2405549536838745|  2.813519437655892|1.7568886313437975|627.3688090499105|6.888864814021651|1384.2782847496915|14.168782175640153|5473.464499928616| 450.3719818934603| 78.89573286975217|2.5057418558363356|\n",
            "| stddev|10.612139839452547| null|       null|       null|       null| 8.065007005084508|108.29550835482209|27311.015607177633|1.4405978403711504| 1311.9776238399359|       null|          null|14.466083861575237|5.3115827425629885|     null|7.326111761912249|  7.1049475542523055| 85.65004202341797|  49254.437036010124| 5.000563165506511|19.728630325982664|       null|         null|13.861869264110172|                null|      null|1.1413298551459516|44497.12673159059|765.1391646283436|0.6954421586320675| 15178.17601549886|962.6582601585504|0.5089753898842838|   132359.27822546|                null|        null|  0.9896728011849515|        null|14.149761991873149|7.465642730495475| 4.148136701571006|107.72906794260935|1152.5251163474866| 0.1111355411249494|1.8537867557315117|1.4621862183350645|0.8337393507699553|0.42859417996579335|0.8024999199167362|758.9883265502466|7.296148579023602|1450.8472769748146| 17.07971875434931|5100.142109093587|1927.0942696961583|103.99791312070418| 2.192957836081281|0.39042301966942067|10.612139839452547| 8.065007005084508|108.29550835482209|1.4405978403711504|1311.9776238399359|0.31627610688848395| 0.07055937243118839|0.3913087001441261|14.466083861575237|5.3115827425629885| 0.2727022875080921|7.326111761912249|7.1049475542523055| 85.65004202341797|0.29536150930621696|0.23914475004746377|  0.499894897784208|0.20774455334780317| 5.000563165506511|19.728630325982664|13.861869264110172|1.1413298551459516|44497.12673159059|0.6954421586320675| 15178.17601549886|  0.0590038589754402|962.6582601585504|0.5089753898842838|0.041998381063035606|0.9896728011849515|14.149761991873149|7.465642730495475| 4.148136701571006|107.72906794260935|1152.5251163474866| 0.1111355411249494|1.8537867557315117|1.4621862183350645|0.8337393507699553|0.42859417996579335|0.8024999199167362|758.9883265502466|7.296148579023602|1450.8472769748146| 17.07971875434931|5100.142109093587|1927.0942696961583|103.99791312070418| 2.192957836081281|\n",
            "|    min|               0.0| Long|Convertible|   Crew Cab|  Abbeville|               8.0|                 0|        08816-4351|                 2|                700|          H|         Acura|               4.0|               1.9|Biodiesel|             43.9|                10.0|                67|              #NAME?|           18.3988|             106.1|       null|        BLACK|          -157.913|101A Mid Equipmen...|AM General|                 2|                0|         1 Series|                 1|             165.0|                0|               1.0|             41593|\"The Kar Store \"\"...|           A|                   1|         AWD|              73.5|             42.5|              1923|                68|               200|        0.015111111|                 1|                 1|                 1|                  1|                 1|              0.0|              0.0|               0.0|               0.0|              0.0|               0.0|               0.0|               0.0|                  0|               0.0|               8.0|               0.0|               2.0|             700.0|                0.0|                 0.0|               0.0|               4.0|               1.9|                0.0|             43.9|              10.0|              67.0|                0.0|                0.0|                0.0|                0.0|           18.3988|             106.1|          -157.913|               2.0|              0.0|               1.0|             165.0|                 0.0|              0.0|               1.0|                 0.0|               1.0|              73.5|             42.5|            1923.0|              68.0|             200.0|        0.015111111|               1.0|               1.0|               1.0|                1.0|               1.0|              0.0|              0.0|               0.0|               0.0|              0.0|               0.0|               0.0|               0.0|\n",
            "|    max|              47.5|Short|      Wagon|Regular Cab|north hills|             127.0|              2979|             99403|                12|               8400|          W|         smart|              67.0|              63.5|  Propane|            117.6|               127.0|               797|xDrive35i Premium...|           48.9237|             290.0|       null|       YELLOW|          -66.0785|         Z71 Package|     smart|                15|          1225238|               xD|                16|           89999.0|            31773|               5.0|            440951|      vBs Auto Sales|           M|                  10|         RWD|             201.0|            109.0|              2021|               935|              7500|                1.0|                 9|                 5|                 8|                  3|                 3|           4494.0|             74.0|            7878.0|              47.0|          24600.0|           18299.0|            1115.0|              14.0|                  1|              47.5|             127.0|            2979.0|              12.0|            8400.0|                1.0|                 1.0|               1.0|              67.0|              63.5|                1.0|            117.6|             127.0|             797.0|                1.0|                1.0|                1.0|                1.0|           48.9237|             290.0|          -66.0785|              15.0|        1225238.0|              16.0|           89999.0|                 1.0|          31773.0|               5.0|                 1.0|              10.0|             201.0|            109.0|            2021.0|             935.0|            7500.0|                1.0|               9.0|               5.0|               8.0|                3.0|               3.0|           4494.0|             74.0|            7878.0|              47.0|          24600.0|           18299.0|            1115.0|              14.0|\n",
            "+-------+------------------+-----+-----------+-----------+-----------+------------------+------------------+------------------+------------------+-------------------+-----------+--------------+------------------+------------------+---------+-----------------+--------------------+------------------+--------------------+------------------+------------------+-----------+-------------+------------------+--------------------+----------+------------------+-----------------+-----------------+------------------+------------------+-----------------+------------------+------------------+--------------------+------------+--------------------+------------+------------------+-----------------+------------------+------------------+------------------+-------------------+------------------+------------------+------------------+-------------------+------------------+-----------------+-----------------+------------------+------------------+-----------------+------------------+------------------+------------------+-------------------+------------------+------------------+------------------+------------------+------------------+-------------------+--------------------+------------------+------------------+------------------+-------------------+-----------------+------------------+------------------+-------------------+-------------------+-------------------+-------------------+------------------+------------------+------------------+------------------+-----------------+------------------+------------------+--------------------+-----------------+------------------+--------------------+------------------+------------------+-----------------+------------------+------------------+------------------+-------------------+------------------+------------------+------------------+-------------------+------------------+-----------------+-----------------+------------------+------------------+-----------------+------------------+------------------+------------------+\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WLNZq6e-AnAK",
        "outputId": "634eef04-4ac3-4711-e532-e62d15329f21"
      },
      "source": [
        "new_df.printSchema()"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "root\n",
            " |-- back_legroom: double (nullable = true)\n",
            " |-- bed: string (nullable = true)\n",
            " |-- body_type: string (nullable = true)\n",
            " |-- cabin: string (nullable = true)\n",
            " |-- city: string (nullable = true)\n",
            " |-- city_fuel_economy: double (nullable = true)\n",
            " |-- daysonmarket: integer (nullable = true)\n",
            " |-- dealer_zip: string (nullable = true)\n",
            " |-- engine_cylinders: integer (nullable = true)\n",
            " |-- engine_displacement: integer (nullable = true)\n",
            " |-- engine_type: string (nullable = true)\n",
            " |-- fleet: boolean (nullable = true)\n",
            " |-- frame_damaged: boolean (nullable = true)\n",
            " |-- franchise_dealer: boolean (nullable = true)\n",
            " |-- franchise_make: string (nullable = true)\n",
            " |-- front_legroom: double (nullable = true)\n",
            " |-- fuel_tank_volume: double (nullable = true)\n",
            " |-- fuel_type: string (nullable = true)\n",
            " |-- has_accidents: boolean (nullable = true)\n",
            " |-- height: double (nullable = true)\n",
            " |-- highway_fuel_economy: double (nullable = true)\n",
            " |-- horsepower: integer (nullable = true)\n",
            " |-- interior_color: string (nullable = true)\n",
            " |-- isCab: boolean (nullable = true)\n",
            " |-- is_cpo: boolean (nullable = true)\n",
            " |-- is_new: boolean (nullable = true)\n",
            " |-- is_oemcpo: boolean (nullable = true)\n",
            " |-- latitude: double (nullable = true)\n",
            " |-- length: double (nullable = true)\n",
            " |-- listed_date: string (nullable = true)\n",
            " |-- listing_color: string (nullable = true)\n",
            " |-- longitude: double (nullable = true)\n",
            " |-- major_options: string (nullable = true)\n",
            " |-- make_name: string (nullable = true)\n",
            " |-- maximum_seating: integer (nullable = true)\n",
            " |-- mileage: integer (nullable = true)\n",
            " |-- model_name: string (nullable = true)\n",
            " |-- owner_count: integer (nullable = true)\n",
            " |-- price: double (nullable = true)\n",
            " |-- salvage: boolean (nullable = true)\n",
            " |-- savings_amount: integer (nullable = true)\n",
            " |-- seller_rating: double (nullable = true)\n",
            " |-- sp_id: integer (nullable = true)\n",
            " |-- sp_name: string (nullable = true)\n",
            " |-- theft_title: boolean (nullable = true)\n",
            " |-- transmission: string (nullable = true)\n",
            " |-- transmission_display: integer (nullable = true)\n",
            " |-- wheel_system: string (nullable = true)\n",
            " |-- wheelbase: double (nullable = true)\n",
            " |-- width: double (nullable = true)\n",
            " |-- year: integer (nullable = true)\n",
            " |-- torque_ftlb: integer (nullable = true)\n",
            " |-- torque_rpm: integer (nullable = true)\n",
            " |-- torque_grade: double (nullable = true)\n",
            " |-- n_body_type: integer (nullable = true)\n",
            " |-- n_engine_type: integer (nullable = true)\n",
            " |-- n_fuel_type: integer (nullable = true)\n",
            " |-- n_transmission: integer (nullable = true)\n",
            " |-- n_wheel_system: integer (nullable = true)\n",
            " |-- n_city: double (nullable = true)\n",
            " |-- n_make_name: double (nullable = true)\n",
            " |-- n_dealer_zip: double (nullable = true)\n",
            " |-- n_franchise_make: double (nullable = true)\n",
            " |-- n_sp_name: double (nullable = true)\n",
            " |-- n_interior_color: double (nullable = true)\n",
            " |-- n_model_name: double (nullable = true)\n",
            " |-- n_listing_color: double (nullable = true)\n",
            " |-- kmeans: integer (nullable = true)\n",
            " |-- pc1: double (nullable = true)\n",
            " |-- pc2: double (nullable = true)\n",
            " |-- pc3: double (nullable = true)\n",
            " |-- pc4: double (nullable = true)\n",
            " |-- pc5: double (nullable = true)\n",
            " |-- pc6: double (nullable = true)\n",
            " |-- pc7: double (nullable = true)\n",
            " |-- pc8: double (nullable = true)\n",
            " |-- pc9: double (nullable = true)\n",
            " |-- pc10: double (nullable = true)\n",
            " |-- pc11: double (nullable = true)\n",
            " |-- pc12: double (nullable = true)\n",
            " |-- pc13: double (nullable = true)\n",
            " |-- pc14: double (nullable = true)\n",
            " |-- pc15: double (nullable = true)\n",
            " |-- pc16: double (nullable = true)\n",
            " |-- pc17: double (nullable = true)\n",
            " |-- pc18: double (nullable = true)\n",
            " |-- pc19: double (nullable = true)\n",
            " |-- pc20: double (nullable = true)\n",
            " |-- pc21: double (nullable = true)\n",
            " |-- pc22: double (nullable = true)\n",
            " |-- pc23: double (nullable = true)\n",
            " |-- pc24: double (nullable = true)\n",
            " |-- pc25: double (nullable = true)\n",
            " |-- pc26: double (nullable = true)\n",
            " |-- pc27: double (nullable = true)\n",
            " |-- pc28: double (nullable = true)\n",
            " |-- pc29: double (nullable = true)\n",
            " |-- pc30: double (nullable = true)\n",
            " |-- pc31: double (nullable = true)\n",
            " |-- pc32: double (nullable = true)\n",
            " |-- pc33: double (nullable = true)\n",
            " |-- pc34: double (nullable = true)\n",
            " |-- pc35: double (nullable = true)\n",
            " |-- pc36: double (nullable = true)\n",
            " |-- pc37: double (nullable = true)\n",
            " |-- pc38: double (nullable = true)\n",
            " |-- pc39: double (nullable = true)\n",
            " |-- pc40: double (nullable = true)\n",
            " |-- pc41: double (nullable = true)\n",
            " |-- pc42: double (nullable = true)\n",
            " |-- pc43: double (nullable = true)\n",
            " |-- pc44: double (nullable = true)\n",
            " |-- pc45: double (nullable = true)\n",
            " |-- pc46: double (nullable = true)\n",
            " |-- pc47: double (nullable = true)\n",
            " |-- pc48: double (nullable = true)\n",
            " |-- pc49: double (nullable = true)\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TXT0XWPYAywj"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}