Colab_Spark_template.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Colab-Spark-template.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/kiat/BigDataAnalytics/blob/master/Colab_Spark_template.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LmxLySb20jtX",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "02a4ab35-d030-444c-f1b9-e50ce654dede"
      },
      "source": [
        "# Spark installation on Colab\n",
        "# !apt-get install openjdk-8-jdk-headless -qq > /dev/null\n",
        "# !wget -q https://downloads.apache.org/spark/spark-3.0.1/spark-3.0.1-bin-hadoop3.2.tgz\n",
        "# !tar xf spark-3.0.1-bin-hadoop3.2.tgz\n",
        "# !pip install -q findspark\n",
        "# !rm -rf spark-3.0.1-bin-hadoop3.2.tgz\n",
        "\n",
        "!pip install --ignore-installed -q pyspark==3.1.2 "
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[K     |████████████████████████████████| 212.4 MB 56 kB/s \n",
            "\u001b[K     |████████████████████████████████| 198 kB 19.6 MB/s \n",
            "\u001b[?25h  Building wheel for pyspark (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Nor9olN20p0M"
      },
      "source": [
        "# Set JAVA_HOME and SPARK_HOME\n",
        "import os\n",
        "# os.environ[\"JAVA_HOME\"] = \"/usr/lib/jvm/java-11-openjdk-amd64\"\n",
        "# os.environ[\"SPARK_HOME\"] = \"spark-3.0.1-bin-hadoop3.2\"\n",
        "\n",
        "# import findspark\n",
        "# findspark.init(\"spark-3.0.1-bin-hadoop3.2\")# SPARK_HOME\n",
        "\n",
        "\n",
        "import sys\n",
        "import requests\n",
        "from operator import add\n",
        "\n",
        "from pyspark import SparkConf,SparkContext\n",
        "from pyspark.streaming import StreamingContext\n",
        "\n",
        "from pyspark.sql import SparkSession\n",
        "from pyspark.sql import SQLContext\n",
        "\n",
        "from pyspark.sql.types import *\n",
        "from pyspark.sql import functions as func\n",
        "from pyspark.sql.functions import *\n",
        "\n",
        "spark = SparkSession.builder.master(\"local[*]\").getOrCreate()\n",
        "sc = SparkContext.getOrCreate()\n",
        "sqlContext = SQLContext(sc)"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SY060QsjT8Wr",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "16362343-9aac-4d37-a3b8-fba98affb036"
      },
      "source": [
        "ls -lah"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "total 16K\n",
            "drwxr-xr-x 1 root root 4.0K Aug 31 13:18 \u001b[0m\u001b[01;34m.\u001b[0m/\n",
            "drwxr-xr-x 1 root root 4.0K Sep  2 06:56 \u001b[01;34m..\u001b[0m/\n",
            "drwxr-xr-x 4 root root 4.0K Aug 25 13:35 \u001b[01;34m.config\u001b[0m/\n",
            "drwxr-xr-x 1 root root 4.0K Aug 31 13:18 \u001b[01;34msample_data\u001b[0m/\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OYvYeA5RUImn",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "65f1e44d-4e67-4747-9986-2c08404860ed"
      },
      "source": [
        "ls -la sample_data"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "total 55516\n",
            "drwxr-xr-x 1 root root     4096 Aug 31 13:18 \u001b[0m\u001b[01;34m.\u001b[0m/\n",
            "drwxr-xr-x 1 root root     4096 Aug 31 13:18 \u001b[01;34m..\u001b[0m/\n",
            "-rwxr-xr-x 1 root root     1697 Jan  1  2000 \u001b[01;32manscombe.json\u001b[0m*\n",
            "-rw-r--r-- 1 root root   301141 Aug 31 13:18 california_housing_test.csv\n",
            "-rw-r--r-- 1 root root  1706430 Aug 31 13:18 california_housing_train.csv\n",
            "-rw-r--r-- 1 root root 18289443 Aug 31 13:18 mnist_test.csv\n",
            "-rw-r--r-- 1 root root 36523880 Aug 31 13:18 mnist_train_small.csv\n",
            "-rwxr-xr-x 1 root root      930 Jan  1  2000 \u001b[01;32mREADME.md\u001b[0m*\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XL3CjM3406y4"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wsNqWX8hUbeh",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4c83bb60-981f-42f5-c3f1-0b56dca9ee72"
      },
      "source": [
        "rdd1 = sc.parallelize(range(1, 10))\n",
        "rdd1.collect()"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[1, 2, 3, 4, 5, 6, 7, 8, 9]"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dCw_vFd7YF2s",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "46e10c96-cae7-4698-b2e3-6311bc68eff6"
      },
      "source": [
        "californiaHousing = sc.textFile(\"./sample_data/california_housing_test.csv\")\n",
        "californiaHousing.take(2)"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['\"longitude\",\"latitude\",\"housing_median_age\",\"total_rooms\",\"total_bedrooms\",\"population\",\"households\",\"median_income\",\"median_house_value\"',\n",
              " '-122.050000,37.370000,27.000000,3885.000000,661.000000,1537.000000,606.000000,6.608500,344700.000000']"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2bNmkjaMYk7r"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}