tutorials/data-access/01-weather-to-spark-dataframe.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "Copyright (c) Microsoft Corporation. All rights reserved.\n\nLicensed under the MIT License."
      ],
      "metadata": {}
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Tutorial: Load NOAA ISD Weather Data\n\nIn this tutorial, you load the NOAA ISD Weather Data into Spark DataFrame by calling to_spark_dataframe() method.\n\nPrerequesits:\n> You must install the PyPi package on the cluster:\n> * azureml-opendatasets"
      ],
      "metadata": {}
    },
    {
      "cell_type": "markdown",
      "source": [
        "Import NoaaIsdWeather class from azureml-opendatasets"
      ],
      "metadata": {}
    },
    {
      "cell_type": "code",
      "source": [
        "from azureml.opendatasets import NoaaIsdWeather\n\nfrom datetime import datetime\nfrom dateutil import parser\nfrom dateutil.relativedelta import relativedelta"
      ],
      "metadata": {},
      "outputs": [
        {
          "metadata": {},
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>\n<div class=\"ansiout\"></div>"
            ]
          }
        }
      ],
      "execution_count": 4
    },
    {
      "cell_type": "markdown",
      "source": [
        "> 1. Set start_date and end_date.\n> 2. New an instance of NoaaIsdWeather.\n> 3. Call to_spark_dataframe() method to get a Spark DataFrame for the given date range."
      ],
      "metadata": {}
    },
    {
      "cell_type": "code",
      "source": [
        "start_date = parser.parse('2019-1-1')\nend_date = parser.parse('2019-3-31')\nisd = NoaaIsdWeather(start_date, end_date)\ndf = isd.to_spark_dataframe()\ndisplay(df.limit(10))"
      ],
      "metadata": {
        "scrolled": true
      },
      "outputs": [
        {
          "metadata": {},
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n  .table-result-container {\n    max-height: 300px;\n    overflow: auto;\n  }\n  table, th, td {\n    border: 1px solid black;\n    border-collapse: collapse;\n  }\n  th, td {\n    padding: 5px;\n  }\n  th {\n    text-align: left;\n  }\n</style><div class='table-result-container'><table class='table-result'><thead style='background-color: white'><tr><th>usaf</th><th>wban</th><th>datetime</th><th>latitude</th><th>longitude</th><th>elevation</th><th>windAngle</th><th>windSpeed</th><th>temperature</th><th>seaLvlPressure</th><th>cloudCoverage</th><th>presentWeatherIndicator</th><th>pastWeatherIndicator</th><th>precipTime</th><th>precipDepth</th><th>snowDepth</th><th>stationName</th><th>countryOrRegion</th><th>p_k</th><th>year</th><th>day</th><th>version</th><th>month</th></tr></thead><tbody><tr><td>999999</td><td>03055</td><td>2019-01-01T00:00:00.000+0000</td><td>36.599</td><td>-101.595</td><td>995.0</td><td>null</td><td>9.3</td><td>-6.4</td><td>null</td><td>null</td><td>null</td><td>null</td><td>1.0</td><td>0.0</td><td>null</td><td>GOODWELL 2 E</td><td>US</td><td>999999-03055</td><td>2019</td><td>1</td><td>1.0</td><td>1</td></tr><tr><td>061830</td><td>99999</td><td>2019-01-26T16:00:00.000+0000</td><td>55.533</td><td>12.717</td><td>6.0</td><td>140</td><td>7.7</td><td>0.3</td><td>999.4</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>DROGDEN</td><td>DA</td><td>061830-99999</td><td>2019</td><td>26</td><td>1.0</td><td>1</td></tr><tr><td>999999</td><td>03055</td><td>2019-01-01T00:05:00.000+0000</td><td>36.599</td><td>-101.595</td><td>995.0</td><td>null</td><td>null</td><td>-6.4</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>GOODWELL 2 E</td><td>US</td><td>999999-03055</td><td>2019</td><td>1</td><td>1.0</td><td>1</td></tr><tr><td>061830</td><td>99999</td><td>2019-01-26T17:00:00.000+0000</td><td>55.533</td><td>12.717</td><td>6.0</td><td>140</td><td>7.7</td><td>0.6</td><td>998.9</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>DROGDEN</td><td>DA</td><td>061830-99999</td><td>2019</td><td>26</td><td>1.0</td><td>1</td></tr><tr><td>999999</td><td>03055</td><td>2019-01-01T00:10:00.000+0000</td><td>36.599</td><td>-101.595</td><td>995.0</td><td>null</td><td>null</td><td>-6.5</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>GOODWELL 2 E</td><td>US</td><td>999999-03055</td><td>2019</td><td>1</td><td>1.0</td><td>1</td></tr><tr><td>061830</td><td>99999</td><td>2019-01-26T18:00:00.000+0000</td><td>55.533</td><td>12.717</td><td>6.0</td><td>140</td><td>8.2</td><td>0.8</td><td>998.6</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>DROGDEN</td><td>DA</td><td>061830-99999</td><td>2019</td><td>26</td><td>1.0</td><td>1</td></tr><tr><td>999999</td><td>03055</td><td>2019-01-01T00:15:00.000+0000</td><td>36.599</td><td>-101.595</td><td>995.0</td><td>null</td><td>null</td><td>-6.5</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>GOODWELL 2 E</td><td>US</td><td>999999-03055</td><td>2019</td><td>1</td><td>1.0</td><td>1</td></tr><tr><td>061830</td><td>99999</td><td>2019-01-26T19:00:00.000+0000</td><td>55.533</td><td>12.717</td><td>6.0</td><td>140</td><td>10.3</td><td>1.0</td><td>997.8</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>DROGDEN</td><td>DA</td><td>061830-99999</td><td>2019</td><td>26</td><td>1.0</td><td>1</td></tr><tr><td>999999</td><td>03055</td><td>2019-01-01T00:20:00.000+0000</td><td>36.599</td><td>-101.595</td><td>995.0</td><td>null</td><td>null</td><td>-6.5</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>GOODWELL 2 E</td><td>US</td><td>999999-03055</td><td>2019</td><td>1</td><td>1.0</td><td>1</td></tr><tr><td>061830</td><td>99999</td><td>2019-01-26T20:00:00.000+0000</td><td>55.533</td><td>12.717</td><td>6.0</td><td>140</td><td>10.3</td><td>1.1</td><td>997.7</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>null</td><td>DROGDEN</td><td>DA</td><td>061830-99999</td><td>2019</td><td>26</td><td>1.0</td><td>1</td></tr></tbody></table></div>"
            ]
          }
        }
      ],
      "execution_count": 6
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "mimetype": "text/x-python",
      "name": "python",
      "pygments_lexer": "ipython3",
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "version": "3.6.8",
      "nbconvert_exporter": "python",
      "file_extension": ".py"
    },
    "name": "NoaaIsdWeather.to_spark_dataframe",
    "notebookId": 2741195231538712
  },
  "nbformat": 4,
  "nbformat_minor": 0
}