tutorials/data-access/02-weather-to-pandas-dataframe.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "Copyright (c) Microsoft Corporation. All rights reserved.\n\nLicensed under the MIT License."
      ],
      "metadata": {}
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Tutorial: Load NOAA ISD Weather Data"
      ],
      "metadata": {}
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Install azureml-opendatasets SDK"
      ],
      "metadata": {}
    },
    {
      "cell_type": "code",
      "source": [
        "!pip uninstall -y azureml-opendatasets\n!pip install azureml-opendatasets"
      ],
      "metadata": {
        "scrolled": true
      },
      "outputs": [],
      "execution_count": 4
    },
    {
      "cell_type": "markdown",
      "source": [
        "Import NoaaIsdWeather class from azureml-opendatasets"
      ],
      "metadata": {}
    },
    {
      "cell_type": "code",
      "source": [
        "from azureml.opendatasets import NoaaIsdWeather\n\nfrom datetime import datetime\nfrom dateutil import parser\nfrom dateutil.relativedelta import relativedelta"
      ],
      "metadata": {},
      "outputs": [
        {
          "metadata": {},
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>\n<div class=\"ansiout\"></div>"
            ]
          }
        }
      ],
      "execution_count": 6
    },
    {
      "cell_type": "markdown",
      "source": [
        "> 1. Set start_date and end_date.\n> 2. New an instance of NoaaIsdWeather.\n> 3. Call to_pandas_dataframe() method to get a pandas DataFrame."
      ],
      "metadata": {}
    },
    {
      "cell_type": "code",
      "source": [
        "start_date = parser.parse('2018-1-1')\nend_date = parser.parse('2018-2-28')\nisd = NoaaIsdWeather(start_date, end_date)\nisd.to_pandas_dataframe().info()\nprint('isd done')"
      ],
      "metadata": {
        "scrolled": true
      },
      "outputs": [
        {
          "metadata": {},
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<style scoped>\n  .ansiout {\n    display: block;\n    unicode-bidi: embed;\n    white-space: pre-wrap;\n    word-wrap: break-word;\n    word-break: break-all;\n    font-family: \"Source Code Pro\", \"Menlo\", monospace;;\n    font-size: 13px;\n    color: #555;\n    margin-left: 4px;\n    line-height: 19px;\n  }\n</style>\n<div class=\"ansiout\">Target paths: [&apos;/year=2018/month=1/&apos;, &apos;/year=2018/month=2/&apos;]\nLooking for parquet files...\nReading them into Pandas dataframe...\nReading ISDWeather/year=2018/month=1/part-00043-tid-9138739344806125380-ef942066-1d58-49f9-8ecb-3329cbe6e57e-383738.c000.snappy.parquet under container isdweatherdatacontainer\nReading ISDWeather/year=2018/month=2/part-00174-tid-9138739344806125380-ef942066-1d58-49f9-8ecb-3329cbe6e57e-383934.c000.snappy.parquet under container isdweatherdatacontainer\nDone.\n&lt;class &apos;pandas.core.frame.DataFrame&apos;&gt;\nInt64Index: 21666673 entries, 0 to 10482057\nData columns (total 22 columns):\nusaf                       object\nwban                       object\ndatetime                   datetime64[ns]\nlatitude                   float64\nlongitude                  float64\nelevation                  float64\nwindAngle                  float64\nwindSpeed                  float64\ntemperature                float64\nseaLvlPressure             float64\ncloudCoverage              object\npresentWeatherIndicator    float64\npastWeatherIndicator       float64\nprecipTime                 float64\nprecipDepth                float64\nsnowDepth                  float64\nstationName                object\ncountryOrRegion            object\np_k                        object\nyear                       int32\nday                        int32\nversion                    float64\ndtypes: datetime64[ns](1), float64(13), int32(2), object(6)\nmemory usage: 3.6+ GB\nisd done\n</div>"
            ]
          }
        }
      ],
      "execution_count": 8
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "mimetype": "text/x-python",
      "name": "python",
      "pygments_lexer": "ipython3",
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "version": "3.6.8",
      "nbconvert_exporter": "python",
      "file_extension": ".py"
    },
    "name": "02-weather-to-pandas-dataframe",
    "notebookId": 2741195231538698
  },
  "nbformat": 4,
  "nbformat_minor": 0
}