Example_6 Use Zarr in S3 bucket with geoviews+holoviews.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import xarray as xr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from dask_jobqueue import SLURMCluster\n",
    "cluster = SLURMCluster(scheduler_port=8789)\n",
    "cluster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8e0050491e2647da99f85c9f08c206f7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HTML(value='<h2>LocalCluster</h2>'), HBox(children=(HTML(value='\\n<div>\\n  <style scoped>\\n    …"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# from distributed import LocalCluster\n",
    "# cluster = LocalCluster(n_workers=3,threads_per_worker=1)\n",
    "# cluster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table style=\"border: 2px solid white;\">\n",
       "<tr>\n",
       "<td style=\"vertical-align: top; border: 0px solid white\">\n",
       "<h3>Client</h3>\n",
       "<ul>\n",
       "  <li><b>Scheduler: </b>tcp://localhost:8789\n",
       "  <li><b>Dashboard: </b><a href='http://localhost:8787/status' target='_blank'>http://localhost:8787/status</a>\n",
       "</ul>\n",
       "</td>\n",
       "<td style=\"vertical-align: top; border: 0px solid white\">\n",
       "<h3>Cluster</h3>\n",
       "<ul>\n",
       "  <li><b>Workers: </b>40</li>\n",
       "  <li><b>Cores: </b>80</li>\n",
       "  <li><b>Memory: </b>600.00 GB</li>\n",
       "</ul>\n",
       "</td>\n",
       "</tr>\n",
       "</table>"
      ],
      "text/plain": [
       "<Client: scheduler='tcp://146.118.38.56:8789' processes=40 cores=80>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from distributed import Client\n",
    "# client=Client(cluster)\n",
    "client=Client('tcp://localhost:8789')\n",
    "client"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### The earlier created Zarr has been uploaded to an Amazon S3 Public Bucket, so lets test computing directly against that"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<xarray.Dataset>\n",
       "Dimensions:    (latitude: 7001, longitude: 10001, time: 365)\n",
       "Coordinates:\n",
       "  * latitude   (latitude) float64 10.0 9.99 9.98 9.97 ... -59.98 -59.99 -60.0\n",
       "  * longitude  (longitude) float64 80.0 80.01 80.02 80.03 ... 180.0 180.0 180.0\n",
       "  * time       (time) datetime64[ns] 2014-01-01 2014-01-02 ... 2014-12-31\n",
       "Data variables:\n",
       "    K_490      (time, latitude, longitude) float32 dask.array<shape=(365, 7001, 10001), chunksize=(7, 856, 1223)>\n",
       "Attributes:\n",
       "    Conventions:  CF-1.6\n",
       "    history:      File initialised at 2015-12-16T15:53:48.785210\\nInitialised...\n",
       "    source_path:  imos-srs/archive/oc/aqua/1d/v201508/2014/01/A20140101.L2OC_..."
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import s3fs\n",
    "import zarr\n",
    "s3 = s3fs.S3FileSystem(anon=True, client_kwargs=dict(region_name='ap-southeast-2'))\n",
    "store = s3fs.S3Map(root='imos-zarr/aqua.P1D.aust.K_490.2014.zarr', s3=s3, check=False)\n",
    "ds = xr.open_zarr(store)\n",
    "ds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import holoviews as hv\n",
    "import hvplot.xarray\n",
    "import hvplot.pandas\n",
    "import geoviews as gv\n",
    "import numpy as np\n",
    "import cartopy\n",
    "from cartopy import crs as ccrs\n",
    "\n",
    "hv.notebook_extension('bokeh')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "plate=ccrs.PlateCarree(central_longitude=133.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%opts Overlay [width=700 height=400]\n",
    "%%opts Image [projection=plate]\n",
    "\n",
    "(ds.K_490.hvplot('longitude','latitude',groupby='time',dynamic=True, rasterize=True, cmap='magma',crs=ccrs.PlateCarree()).redim(K_490=dict(range=(0, 0.5))))*\\\n",
    "(gv.feature.coastline.geoms('10m')).opts(color='black')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now with zarr the performance of the IO and xarray+dask really starts to sing. The cell below doesnt calculate anything, until the geoview plot is called, then as we browse the months it calculated the mean on the fly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<xarray.Dataset>\n",
       "Dimensions:    (latitude: 7001, longitude: 10001, month: 12)\n",
       "Coordinates:\n",
       "  * latitude   (latitude) float64 10.0 9.99 9.98 9.97 ... -59.98 -59.99 -60.0\n",
       "  * longitude  (longitude) float64 80.0 80.01 80.02 80.03 ... 180.0 180.0 180.0\n",
       "  * month      (month) int64 1 2 3 4 5 6 7 8 9 10 11 12\n",
       "Data variables:\n",
       "    K_490      (month, latitude, longitude) float32 nan nan nan ... 0.083 0.083"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "monthly_mean = ds.groupby('time.month').mean('time').compute() #or if we want to just calculate the result add .compute() to the end, however the results will be large and may kill the memory on your notebook server (needs about 9GB for all australia)\n",
    "monthly_mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%opts Overlay [width=700 height=400]\n",
    "%%opts Image [projection=plate]\n",
    "\n",
    "(monthly_mean.hvplot('longitude','latitude',groupby='month',dynamic=True, rasterize=True, cmap='magma',crs=ccrs.PlateCarree()).redim(K_490=dict(range=(0, 0.5))))*\\\n",
    "(gv.feature.coastline.geoms('10m')).opts(color='black')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:c3dis2] *",
   "language": "python",
   "name": "conda-env-c3dis2-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}