diff --git a/.ipynb_checkpoints/Gauge_getter_example-checkpoint.ipynb b/.ipynb_checkpoints/Gauge_getter_example-checkpoint.ipynb new file mode 100644 index 0000000..14cfa4c --- /dev/null +++ b/.ipynb_checkpoints/Gauge_getter_example-checkpoint.ipynb @@ -0,0 +1,551 @@ +{ + "cells": [ + { + "cell_type": "code", + "source": [ + "pip install bomwater" + ], + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": "Collecting bomwater\n Downloading bomwater-0.0.8-py3-none-any.whl (10 kB)\nRequirement already satisfied: pytz in /anaconda/envs/azureml_py38/lib/python3.8/site-packages (from bomwater) (2022.5)\nRequirement already satisfied: pandas in /anaconda/envs/azureml_py38/lib/python3.8/site-packages (from bomwater) (1.1.5)\nRequirement already satisfied: json5 in /anaconda/envs/azureml_py38/lib/python3.8/site-packages (from bomwater) (0.9.10)\nRequirement already satisfied: requests in /anaconda/envs/azureml_py38/lib/python3.8/site-packages (from bomwater) (2.28.1)\nRequirement already satisfied: xmltodict in /anaconda/envs/azureml_py38/lib/python3.8/site-packages (from bomwater) (0.13.0)\nCollecting shapely\n Downloading Shapely-1.8.5.post1-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (2.1 MB)\n\u001b[K |████████████████████████████████| 2.1 MB 7.0 MB/s eta 0:00:01\n\u001b[?25hCollecting geojson\n Downloading geojson-2.5.0-py2.py3-none-any.whl (14 kB)\nCollecting iso8601\n Downloading iso8601-1.1.0-py3-none-any.whl (9.9 kB)\nRequirement already satisfied: numpy>=1.15.4 in /anaconda/envs/azureml_py38/lib/python3.8/site-packages (from pandas->bomwater) (1.21.6)\nRequirement already satisfied: python-dateutil>=2.7.3 in /anaconda/envs/azureml_py38/lib/python3.8/site-packages (from pandas->bomwater) (2.8.2)\nRequirement already satisfied: urllib3<1.27,>=1.21.1 in /anaconda/envs/azureml_py38/lib/python3.8/site-packages (from requests->bomwater) (1.26.12)\nRequirement already satisfied: charset-normalizer<3,>=2 in /anaconda/envs/azureml_py38/lib/python3.8/site-packages (from requests->bomwater) (2.1.1)\nRequirement already satisfied: idna<4,>=2.5 in /anaconda/envs/azureml_py38/lib/python3.8/site-packages (from requests->bomwater) (3.4)\nRequirement already satisfied: certifi>=2017.4.17 in /anaconda/envs/azureml_py38/lib/python3.8/site-packages (from requests->bomwater) (2022.9.24)\nRequirement already satisfied: six>=1.5 in /anaconda/envs/azureml_py38/lib/python3.8/site-packages (from python-dateutil>=2.7.3->pandas->bomwater) (1.16.0)\nInstalling collected packages: shapely, geojson, iso8601, bomwater\nSuccessfully installed bomwater-0.0.8 geojson-2.5.0 iso8601-1.1.0 shapely-1.8.5.post1\nNote: you may need to restart the kernel to use updated packages.\n" + } + ], + "execution_count": 2, + "metadata": { + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "gather": { + "logged": 1669337212145 + } + } + }, + { + "cell_type": "code", + "source": [ + "import mdba_gauge_getter as gg" + ], + "outputs": [], + "execution_count": 1, + "metadata": { + "collapsed": true, + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "gather": { + "logged": 1669960269404 + } + } + }, + { + "cell_type": "code", + "source": [ + "\r\n", + "help(gg.gauge_getter)" + ], + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": "Help on module mdba_gauge_getter.gauge_getter in mdba_gauge_getter:\n\nNAME\n mdba_gauge_getter.gauge_getter\n\nFUNCTIONS\n call_state_api(state:str, indicative_sites:List[str], start_time:datetime.date, end_time:datetime.date, data_source:str, var:str, interval:str, data_type:str) -> Dict[str, Any]\n Sends a web request with a destination based on `state` of the gauge.\n \n Returns a JSON dict object containing web responses, and will fail if the server returns\n either a non HTTP-200 error code, or invalid JSON\n \n extract_data(state:str, data) -> List[List[Any]]\n Collects\n \n fixdate(timestamp)\n \n gauge_pull(gauge_numbers:List[str], start_time_user:datetime.date, end_time_user:datetime.date, var:str='F', interval:str='day', data_type:str='mean', data_source:str='state') -> pandas.core.frame.DataFrame\n Given a list of gauge numbers, sorts the list into state groups, and queries relevant\n HTTP endpoints for data, returning as a Pandas dataframe object.\n \n gauge_pull_bom(gauge_numbers:List[str], start_time_user:datetime.date, end_time_user:datetime.date, var:str='F', interval:str='day', data_type:str='mean') -> pandas.core.frame.DataFrame\n Given a list of gauge numbers, breaks the list into individual gauges, and uses BomWater to get data, \n returning as a Pandas dataframe object in a gauge getter format.\n \n get_states_for_gauge(gauge_number:str) -> Set[str]\n Given a gauge number, returns a set of states which that gauge may belong to.\n \n init() -> None\n Loads gauges from disk. This will dynamically trigger when other libraries require\n gague data.\n \n process_gauge_pull(sitelist:List[str], callstate:str, call_data_source:str, start_time_user:datetime.date, end_time_user:datetime.date, var:str, interval:str, data_type:str) -> List[List[Any]]\n Intermediate function which splits many gauge_pull records into separate web requests\n and provides user feedback on progress\n \n sort_gauges_by_state(gauge_numbers:List[str]) -> Dict[str, List[str]]\n Splits the listed gauges into state-based lists.\n \n split_into_chunks(input_list:List[~T], maxlen:int) -> List[List[~T]]\n Splits a list into many lists of maximum `maxlen` length. Let input_list = [1,2,3,4].\n When:\n \n - `maxlen=1` => `[[1],[2],[3],[4]]`\n - `maxlen=2` => `[[1, 2],[3, 4]]`\n - `maxlen=3` => `[[1, 2, 3], [4]]`\n - `maxlen=4` => `[[1, 2, 3, 4]]`\n - `maxlen=5` => `[[1, 2, 3, 4]]`\n\nDATA\n Any = typing.Any\n MAX_SITES_PER_REQUEST = {'NSW': 5, 'QLD': 5, 'SA': 5, 'VIC': 5}\n Optional = typing.Optional\n STATE_FLOW_VarFrom = {'NSW': Decimal('100.00'), 'QLD': Decimal('100.00...\n STATE_FLOW_VarTo = {'NSW': Decimal('141.00'), 'QLD': Decimal('141.00')...\n STATE_LAKELEVEL_VarFrom = {'NSW': Decimal('130.00'), 'QLD': Decimal('1...\n STATE_LAKELEVEL_VarTo = {'NSW': Decimal('130.00'), 'QLD': Decimal('130...\n STATE_LEVEL_VarFrom = {'NSW': Decimal('100.00'), 'QLD': Decimal('100.0...\n STATE_LEVEL_VarTo = {'NSW': Decimal('100.00'), 'QLD': Decimal('100.00'...\n STATE_URLS = {'NSW': 'realtimedata.waternsw.com.au', 'QLD': 'water-mon...\n T = ~T\n __annotations__ = {'gauges': }\n gauge_data_uri = '/mnt/batch/tasks/shared/LS_root/mounts/clusters/..._...\n gauges = None\n log = \n\nFILE\n /mnt/batch/tasks/shared/LS_root/mounts/clusters/npd-data-azml-vm-bb-ds2/code/Users/Ben.Bradshaw/git/MDBA_Gauge_Getter/mdba_gauge_getter/gauge_getter.py\n\n\n" + } + ], + "execution_count": 2, + "metadata": { + "collapsed": true, + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "gather": { + "logged": 1669960271002 + } + } + }, + { + "cell_type": "code", + "source": [ + "import datetime\r\n", + "from decimal import *\r\n", + "BIGEE_gauges = [\"410130\", \"410005\", \"410040\"]\r\n", + "#gg.sort_gauges_by_state(BIGEE_gauges)\r\n", + "d_BIDGEE = gg.gauge_pull(BIGEE_gauges, start_time_user = datetime.date(2000, 12, 4), end_time_user= datetime.date(2020, 12, 4), var='F' )" + ], + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": "INFO:mdba_gauge_getter.gauge_get:NSW - Request 1 of 1\n" + } + ], + "execution_count": 3, + "metadata": { + "collapsed": true, + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "gather": { + "logged": 1669960288628 + } + } + }, + { + "cell_type": "code", + "source": [ + "d_BIDGEE" + ], + "outputs": [ + { + "output_type": "execute_result", + "execution_count": 4, + "data": { + "text/plain": " DATASOURCEID SITEID SUBJECTID DATETIME VALUE QUALITYCODE\n0 NSW 410130 WATER 2000-12-04 649.745 38\n1 NSW 410130 WATER 2000-12-05 598.909 38\n2 NSW 410130 WATER 2000-12-06 578.624 38\n3 NSW 410130 WATER 2000-12-07 555.826 38\n4 NSW 410130 WATER 2000-12-08 527.558 38\n... ... ... ... ... ... ...\n21913 NSW 410040 WATER 2020-11-30 2122.212 140\n21914 NSW 410040 WATER 2020-12-01 2091.129 140\n21915 NSW 410040 WATER 2020-12-02 2085.267 140\n21916 NSW 410040 WATER 2020-12-03 1866.843 140\n21917 NSW 410040 WATER 2020-12-04 1633.865 140\n\n[21918 rows x 6 columns]", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
DATASOURCEIDSITEIDSUBJECTIDDATETIMEVALUEQUALITYCODE
0NSW410130WATER2000-12-04649.74538
1NSW410130WATER2000-12-05598.90938
2NSW410130WATER2000-12-06578.62438
3NSW410130WATER2000-12-07555.82638
4NSW410130WATER2000-12-08527.55838
.....................
21913NSW410040WATER2020-11-302122.212140
21914NSW410040WATER2020-12-012091.129140
21915NSW410040WATER2020-12-022085.267140
21916NSW410040WATER2020-12-031866.843140
21917NSW410040WATER2020-12-041633.865140
\n

21918 rows × 6 columns

\n
" + }, + "metadata": {} + } + ], + "execution_count": 4, + "metadata": { + "collapsed": true, + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "gather": { + "logged": 1669960292495 + } + } + }, + { + "cell_type": "code", + "source": [ + "Paroo =\t[\"424201A\",\"424202A\",\"424002\"]\r\n", + "Warrego = [\"423203A\",\"423204A\"]\r\n", + "Condamine_Balonne = [\"422006\",\"422394A\",\"422310C\",\"422355A\",\"422353A\",\"422316A\",\"422333A\",\"422308C\",\"422325A\",\"422213A\"]\r\n", + "Moonie = [\"417204A\",\"417201B\",\"417001\"]\r\n", + "Border_Rivers =[\"416002\", \"416202A\",\"416011\",\"416007\",\"416043\",\"416048\"]\r\n", + "Gwydir = [\"418036\", \"418027\", \"418037\", \"418060\", \"418008\",\"418012\",\"418013\",\"418001\",\"418002\",\"418053\",\"416027\",\"418060\",\"418066\",\"416027\",\"416052\",\"418055\"]\r\n", + "Namoi =[\"419021\", \"419049\",\"419020\",\"419022\",\"419001\",\"419012\"]\r\n", + "Macquarie_Castlereagh = [\"420004\", \"421039\",\"421001\",\"421012\"]\r\n", + "Barwon_Darling = [\"425003\",\"416050\",\"422001\",\"422004\",\"422028\",\"425004\"] \r\n", + "Lachlan\t= [\"412057\",\"412002\",\"412004\",\"412036\",\"412011\",\"412057\"]\r\n", + "Murrumbidgee = [\"410001\",\"410008\",\"410004\",\"410005\",\"410021\",\"410078\",\"410136\",\"410040\"]\r\n", + "Ovens = [\"403241\"]\r\n", + "Goulburn_Broken\t= [\"405232\",\"405203\",\"405201\"]\r\n", + "Campaspe = [\"406200\",\"406207\",\"406202\"]\r\n", + "Loddon_Avoca = [\"407203\",\"407210\",\"407224\",\"407205\"]\r\n", + "Wimmera\t= [\"415246\"]\r\n", + "Murray = [\"425010\",\"409215\",\"409204\",\"414200\",\"414203\",\"426200\"]\r\n", + "Lower_Darling = [\"425012\",\"425013\",\"425005\"]\r\n", + "Eastern_Mt_Lofty_Ranges = [\"A4260568\"]\r\n", + "\r\n", + "allguages = Paroo + Warrego + Condamine_Balonne + Moonie + Border_Rivers + Gwydir + Namoi + Macquarie_Castlereagh + Barwon_Darling + Lachlan + Murrumbidgee + Ovens + Goulburn_Broken + Campaspe + Loddon_Avoca + Murray + Wimmera + Lower_Darling + Eastern_Mt_Lofty_Ranges" + ], + "outputs": [], + "execution_count": 5, + "metadata": { + "collapsed": true, + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "gather": { + "logged": 1630622912372 + } + } + }, + { + "cell_type": "code", + "source": [ + "gg.gauge_pull(allguages, start_time_user = datetime.date(2021, 6, 1), end_time_user= datetime.date.today() )" + ], + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": "WARNING:mdba_gauge_getter.gauge_get:Gauge 422353A has many state results: ['QLD', 'QLD']\nINFO:mdba_gauge_getter.gauge_get:NSW - Request 1 of 6\nINFO:mdba_gauge_getter.gauge_get:NSW - Request 2 of 6\nINFO:mdba_gauge_getter.gauge_get:NSW - Request 3 of 6\nINFO:mdba_gauge_getter.gauge_get:NSW - Request 4 of 6\nINFO:mdba_gauge_getter.gauge_get:NSW - Request 5 of 6\nINFO:mdba_gauge_getter.gauge_get:NSW - Request 6 of 6\nINFO:mdba_gauge_getter.gauge_get:VIC - Request 1 of 2\nINFO:mdba_gauge_getter.gauge_get:VIC - Request 2 of 2\nINFO:mdba_gauge_getter.gauge_get:QLD - Request 1 of 2\nINFO:mdba_gauge_getter.gauge_get:QLD - Request 2 of 2\n" + }, + { + "output_type": "execute_result", + "execution_count": 6, + "data": { + "text/plain": " DATASOURCEID SITEID SUBJECTID DATETIME VALUE QUALITYCODE\n0 NSW 424002 WATER 2021-06-01 0.000 140\n1 NSW 424002 WATER 2021-06-02 0.000 140\n2 NSW 424002 WATER 2021-06-03 0.028 140\n3 NSW 424002 WATER 2021-06-04 0.028 140\n4 NSW 424002 WATER 2021-06-05 0.004 140\n... ... ... ... ... ... ...\n8079 QLD 416202A WATER 2021-08-29 52.17 130\n8080 QLD 416202A WATER 2021-08-30 48.77 130\n8081 QLD 416202A WATER 2021-08-31 46.76 130\n8082 QLD 416202A WATER 2021-09-01 44.54 130\n8083 QLD 416202A WATER 2021-09-02 40.12 130\n\n[8084 rows x 6 columns]", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
DATASOURCEIDSITEIDSUBJECTIDDATETIMEVALUEQUALITYCODE
0NSW424002WATER2021-06-010.000140
1NSW424002WATER2021-06-020.000140
2NSW424002WATER2021-06-030.028140
3NSW424002WATER2021-06-040.028140
4NSW424002WATER2021-06-050.004140
.....................
8079QLD416202AWATER2021-08-2952.17130
8080QLD416202AWATER2021-08-3048.77130
8081QLD416202AWATER2021-08-3146.76130
8082QLD416202AWATER2021-09-0144.54130
8083QLD416202AWATER2021-09-0240.12130
\n

8084 rows × 6 columns

\n
" + }, + "metadata": {} + } + ], + "execution_count": 6, + "metadata": { + "collapsed": true, + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "gather": { + "logged": 1630622949093 + } + } + }, + { + "cell_type": "code", + "source": [ + "gg.gauge_pull([\"422034\"], start_time_user = datetime.date(2021, 1, 1), end_time_user= datetime.date(2021, 11, 4), var='L' )" + ], + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": "INFO:mdba_gauge_getter.gauge_get:NSW - Request 1 of 1\n" + }, + { + "output_type": "stream", + "name": "stdout", + "text": "https://realtimedata.waternsw.com.au/cgi/webservice.pl?{\"params\":{\"site_list\":\"422034\",\"start_time\":\"20210101000000\",\"varfrom\":\"100\",\"interval\":\"day\",\"varto\":\"100\",\"datasource\":\"CP\",\"end_time\":\"20211104000000\",\"data_type\":\"mean\",\"multiplier\":\"1\"},\"function\":\"get_ts_traces\",\"version\":\"2\"}\n" + }, + { + "output_type": "execute_result", + "execution_count": 7, + "data": { + "text/plain": " DATASOURCEID SITEID SUBJECTID DATETIME VALUE QUALITYCODE\n0 NSW 422034 WATER 2021-01-01 0.638 130\n1 NSW 422034 WATER 2021-01-02 0.638 130\n2 NSW 422034 WATER 2021-01-03 0.638 130\n3 NSW 422034 WATER 2021-01-04 0.638 130\n4 NSW 422034 WATER 2021-01-05 0.638 130\n.. ... ... ... ... ... ...\n303 NSW 422034 WATER 2021-10-31 0.638 130\n304 NSW 422034 WATER 2021-11-01 0.638 130\n305 NSW 422034 WATER 2021-11-02 0.638 130\n306 NSW 422034 WATER 2021-11-03 0.638 130\n307 NSW 422034 WATER 2021-11-04 0.638 130\n\n[308 rows x 6 columns]", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
DATASOURCEIDSITEIDSUBJECTIDDATETIMEVALUEQUALITYCODE
0NSW422034WATER2021-01-010.638130
1NSW422034WATER2021-01-020.638130
2NSW422034WATER2021-01-030.638130
3NSW422034WATER2021-01-040.638130
4NSW422034WATER2021-01-050.638130
.....................
303NSW422034WATER2021-10-310.638130
304NSW422034WATER2021-11-010.638130
305NSW422034WATER2021-11-020.638130
306NSW422034WATER2021-11-030.638130
307NSW422034WATER2021-11-040.638130
\n

308 rows × 6 columns

\n
" + }, + "metadata": {} + } + ], + "execution_count": 7, + "metadata": { + "collapsed": true, + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "gather": { + "logged": 1637709285233 + } + } + }, + { + "cell_type": "code", + "source": [ + "gg.gauge_pull(\"422034\", start_time_user = datetime.date(2000, 12, 4), end_time_user= datetime.date(2021, 11, 4), var='L' )" + ], + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": "INFO:mdba_gauge_getter.gauge_get:NSW - Request 1 of 1\n" + }, + { + "output_type": "execute_result", + "execution_count": 9, + "data": { + "text/plain": "Empty DataFrame\nColumns: [DATASOURCEID, SITEID, SUBJECTID, DATETIME, VALUE, QUALITYCODE]\nIndex: []", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n
DATASOURCEIDSITEIDSUBJECTIDDATETIMEVALUEQUALITYCODE
\n
" + }, + "metadata": {} + } + ], + "execution_count": 9, + "metadata": { + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "gather": { + "logged": 1637302369565 + } + } + }, + { + "cell_type": "code", + "source": [ + "gg.gauge_getter.sort_gauges_by_state([\"407244\"])" + ], + "outputs": [ + { + "output_type": "execute_result", + "execution_count": 24, + "data": { + "text/plain": "{'NSW': [], 'QLD': [], 'VIC': ['407244'], 'rest': []}" + }, + "metadata": {} + } + ], + "execution_count": 24, + "metadata": { + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "gather": { + "logged": 1637292539125 + } + } + }, + { + "cell_type": "markdown", + "source": [ + "To use bom_water to return data in the same format see below" + ], + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + } + }, + { + "cell_type": "code", + "source": [ + "from typing import Tuple, List, Dict, TypeVar, Set, Optional, Any\r\n", + "import bom_water\r\n", + "import requests\r\n", + "import pandas as pd\r\n", + "\r\n", + "import datetime\r\n", + "\r\n", + "def fixdate(timestamp):\r\n", + " date = timestamp.to_pydatetime()\r\n", + " date = date.date()\r\n", + " #datetime.astimzone('Australia/Sydney',date) #date.tz_localize('Australia/Sydney') #datetime.datetime.strptime(date, '%Y-%m-%dT%H:%M:%S')\r\n", + " return date\r\n", + "\r\n", + "def gauge_pull_bom(gauge_numbers: List[str], start_time_user: datetime.date, end_time_user: datetime.date,\r\n", + " var: str = 'F',\r\n", + " interval: str = 'day', data_type: str = 'mean') -> pd.DataFrame:\r\n", + " '''\r\n", + " Given a list of gauge numbers, breaks the list into individual gauges, and uses BomWater to get data, \r\n", + " returning as a Pandas dataframe object in a gauge getter format.\r\n", + " '''\r\n", + " \r\n", + " bm = bom_water.BomWater()\r\n", + "\r\n", + " if (interval == 'day') & (data_type == 'mean'):\r\n", + " procedure = bm.procedures.Pat4_C_B_1_DailyMean\r\n", + " \r\n", + " if var == \"F\":\r\n", + " prop = bm.properties.Water_Course_Discharge\r\n", + "\r\n", + " t_begin = start_time_user.strftime(\"%Y-%m-%dT%H:%M:%S%z\")\r\n", + " t_end = end_time_user.strftime(\"%Y-%m-%dT%H:%M:%S%z\")\r\n", + "\r\n", + " # t_begin = \"1800-01-01T00:00:00+10\"\r\n", + " # t_end = \"2030-12-31T00:00:00+10\"\r\n", + " collect=[]\r\n", + " for gauge in gauge_numbers:\r\n", + " \r\n", + " response = bm.request(bm.actions.GetObservation, gauge, prop, procedure, t_begin, t_end)\r\n", + "\r\n", + " response_json = bm.xml_to_json(response.text) \r\n", + " ts = bm.parse_get_data(response)\r\n", + " # print(ts)\r\n", + " if ts.empty:\r\n", + " ts = pd.DataFrame(columns=[\"DATASOURCEID\",\"SITEID\",\t\"SUBJECTID\", \"DATETIME\", \"VALUE\", \"QUALITYCODE\"])\r\n", + " collect.append(ts)\r\n", + " else:\r\n", + " # move to format DATASOURCEID\tSITEID\tSUBJECTID\tDATETIME\tVALUE\tQUALITYCODE\r\n", + " ts[\"DATASOURCEID\"] = \"BOM\"\r\n", + " ts[\"SITEID\"] = gauge\r\n", + " ts[\"SUBJECTID\"] = \"WATER\"\r\n", + " ts[\"DATETIME\"] = ts.index.to_pydatetime()\r\n", + " ts[\"DATETIME\"] = pd.to_datetime(ts[\"DATETIME\"])\r\n", + " ts[\"DATETIME\"] = ts[\"DATETIME\"].apply(fixdate)\r\n", + " ts[\"VALUE\"] = ts[\"Value[cumec]\"]\r\n", + " ts[\"QUALITYCODE\"] = ts[\"Quality\"]\r\n", + " ts.reset_index(drop=True, inplace=True)\r\n", + "\r\n", + " collect.append(ts[[\"DATASOURCEID\",\"SITEID\",\t\"SUBJECTID\", \"DATETIME\", \"VALUE\", \"QUALITYCODE\"]])\r\n", + " output = pd.concat(collect)\r\n", + " return output\r\n" + ], + "outputs": [], + "execution_count": 26, + "metadata": { + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "gather": { + "logged": 1669344979378 + } + } + }, + { + "cell_type": "code", + "source": [ + "gg.gauge_getter.gauge_pull_bom([\"410777\", \"412189\"], start_time_user = datetime.date(2022, 11, 1), end_time_user= datetime.date(2022, 11, 25), var='F' )" + ], + "outputs": [ + { + "output_type": "execute_result", + "execution_count": 6, + "data": { + "text/plain": "[['BOM', '410777', 'WATER', datetime.date(2022, 10, 31), nan, -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 1), nan, -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 2), nan, -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 3), nan, -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 4), nan, -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 5), nan, -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 6), nan, -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 7), nan, -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 8), nan, -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 9), nan, -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 10), nan, -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 11), nan, -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 12), nan, -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 13), nan, -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 14), nan, -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 15), nan, -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 16), nan, -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 17), nan, -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 18), nan, -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 10, 31), 44104.176, -1],\n ['BOM',\n '410777',\n 'WATER',\n datetime.date(2022, 11, 1),\n 30493.584000000003,\n -1],\n ['BOM',\n '410777',\n 'WATER',\n datetime.date(2022, 11, 2),\n 21748.176000000003,\n -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 3), 16050.096, -1],\n ['BOM',\n '410777',\n 'WATER',\n datetime.date(2022, 11, 4),\n 13022.985600000002,\n -1],\n ['BOM',\n '410777',\n 'WATER',\n datetime.date(2022, 11, 5),\n 11120.371200000001,\n -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 6), 10379.4048, -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 7), 11848.6368, -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 8), 9429.264000000001, -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 9), 9240.1344, -1],\n ['BOM',\n '410777',\n 'WATER',\n datetime.date(2022, 11, 10),\n 8013.168000000001,\n -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 11), 7179.5808, -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 12), 8835.0912, -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 13), 16183.3248, -1],\n ['BOM',\n '410777',\n 'WATER',\n datetime.date(2022, 11, 14),\n 15531.523200000001,\n -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 15), 11910.7584, -1],\n ['BOM',\n '410777',\n 'WATER',\n datetime.date(2022, 11, 16),\n 9049.190400000001,\n -1],\n ['BOM',\n '410777',\n 'WATER',\n datetime.date(2022, 11, 17),\n 7616.160000000001,\n -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 18), nan, -1],\n ['BOM', '412189', 'WATER', datetime.date(2022, 10, 31), nan, -1],\n ['BOM', '412189', 'WATER', datetime.date(2022, 11, 1), nan, -1],\n ['BOM', '412189', 'WATER', datetime.date(2022, 11, 2), nan, -1],\n ['BOM', '412189', 'WATER', datetime.date(2022, 11, 3), nan, -1],\n ['BOM', '412189', 'WATER', datetime.date(2022, 11, 4), nan, -1],\n ['BOM', '412189', 'WATER', datetime.date(2022, 11, 5), nan, -1],\n ['BOM', '412189', 'WATER', datetime.date(2022, 11, 6), nan, -1],\n ['BOM', '412189', 'WATER', datetime.date(2022, 11, 7), nan, -1],\n ['BOM', '412189', 'WATER', datetime.date(2022, 11, 8), nan, -1],\n ['BOM', '412189', 'WATER', datetime.date(2022, 11, 9), nan, -1],\n ['BOM', '412189', 'WATER', datetime.date(2022, 11, 10), nan, -1],\n ['BOM', '412189', 'WATER', datetime.date(2022, 11, 11), nan, -1],\n ['BOM', '412189', 'WATER', datetime.date(2022, 11, 12), nan, -1],\n ['BOM', '412189', 'WATER', datetime.date(2022, 11, 13), nan, -1],\n ['BOM', '412189', 'WATER', datetime.date(2022, 11, 14), nan, -1],\n ['BOM', '412189', 'WATER', datetime.date(2022, 11, 15), nan, -1],\n ['BOM', '412189', 'WATER', datetime.date(2022, 11, 16), nan, -1],\n ['BOM', '412189', 'WATER', datetime.date(2022, 11, 17), nan, -1],\n ['BOM', '412189', 'WATER', datetime.date(2022, 11, 18), nan, -1],\n ['BOM', '412189', 'WATER', datetime.date(2022, 11, 19), nan, -1],\n ['BOM', '412189', 'WATER', datetime.date(2022, 11, 20), nan, -1],\n ['BOM', '412189', 'WATER', datetime.date(2022, 11, 21), nan, -1],\n ['BOM', '412189', 'WATER', datetime.date(2022, 11, 22), nan, -1],\n ['BOM', '412189', 'WATER', datetime.date(2022, 11, 23), nan, -1],\n ['BOM', '412189', 'WATER', datetime.date(2022, 11, 24), nan, -1]]" + }, + "metadata": {} + } + ], + "execution_count": 6, + "metadata": { + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "gather": { + "logged": 1669960320102 + } + } + }, + { + "cell_type": "code", + "source": [ + "gg.gauge_pull([\"425010\"], start_time_user = datetime.date(2012, 11, 1), end_time_user= datetime.date(2022, 11, 25) ,data_source= \"bom\")" + ], + "outputs": [ + { + "output_type": "execute_result", + "execution_count": 16, + "data": { + "text/plain": " DATASOURCEID SITEID SUBJECTID DATETIME VALUE QUALITYCODE\n0 BOM 425010 WATER 2012-10-31 20550.8448 140\n1 BOM 425010 WATER 2012-11-01 20626.6176 140\n2 BOM 425010 WATER 2012-11-02 20499.0912 140\n3 BOM 425010 WATER 2012-11-03 19806.3360 140\n4 BOM 425010 WATER 2012-11-04 19042.3872 140\n... ... ... ... ... ... ...\n3672 BOM 425010 WATER 2022-11-20 NaN -1\n3673 BOM 425010 WATER 2022-11-21 NaN -1\n3674 BOM 425010 WATER 2022-11-22 NaN -1\n3675 BOM 425010 WATER 2022-11-23 NaN -1\n3676 BOM 425010 WATER 2022-11-24 NaN -1\n\n[3677 rows x 6 columns]", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
DATASOURCEIDSITEIDSUBJECTIDDATETIMEVALUEQUALITYCODE
0BOM425010WATER2012-10-3120550.8448140
1BOM425010WATER2012-11-0120626.6176140
2BOM425010WATER2012-11-0220499.0912140
3BOM425010WATER2012-11-0319806.3360140
4BOM425010WATER2012-11-0419042.3872140
.....................
3672BOM425010WATER2022-11-20NaN-1
3673BOM425010WATER2022-11-21NaN-1
3674BOM425010WATER2022-11-22NaN-1
3675BOM425010WATER2022-11-23NaN-1
3676BOM425010WATER2022-11-24NaN-1
\n

3677 rows × 6 columns

\n
" + }, + "metadata": {} + } + ], + "execution_count": 16, + "metadata": { + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "gather": { + "logged": 1669960600928 + } + } + }, + { + "cell_type": "code", + "source": [], + "outputs": [], + "execution_count": null, + "metadata": { + "jupyter": { + "source_hidden": false, + "outputs_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + } + } + } + ], + "metadata": { + "kernelspec": { + "name": "python3", + "language": "python", + "display_name": "Python 3" + }, + "language_info": { + "name": "python", + "version": "3.6.9", + "mimetype": "text/x-python", + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "pygments_lexer": "ipython3", + "nbconvert_exporter": "python", + "file_extension": ".py" + }, + "kernel_info": { + "name": "python3" + }, + "nteract": { + "version": "nteract-front-end@1.0.0" + }, + "microsoft": { + "host": { + "AzureML": { + "notebookHasBeenCompleted": true + } + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} \ No newline at end of file diff --git a/CITATION.cff b/CITATION.cff index 052cf42..4579655 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -1,4 +1,4 @@ -cff-version: 0.4.5 +cff-version: 0.4.6 message: "If you use this software, please cite it as below." authors: - family-names: Bradshaw @@ -15,6 +15,6 @@ authors: given-names: Ahsanul title: "MDBA_Gauge_Getter" -version: 0.4.5 +version: 0.4.6 doi: 10.5281/zenodo.7435693 date-released: 2022-12-14 diff --git a/Gauge_getter_example.ipynb b/Gauge_getter_example.ipynb index 14cfa4c..bd1f360 100644 --- a/Gauge_getter_example.ipynb +++ b/Gauge_getter_example.ipynb @@ -1,551 +1,1999 @@ { - "cells": [ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": false, + "gather": { + "logged": 1669337212145 + }, + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [ { - "cell_type": "code", - "source": [ - "pip install bomwater" - ], - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": "Collecting bomwater\n Downloading bomwater-0.0.8-py3-none-any.whl (10 kB)\nRequirement already satisfied: pytz in /anaconda/envs/azureml_py38/lib/python3.8/site-packages (from bomwater) (2022.5)\nRequirement already satisfied: pandas in /anaconda/envs/azureml_py38/lib/python3.8/site-packages (from bomwater) (1.1.5)\nRequirement already satisfied: json5 in /anaconda/envs/azureml_py38/lib/python3.8/site-packages (from bomwater) (0.9.10)\nRequirement already satisfied: requests in /anaconda/envs/azureml_py38/lib/python3.8/site-packages (from bomwater) (2.28.1)\nRequirement already satisfied: xmltodict in /anaconda/envs/azureml_py38/lib/python3.8/site-packages (from bomwater) (0.13.0)\nCollecting shapely\n Downloading Shapely-1.8.5.post1-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl (2.1 MB)\n\u001b[K |████████████████████████████████| 2.1 MB 7.0 MB/s eta 0:00:01\n\u001b[?25hCollecting geojson\n Downloading geojson-2.5.0-py2.py3-none-any.whl (14 kB)\nCollecting iso8601\n Downloading iso8601-1.1.0-py3-none-any.whl (9.9 kB)\nRequirement already satisfied: numpy>=1.15.4 in /anaconda/envs/azureml_py38/lib/python3.8/site-packages (from pandas->bomwater) (1.21.6)\nRequirement already satisfied: python-dateutil>=2.7.3 in /anaconda/envs/azureml_py38/lib/python3.8/site-packages (from pandas->bomwater) (2.8.2)\nRequirement already satisfied: urllib3<1.27,>=1.21.1 in /anaconda/envs/azureml_py38/lib/python3.8/site-packages (from requests->bomwater) (1.26.12)\nRequirement already satisfied: charset-normalizer<3,>=2 in /anaconda/envs/azureml_py38/lib/python3.8/site-packages (from requests->bomwater) (2.1.1)\nRequirement already satisfied: idna<4,>=2.5 in /anaconda/envs/azureml_py38/lib/python3.8/site-packages (from requests->bomwater) (3.4)\nRequirement already satisfied: certifi>=2017.4.17 in /anaconda/envs/azureml_py38/lib/python3.8/site-packages (from requests->bomwater) (2022.9.24)\nRequirement already satisfied: six>=1.5 in /anaconda/envs/azureml_py38/lib/python3.8/site-packages (from python-dateutil>=2.7.3->pandas->bomwater) (1.16.0)\nInstalling collected packages: shapely, geojson, iso8601, bomwater\nSuccessfully installed bomwater-0.0.8 geojson-2.5.0 iso8601-1.1.0 shapely-1.8.5.post1\nNote: you may need to restart the kernel to use updated packages.\n" - } - ], - "execution_count": 2, - "metadata": { - "jupyter": { - "source_hidden": false, - "outputs_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - }, - "gather": { - "logged": 1669337212145 - } - } + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: bomwater in /home/azureuser/.local/lib/python3.8/site-packages (0.0.8)\n", + "Requirement already satisfied: iso8601 in /home/azureuser/.local/lib/python3.8/site-packages (from bomwater) (1.1.0)\n", + "Requirement already satisfied: pytz in /home/azureuser/.local/lib/python3.8/site-packages (from bomwater) (2022.6)\n", + "Requirement already satisfied: xmltodict in /home/azureuser/.local/lib/python3.8/site-packages (from bomwater) (0.13.0)\n", + "Requirement already satisfied: geojson in /home/azureuser/.local/lib/python3.8/site-packages (from bomwater) (2.5.0)\n", + "Requirement already satisfied: json5 in /home/azureuser/.local/lib/python3.8/site-packages (from bomwater) (0.9.10)\n", + "Requirement already satisfied: requests in /anaconda/envs/azureml_py38/lib/python3.8/site-packages (from bomwater) (2.28.1)\n", + "Requirement already satisfied: shapely in /home/azureuser/.local/lib/python3.8/site-packages (from bomwater) (1.8.5.post1)\n", + "Requirement already satisfied: pandas in /home/azureuser/.local/lib/python3.8/site-packages (from bomwater) (1.5.2)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /anaconda/envs/azureml_py38/lib/python3.8/site-packages (from requests->bomwater) (2022.9.24)\n", + "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /home/azureuser/.local/lib/python3.8/site-packages (from requests->bomwater) (1.26.13)\n", + "Requirement already satisfied: charset-normalizer<3,>=2 in /anaconda/envs/azureml_py38/lib/python3.8/site-packages (from requests->bomwater) (2.1.1)\n", + "Requirement already satisfied: idna<4,>=2.5 in /anaconda/envs/azureml_py38/lib/python3.8/site-packages (from requests->bomwater) (3.4)\n", + "Requirement already satisfied: python-dateutil>=2.8.1 in /home/azureuser/.local/lib/python3.8/site-packages (from pandas->bomwater) (2.8.2)\n", + "Requirement already satisfied: numpy>=1.20.3; python_version < \"3.10\" in /home/azureuser/.local/lib/python3.8/site-packages (from pandas->bomwater) (1.23.5)\n", + "Requirement already satisfied: six>=1.5 in /anaconda/envs/azureml_py38/lib/python3.8/site-packages (from python-dateutil>=2.8.1->pandas->bomwater) (1.16.0)\n", + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "source": [ + "pip install bomwater" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "gather": { + "logged": 1669960269404 + }, + "nteract": { + "transient": { + "deleting": false + } }, + "tags": [] + }, + "outputs": [], + "source": [ + "import mdba_gauge_getter as gg" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "gather": { + "logged": 1669960271002 + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "tags": [] + }, + "outputs": [], + "source": [ + "# help(gg.gauge_getter)" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "gather": { + "logged": 1669960288628 + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "tags": [] + }, + "outputs": [ { - "cell_type": "code", - "source": [ - "import mdba_gauge_getter as gg" - ], - "outputs": [], - "execution_count": 1, - "metadata": { - "collapsed": true, - "jupyter": { - "source_hidden": false, - "outputs_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - }, - "gather": { - "logged": 1669960269404 - } - } + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:mdba_gauge_getter.gauge_get:NSW - Request 1 of 1\n" + ] + } + ], + "source": [ + "import datetime\n", + "from decimal import *\n", + "BIGEE_gauges = [\"410130\", \"410005\", \"410040\"]\n", + "#gg.sort_gauges_by_state(BIGEE_gauges)\n", + "d_BIDGEE = gg.gauge_pull(BIGEE_gauges, start_time_user = datetime.date(2000, 12, 4), end_time_user= datetime.date(2020, 12, 4), var='L' )" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": { + "gather": { + "logged": 1669960292495 + }, + "nteract": { + "transient": { + "deleting": false + } }, + "tags": [] + }, + "outputs": [ { - "cell_type": "code", - "source": [ - "\r\n", - "help(gg.gauge_getter)" - ], - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": "Help on module mdba_gauge_getter.gauge_getter in mdba_gauge_getter:\n\nNAME\n mdba_gauge_getter.gauge_getter\n\nFUNCTIONS\n call_state_api(state:str, indicative_sites:List[str], start_time:datetime.date, end_time:datetime.date, data_source:str, var:str, interval:str, data_type:str) -> Dict[str, Any]\n Sends a web request with a destination based on `state` of the gauge.\n \n Returns a JSON dict object containing web responses, and will fail if the server returns\n either a non HTTP-200 error code, or invalid JSON\n \n extract_data(state:str, data) -> List[List[Any]]\n Collects\n \n fixdate(timestamp)\n \n gauge_pull(gauge_numbers:List[str], start_time_user:datetime.date, end_time_user:datetime.date, var:str='F', interval:str='day', data_type:str='mean', data_source:str='state') -> pandas.core.frame.DataFrame\n Given a list of gauge numbers, sorts the list into state groups, and queries relevant\n HTTP endpoints for data, returning as a Pandas dataframe object.\n \n gauge_pull_bom(gauge_numbers:List[str], start_time_user:datetime.date, end_time_user:datetime.date, var:str='F', interval:str='day', data_type:str='mean') -> pandas.core.frame.DataFrame\n Given a list of gauge numbers, breaks the list into individual gauges, and uses BomWater to get data, \n returning as a Pandas dataframe object in a gauge getter format.\n \n get_states_for_gauge(gauge_number:str) -> Set[str]\n Given a gauge number, returns a set of states which that gauge may belong to.\n \n init() -> None\n Loads gauges from disk. This will dynamically trigger when other libraries require\n gague data.\n \n process_gauge_pull(sitelist:List[str], callstate:str, call_data_source:str, start_time_user:datetime.date, end_time_user:datetime.date, var:str, interval:str, data_type:str) -> List[List[Any]]\n Intermediate function which splits many gauge_pull records into separate web requests\n and provides user feedback on progress\n \n sort_gauges_by_state(gauge_numbers:List[str]) -> Dict[str, List[str]]\n Splits the listed gauges into state-based lists.\n \n split_into_chunks(input_list:List[~T], maxlen:int) -> List[List[~T]]\n Splits a list into many lists of maximum `maxlen` length. Let input_list = [1,2,3,4].\n When:\n \n - `maxlen=1` => `[[1],[2],[3],[4]]`\n - `maxlen=2` => `[[1, 2],[3, 4]]`\n - `maxlen=3` => `[[1, 2, 3], [4]]`\n - `maxlen=4` => `[[1, 2, 3, 4]]`\n - `maxlen=5` => `[[1, 2, 3, 4]]`\n\nDATA\n Any = typing.Any\n MAX_SITES_PER_REQUEST = {'NSW': 5, 'QLD': 5, 'SA': 5, 'VIC': 5}\n Optional = typing.Optional\n STATE_FLOW_VarFrom = {'NSW': Decimal('100.00'), 'QLD': Decimal('100.00...\n STATE_FLOW_VarTo = {'NSW': Decimal('141.00'), 'QLD': Decimal('141.00')...\n STATE_LAKELEVEL_VarFrom = {'NSW': Decimal('130.00'), 'QLD': Decimal('1...\n STATE_LAKELEVEL_VarTo = {'NSW': Decimal('130.00'), 'QLD': Decimal('130...\n STATE_LEVEL_VarFrom = {'NSW': Decimal('100.00'), 'QLD': Decimal('100.0...\n STATE_LEVEL_VarTo = {'NSW': Decimal('100.00'), 'QLD': Decimal('100.00'...\n STATE_URLS = {'NSW': 'realtimedata.waternsw.com.au', 'QLD': 'water-mon...\n T = ~T\n __annotations__ = {'gauges': }\n gauge_data_uri = '/mnt/batch/tasks/shared/LS_root/mounts/clusters/..._...\n gauges = None\n log = \n\nFILE\n /mnt/batch/tasks/shared/LS_root/mounts/clusters/npd-data-azml-vm-bb-ds2/code/Users/Ben.Bradshaw/git/MDBA_Gauge_Getter/mdba_gauge_getter/gauge_getter.py\n\n\n" - } + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
DATASOURCEIDSITEIDSUBJECTIDDATETIMEVALUEQUALITYCODE
0NSW410130WATER2000-12-041.02132
1NSW410130WATER2000-12-050.96632
2NSW410130WATER2000-12-060.94432
3NSW410130WATER2000-12-070.91832
4NSW410130WATER2000-12-080.88332
.....................
21913NSW410040WATER2020-11-301.7213
21914NSW410040WATER2020-12-011.7053
21915NSW410040WATER2020-12-021.7023
21916NSW410040WATER2020-12-031.5873
21917NSW410040WATER2020-12-041.4563
\n", + "

21918 rows × 6 columns

\n", + "
" ], - "execution_count": 2, - "metadata": { - "collapsed": true, - "jupyter": { - "source_hidden": false, - "outputs_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - }, - "gather": { - "logged": 1669960271002 - } - } + "text/plain": [ + " DATASOURCEID SITEID SUBJECTID DATETIME VALUE QUALITYCODE\n", + "0 NSW 410130 WATER 2000-12-04 1.021 32\n", + "1 NSW 410130 WATER 2000-12-05 0.966 32\n", + "2 NSW 410130 WATER 2000-12-06 0.944 32\n", + "3 NSW 410130 WATER 2000-12-07 0.918 32\n", + "4 NSW 410130 WATER 2000-12-08 0.883 32\n", + "... ... ... ... ... ... ...\n", + "21913 NSW 410040 WATER 2020-11-30 1.721 3\n", + "21914 NSW 410040 WATER 2020-12-01 1.705 3\n", + "21915 NSW 410040 WATER 2020-12-02 1.702 3\n", + "21916 NSW 410040 WATER 2020-12-03 1.587 3\n", + "21917 NSW 410040 WATER 2020-12-04 1.456 3\n", + "\n", + "[21918 rows x 6 columns]" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "d_BIDGEE" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "gather": { + "logged": 1630622912372 + }, + "nteract": { + "transient": { + "deleting": false + } + }, + "tags": [] + }, + "outputs": [], + "source": [ + "Paroo =\t[\"424201A\",\"424202A\",\"424002\"]\n", + "Warrego = [\"423203A\",\"423204A\"]\n", + "Condamine_Balonne = [\"422006\",\"422394A\",\"422310C\",\"422355A\",\"422353A\",\"422316A\",\"422333A\",\"422308C\",\"422325A\",\"422213A\"]\n", + "Moonie = [\"417204A\",\"417201B\",\"417001\"]\n", + "Border_Rivers =[\"416002\", \"416202A\",\"416011\",\"416007\",\"416043\",\"416048\"]\n", + "Gwydir = [\"418036\", \"418027\", \"418037\", \"418060\", \"418008\",\"418012\",\"418013\",\"418001\",\"418002\",\"418053\",\"416027\",\"418060\",\"418066\",\"416027\",\"416052\",\"418055\"]\n", + "Namoi =[\"419021\", \"419049\",\"419020\",\"419022\",\"419001\",\"419012\"]\n", + "Macquarie_Castlereagh = [\"420004\", \"421039\",\"421001\",\"421012\"]\n", + "Barwon_Darling = [\"425003\",\"416050\",\"422001\",\"422004\",\"422028\",\"425004\"] \n", + "Lachlan\t= [\"412057\",\"412002\",\"412004\",\"412036\",\"412011\",\"412057\"]\n", + "Murrumbidgee = [\"410001\",\"410008\",\"410004\",\"410005\",\"410021\",\"410078\",\"410136\",\"410040\"]\n", + "Ovens = [\"403241\"]\n", + "Goulburn_Broken\t= [\"405232\",\"405203\",\"405201\"]\n", + "Campaspe = [\"406200\",\"406207\",\"406202\"]\n", + "Loddon_Avoca = [\"407203\",\"407210\",\"407224\",\"407205\"]\n", + "Wimmera\t= [\"415246\"]\n", + "Murray = [\"425010\",\"409215\",\"409204\",\"414200\",\"414203\",\"426200\"]\n", + "Lower_Darling = [\"425012\",\"425013\",\"425005\"]\n", + "Eastern_Mt_Lofty_Ranges = [\"A4260568\"]\n", + "\n", + "allguages = Paroo + Warrego + Condamine_Balonne + Moonie + Border_Rivers + Gwydir + Namoi + Macquarie_Castlereagh + Barwon_Darling + Lachlan + Murrumbidgee + Ovens + Goulburn_Broken + Campaspe + Loddon_Avoca + Murray + Wimmera + Lower_Darling + Eastern_Mt_Lofty_Ranges" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "gather": { + "logged": 1630622949093 + }, + "nteract": { + "transient": { + "deleting": false + } }, + "tags": [] + }, + "outputs": [ { - "cell_type": "code", - "source": [ - "import datetime\r\n", - "from decimal import *\r\n", - "BIGEE_gauges = [\"410130\", \"410005\", \"410040\"]\r\n", - "#gg.sort_gauges_by_state(BIGEE_gauges)\r\n", - "d_BIDGEE = gg.gauge_pull(BIGEE_gauges, start_time_user = datetime.date(2000, 12, 4), end_time_user= datetime.date(2020, 12, 4), var='F' )" - ], - "outputs": [ - { - "output_type": "stream", - "name": "stderr", - "text": "INFO:mdba_gauge_getter.gauge_get:NSW - Request 1 of 1\n" - } - ], - "execution_count": 3, - "metadata": { - "collapsed": true, - "jupyter": { - "source_hidden": false, - "outputs_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - }, - "gather": { - "logged": 1669960288628 - } - } + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:mdba_gauge_getter.gauge_get:Gauge 422353A has 2 state results: ['QLD', 'QLD']\n", + "INFO:mdba_gauge_getter.gauge_get:NSW - Request 1 of 11\n", + "INFO:mdba_gauge_getter.gauge_get:NSW - Request 2 of 11\n", + "INFO:mdba_gauge_getter.gauge_get:NSW - Request 3 of 11\n", + "INFO:mdba_gauge_getter.gauge_get:NSW - Request 4 of 11\n", + "INFO:mdba_gauge_getter.gauge_get:NSW - Request 5 of 11\n", + "INFO:mdba_gauge_getter.gauge_get:NSW - Request 6 of 11\n", + "INFO:mdba_gauge_getter.gauge_get:NSW - Request 7 of 11\n", + "INFO:mdba_gauge_getter.gauge_get:NSW - Request 8 of 11\n", + "INFO:mdba_gauge_getter.gauge_get:NSW - Request 9 of 11\n", + "INFO:mdba_gauge_getter.gauge_get:NSW - Request 10 of 11\n", + "INFO:mdba_gauge_getter.gauge_get:NSW - Request 11 of 11\n", + "INFO:mdba_gauge_getter.gauge_get:VIC - Request 1 of 4\n", + "INFO:mdba_gauge_getter.gauge_get:VIC - Request 2 of 4\n", + "INFO:mdba_gauge_getter.gauge_get:VIC - Request 3 of 4\n", + "INFO:mdba_gauge_getter.gauge_get:VIC - Request 4 of 4\n", + "INFO:mdba_gauge_getter.gauge_get:QLD - Request 1 of 4\n", + "INFO:mdba_gauge_getter.gauge_get:QLD - Request 2 of 4\n", + "INFO:mdba_gauge_getter.gauge_get:QLD - Request 3 of 4\n", + "INFO:mdba_gauge_getter.gauge_get:QLD - Request 4 of 4\n" + ] }, { - "cell_type": "code", - "source": [ - "d_BIDGEE" - ], - "outputs": [ - { - "output_type": "execute_result", - "execution_count": 4, - "data": { - "text/plain": " DATASOURCEID SITEID SUBJECTID DATETIME VALUE QUALITYCODE\n0 NSW 410130 WATER 2000-12-04 649.745 38\n1 NSW 410130 WATER 2000-12-05 598.909 38\n2 NSW 410130 WATER 2000-12-06 578.624 38\n3 NSW 410130 WATER 2000-12-07 555.826 38\n4 NSW 410130 WATER 2000-12-08 527.558 38\n... ... ... ... ... ... ...\n21913 NSW 410040 WATER 2020-11-30 2122.212 140\n21914 NSW 410040 WATER 2020-12-01 2091.129 140\n21915 NSW 410040 WATER 2020-12-02 2085.267 140\n21916 NSW 410040 WATER 2020-12-03 1866.843 140\n21917 NSW 410040 WATER 2020-12-04 1633.865 140\n\n[21918 rows x 6 columns]", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
DATASOURCEIDSITEIDSUBJECTIDDATETIMEVALUEQUALITYCODE
0NSW410130WATER2000-12-04649.74538
1NSW410130WATER2000-12-05598.90938
2NSW410130WATER2000-12-06578.62438
3NSW410130WATER2000-12-07555.82638
4NSW410130WATER2000-12-08527.55838
.....................
21913NSW410040WATER2020-11-302122.212140
21914NSW410040WATER2020-12-012091.129140
21915NSW410040WATER2020-12-022085.267140
21916NSW410040WATER2020-12-031866.843140
21917NSW410040WATER2020-12-041633.865140
\n

21918 rows × 6 columns

\n
" - }, - "metadata": {} - } - ], - "execution_count": 4, - "metadata": { - "collapsed": true, - "jupyter": { - "source_hidden": false, - "outputs_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - }, - "gather": { - "logged": 1669960292495 - } - } + "name": "stdout", + "output_type": "stream", + "text": [ + "Empty DataFrame\n", + "Columns: [Value[cumec], Quality, Interpolation]\n", + "Index: []\n" + ] }, { - "cell_type": "code", - "source": [ - "Paroo =\t[\"424201A\",\"424202A\",\"424002\"]\r\n", - "Warrego = [\"423203A\",\"423204A\"]\r\n", - "Condamine_Balonne = [\"422006\",\"422394A\",\"422310C\",\"422355A\",\"422353A\",\"422316A\",\"422333A\",\"422308C\",\"422325A\",\"422213A\"]\r\n", - "Moonie = [\"417204A\",\"417201B\",\"417001\"]\r\n", - "Border_Rivers =[\"416002\", \"416202A\",\"416011\",\"416007\",\"416043\",\"416048\"]\r\n", - "Gwydir = [\"418036\", \"418027\", \"418037\", \"418060\", \"418008\",\"418012\",\"418013\",\"418001\",\"418002\",\"418053\",\"416027\",\"418060\",\"418066\",\"416027\",\"416052\",\"418055\"]\r\n", - "Namoi =[\"419021\", \"419049\",\"419020\",\"419022\",\"419001\",\"419012\"]\r\n", - "Macquarie_Castlereagh = [\"420004\", \"421039\",\"421001\",\"421012\"]\r\n", - "Barwon_Darling = [\"425003\",\"416050\",\"422001\",\"422004\",\"422028\",\"425004\"] \r\n", - "Lachlan\t= [\"412057\",\"412002\",\"412004\",\"412036\",\"412011\",\"412057\"]\r\n", - "Murrumbidgee = [\"410001\",\"410008\",\"410004\",\"410005\",\"410021\",\"410078\",\"410136\",\"410040\"]\r\n", - "Ovens = [\"403241\"]\r\n", - "Goulburn_Broken\t= [\"405232\",\"405203\",\"405201\"]\r\n", - "Campaspe = [\"406200\",\"406207\",\"406202\"]\r\n", - "Loddon_Avoca = [\"407203\",\"407210\",\"407224\",\"407205\"]\r\n", - "Wimmera\t= [\"415246\"]\r\n", - "Murray = [\"425010\",\"409215\",\"409204\",\"414200\",\"414203\",\"426200\"]\r\n", - "Lower_Darling = [\"425012\",\"425013\",\"425005\"]\r\n", - "Eastern_Mt_Lofty_Ranges = [\"A4260568\"]\r\n", - "\r\n", - "allguages = Paroo + Warrego + Condamine_Balonne + Moonie + Border_Rivers + Gwydir + Namoi + Macquarie_Castlereagh + Barwon_Darling + Lachlan + Murrumbidgee + Ovens + Goulburn_Broken + Campaspe + Loddon_Avoca + Murray + Wimmera + Lower_Darling + Eastern_Mt_Lofty_Ranges" + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
DATASOURCEIDSITEIDSUBJECTIDDATETIMEVALUEQUALITYCODE
0NSW424002WATER2021-06-010.000140
1NSW424002WATER2021-06-020.000140
2NSW424002WATER2021-06-030.027140
3NSW424002WATER2021-06-040.027140
4NSW424002WATER2021-06-050.003140
.....................
48843QLD416202AWATER2022-12-160.00130
48844QLD416202AWATER2022-12-170.00130
48845QLD416202AWATER2022-12-180.00130
48846QLD416202AWATER2022-12-190.00130
48847QLD416202AWATER2022-12-200.00255
\n", + "

48848 rows × 6 columns

\n", + "
" ], - "outputs": [], - "execution_count": 5, - "metadata": { - "collapsed": true, - "jupyter": { - "source_hidden": false, - "outputs_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - }, - "gather": { - "logged": 1630622912372 - } - } + "text/plain": [ + " DATASOURCEID SITEID SUBJECTID DATETIME VALUE QUALITYCODE\n", + "0 NSW 424002 WATER 2021-06-01 0.000 140\n", + "1 NSW 424002 WATER 2021-06-02 0.000 140\n", + "2 NSW 424002 WATER 2021-06-03 0.027 140\n", + "3 NSW 424002 WATER 2021-06-04 0.027 140\n", + "4 NSW 424002 WATER 2021-06-05 0.003 140\n", + "... ... ... ... ... ... ...\n", + "48843 QLD 416202A WATER 2022-12-16 0.00 130\n", + "48844 QLD 416202A WATER 2022-12-17 0.00 130\n", + "48845 QLD 416202A WATER 2022-12-18 0.00 130\n", + "48846 QLD 416202A WATER 2022-12-19 0.00 130\n", + "48847 QLD 416202A WATER 2022-12-20 0.00 255\n", + "\n", + "[48848 rows x 6 columns]" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "gg.gauge_pull(allguages, start_time_user = datetime.date(2021, 6, 1), end_time_user= datetime.date.today() )" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "gather": { + "logged": 1637709285233 + }, + "nteract": { + "transient": { + "deleting": false + } }, + "tags": [] + }, + "outputs": [ { - "cell_type": "code", - "source": [ - "gg.gauge_pull(allguages, start_time_user = datetime.date(2021, 6, 1), end_time_user= datetime.date.today() )" - ], - "outputs": [ - { - "output_type": "stream", - "name": "stderr", - "text": "WARNING:mdba_gauge_getter.gauge_get:Gauge 422353A has many state results: ['QLD', 'QLD']\nINFO:mdba_gauge_getter.gauge_get:NSW - Request 1 of 6\nINFO:mdba_gauge_getter.gauge_get:NSW - Request 2 of 6\nINFO:mdba_gauge_getter.gauge_get:NSW - Request 3 of 6\nINFO:mdba_gauge_getter.gauge_get:NSW - Request 4 of 6\nINFO:mdba_gauge_getter.gauge_get:NSW - Request 5 of 6\nINFO:mdba_gauge_getter.gauge_get:NSW - Request 6 of 6\nINFO:mdba_gauge_getter.gauge_get:VIC - Request 1 of 2\nINFO:mdba_gauge_getter.gauge_get:VIC - Request 2 of 2\nINFO:mdba_gauge_getter.gauge_get:QLD - Request 1 of 2\nINFO:mdba_gauge_getter.gauge_get:QLD - Request 2 of 2\n" - }, - { - "output_type": "execute_result", - "execution_count": 6, - "data": { - "text/plain": " DATASOURCEID SITEID SUBJECTID DATETIME VALUE QUALITYCODE\n0 NSW 424002 WATER 2021-06-01 0.000 140\n1 NSW 424002 WATER 2021-06-02 0.000 140\n2 NSW 424002 WATER 2021-06-03 0.028 140\n3 NSW 424002 WATER 2021-06-04 0.028 140\n4 NSW 424002 WATER 2021-06-05 0.004 140\n... ... ... ... ... ... ...\n8079 QLD 416202A WATER 2021-08-29 52.17 130\n8080 QLD 416202A WATER 2021-08-30 48.77 130\n8081 QLD 416202A WATER 2021-08-31 46.76 130\n8082 QLD 416202A WATER 2021-09-01 44.54 130\n8083 QLD 416202A WATER 2021-09-02 40.12 130\n\n[8084 rows x 6 columns]", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
DATASOURCEIDSITEIDSUBJECTIDDATETIMEVALUEQUALITYCODE
0NSW424002WATER2021-06-010.000140
1NSW424002WATER2021-06-020.000140
2NSW424002WATER2021-06-030.028140
3NSW424002WATER2021-06-040.028140
4NSW424002WATER2021-06-050.004140
.....................
8079QLD416202AWATER2021-08-2952.17130
8080QLD416202AWATER2021-08-3048.77130
8081QLD416202AWATER2021-08-3146.76130
8082QLD416202AWATER2021-09-0144.54130
8083QLD416202AWATER2021-09-0240.12130
\n

8084 rows × 6 columns

\n
" - }, - "metadata": {} - } - ], - "execution_count": 6, - "metadata": { - "collapsed": true, - "jupyter": { - "source_hidden": false, - "outputs_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - }, - "gather": { - "logged": 1630622949093 - } - } + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:mdba_gauge_getter.gauge_get:NSW - Request 1 of 1\n" + ] }, { - "cell_type": "code", - "source": [ - "gg.gauge_pull([\"422034\"], start_time_user = datetime.date(2021, 1, 1), end_time_user= datetime.date(2021, 11, 4), var='L' )" - ], - "outputs": [ - { - "output_type": "stream", - "name": "stderr", - "text": "INFO:mdba_gauge_getter.gauge_get:NSW - Request 1 of 1\n" - }, - { - "output_type": "stream", - "name": "stdout", - "text": "https://realtimedata.waternsw.com.au/cgi/webservice.pl?{\"params\":{\"site_list\":\"422034\",\"start_time\":\"20210101000000\",\"varfrom\":\"100\",\"interval\":\"day\",\"varto\":\"100\",\"datasource\":\"CP\",\"end_time\":\"20211104000000\",\"data_type\":\"mean\",\"multiplier\":\"1\"},\"function\":\"get_ts_traces\",\"version\":\"2\"}\n" - }, - { - "output_type": "execute_result", - "execution_count": 7, - "data": { - "text/plain": " DATASOURCEID SITEID SUBJECTID DATETIME VALUE QUALITYCODE\n0 NSW 422034 WATER 2021-01-01 0.638 130\n1 NSW 422034 WATER 2021-01-02 0.638 130\n2 NSW 422034 WATER 2021-01-03 0.638 130\n3 NSW 422034 WATER 2021-01-04 0.638 130\n4 NSW 422034 WATER 2021-01-05 0.638 130\n.. ... ... ... ... ... ...\n303 NSW 422034 WATER 2021-10-31 0.638 130\n304 NSW 422034 WATER 2021-11-01 0.638 130\n305 NSW 422034 WATER 2021-11-02 0.638 130\n306 NSW 422034 WATER 2021-11-03 0.638 130\n307 NSW 422034 WATER 2021-11-04 0.638 130\n\n[308 rows x 6 columns]", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
DATASOURCEIDSITEIDSUBJECTIDDATETIMEVALUEQUALITYCODE
0NSW422034WATER2021-01-010.638130
1NSW422034WATER2021-01-020.638130
2NSW422034WATER2021-01-030.638130
3NSW422034WATER2021-01-040.638130
4NSW422034WATER2021-01-050.638130
.....................
303NSW422034WATER2021-10-310.638130
304NSW422034WATER2021-11-010.638130
305NSW422034WATER2021-11-020.638130
306NSW422034WATER2021-11-030.638130
307NSW422034WATER2021-11-040.638130
\n

308 rows × 6 columns

\n
" - }, - "metadata": {} - } - ], - "execution_count": 7, - "metadata": { - "collapsed": true, - "jupyter": { - "source_hidden": false, - "outputs_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - }, - "gather": { - "logged": 1637709285233 - } - } + "name": "stdout", + "output_type": "stream", + "text": [ + "https://realtimedata.waternsw.com.au/cgi/webservice.pl?{\"params\":{\"site_list\":\"422034\",\"start_time\":\"20210101000000\",\"varfrom\":\"100\",\"interval\":\"day\",\"varto\":\"100\",\"datasource\":\"CP\",\"end_time\":\"20211104000000\",\"data_type\":\"mean\",\"multiplier\":\"1\"},\"function\":\"get_ts_traces\",\"version\":\"2\"}\n" + ] }, { - "cell_type": "code", - "source": [ - "gg.gauge_pull(\"422034\", start_time_user = datetime.date(2000, 12, 4), end_time_user= datetime.date(2021, 11, 4), var='L' )" - ], - "outputs": [ - { - "output_type": "stream", - "name": "stderr", - "text": "INFO:mdba_gauge_getter.gauge_get:NSW - Request 1 of 1\n" - }, - { - "output_type": "execute_result", - "execution_count": 9, - "data": { - "text/plain": "Empty DataFrame\nColumns: [DATASOURCEID, SITEID, SUBJECTID, DATETIME, VALUE, QUALITYCODE]\nIndex: []", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n
DATASOURCEIDSITEIDSUBJECTIDDATETIMEVALUEQUALITYCODE
\n
" - }, - "metadata": {} - } + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
DATASOURCEIDSITEIDSUBJECTIDDATETIMEVALUEQUALITYCODE
0NSW422034WATER2021-01-010.638130
1NSW422034WATER2021-01-020.638130
2NSW422034WATER2021-01-030.638130
3NSW422034WATER2021-01-040.638130
4NSW422034WATER2021-01-050.638130
.....................
303NSW422034WATER2021-10-310.638130
304NSW422034WATER2021-11-010.638130
305NSW422034WATER2021-11-020.638130
306NSW422034WATER2021-11-030.638130
307NSW422034WATER2021-11-040.638130
\n", + "

308 rows × 6 columns

\n", + "
" ], - "execution_count": 9, - "metadata": { - "jupyter": { - "source_hidden": false, - "outputs_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - }, - "gather": { - "logged": 1637302369565 - } - } + "text/plain": [ + " DATASOURCEID SITEID SUBJECTID DATETIME VALUE QUALITYCODE\n", + "0 NSW 422034 WATER 2021-01-01 0.638 130\n", + "1 NSW 422034 WATER 2021-01-02 0.638 130\n", + "2 NSW 422034 WATER 2021-01-03 0.638 130\n", + "3 NSW 422034 WATER 2021-01-04 0.638 130\n", + "4 NSW 422034 WATER 2021-01-05 0.638 130\n", + ".. ... ... ... ... ... ...\n", + "303 NSW 422034 WATER 2021-10-31 0.638 130\n", + "304 NSW 422034 WATER 2021-11-01 0.638 130\n", + "305 NSW 422034 WATER 2021-11-02 0.638 130\n", + "306 NSW 422034 WATER 2021-11-03 0.638 130\n", + "307 NSW 422034 WATER 2021-11-04 0.638 130\n", + "\n", + "[308 rows x 6 columns]" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "gg.gauge_pull([\"422034\"], start_time_user = datetime.date(2021, 1, 1), end_time_user= datetime.date(2021, 11, 4), var='L' )" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "SA_Gauges = ['A4260902']\n", + "SA_Interval = [datetime.date(2010, 1, 1), datetime.date(2022, 12, 20)]\n", + "NSW_Gauges = ['425023']\n", + "NSW_Interval = [datetime.date(1980, 1, 1), datetime.date(2022, 12, 20)]" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "collapsed": false, + "gather": { + "logged": 1637302369565 + }, + "jupyter": { + "outputs_hidden": false, + "source_hidden": false }, + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [ { - "cell_type": "code", - "source": [ - "gg.gauge_getter.sort_gauges_by_state([\"407244\"])" - ], - "outputs": [ - { - "output_type": "execute_result", - "execution_count": 24, - "data": { - "text/plain": "{'NSW': [], 'QLD': [], 'VIC': ['407244'], 'rest': []}" - }, - "metadata": {} - } - ], - "execution_count": 24, - "metadata": { - "jupyter": { - "source_hidden": false, - "outputs_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - }, - "gather": { - "logged": 1637292539125 - } - } + "name": "stdout", + "output_type": "stream", + "text": [ + " Value[m] Quality Interpolation\n", + "Timestamp[UTC] \n", + "2009-12-31 14:30:00 3.167 10 ConstSucc\n", + "2010-01-01 14:30:00 3.160 10 ConstSucc\n", + "2010-01-02 14:30:00 3.177 10 ConstSucc\n", + "2010-01-03 14:30:00 3.185 10 ConstSucc\n", + "2010-01-04 14:30:00 3.170 10 ConstSucc\n", + "... ... ... ...\n", + "2022-12-14 14:30:00 6.396 140 ConstSucc\n", + "2022-12-15 14:30:00 6.479 140 ConstSucc\n", + "2022-12-16 14:30:00 6.553 140 ConstSucc\n", + "2022-12-17 14:30:00 6.619 140 ConstSucc\n", + "2022-12-18 14:30:00 NaN -1 ConstSucc\n", + "\n", + "[4736 rows x 3 columns]\n" + ] }, { - "cell_type": "markdown", - "source": [ - "To use bom_water to return data in the same format see below" + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
DATASOURCEIDSITEIDSUBJECTIDDATETIMEVALUEQUALITYCODE
0BOMA4260902WATER2009-12-313.16710
1BOMA4260902WATER2010-01-013.16010
2BOMA4260902WATER2010-01-023.17710
3BOMA4260902WATER2010-01-033.18510
4BOMA4260902WATER2010-01-043.17010
.....................
4731BOMA4260902WATER2022-12-146.396140
4732BOMA4260902WATER2022-12-156.479140
4733BOMA4260902WATER2022-12-166.553140
4734BOMA4260902WATER2022-12-176.619140
4735BOMA4260902WATER2022-12-18NaN-1
\n", + "

4736 rows × 6 columns

\n", + "
" ], - "metadata": { - "nteract": { - "transient": { - "deleting": false - } - } - } + "text/plain": [ + " DATASOURCEID SITEID SUBJECTID DATETIME VALUE QUALITYCODE\n", + "0 BOM A4260902 WATER 2009-12-31 3.167 10\n", + "1 BOM A4260902 WATER 2010-01-01 3.160 10\n", + "2 BOM A4260902 WATER 2010-01-02 3.177 10\n", + "3 BOM A4260902 WATER 2010-01-03 3.185 10\n", + "4 BOM A4260902 WATER 2010-01-04 3.170 10\n", + "... ... ... ... ... ... ...\n", + "4731 BOM A4260902 WATER 2022-12-14 6.396 140\n", + "4732 BOM A4260902 WATER 2022-12-15 6.479 140\n", + "4733 BOM A4260902 WATER 2022-12-16 6.553 140\n", + "4734 BOM A4260902 WATER 2022-12-17 6.619 140\n", + "4735 BOM A4260902 WATER 2022-12-18 NaN -1\n", + "\n", + "[4736 rows x 6 columns]" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "gg.gauge_pull(SA_Gauges, start_time_user = SA_Interval[0], end_time_user= SA_Interval[1], var='SL' )" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " Value[Ml] Quality Interpolation\n", + "Timestamp[UTC] \n", + "2009-12-31 14:30:00 27511.952 10 ConstSucc\n", + "2010-01-01 14:30:00 27427.927 10 ConstSucc\n", + "2010-01-02 14:30:00 27637.785 10 ConstSucc\n", + "2010-01-03 14:30:00 27736.984 10 ConstSucc\n", + "2010-01-04 14:30:00 27541.414 10 ConstSucc\n", + "... ... ... ...\n", + "2022-12-14 14:30:00 68036.643 140 ConstSucc\n", + "2022-12-15 14:30:00 69085.746 140 ConstSucc\n", + "2022-12-16 14:30:00 70014.974 140 ConstSucc\n", + "2022-12-17 14:30:00 70838.281 140 ConstSucc\n", + "2022-12-18 14:30:00 NaN -1 ConstSucc\n", + "\n", + "[4736 rows x 3 columns]\n" + ] }, { - "cell_type": "code", - "source": [ - "from typing import Tuple, List, Dict, TypeVar, Set, Optional, Any\r\n", - "import bom_water\r\n", - "import requests\r\n", - "import pandas as pd\r\n", - "\r\n", - "import datetime\r\n", - "\r\n", - "def fixdate(timestamp):\r\n", - " date = timestamp.to_pydatetime()\r\n", - " date = date.date()\r\n", - " #datetime.astimzone('Australia/Sydney',date) #date.tz_localize('Australia/Sydney') #datetime.datetime.strptime(date, '%Y-%m-%dT%H:%M:%S')\r\n", - " return date\r\n", - "\r\n", - "def gauge_pull_bom(gauge_numbers: List[str], start_time_user: datetime.date, end_time_user: datetime.date,\r\n", - " var: str = 'F',\r\n", - " interval: str = 'day', data_type: str = 'mean') -> pd.DataFrame:\r\n", - " '''\r\n", - " Given a list of gauge numbers, breaks the list into individual gauges, and uses BomWater to get data, \r\n", - " returning as a Pandas dataframe object in a gauge getter format.\r\n", - " '''\r\n", - " \r\n", - " bm = bom_water.BomWater()\r\n", - "\r\n", - " if (interval == 'day') & (data_type == 'mean'):\r\n", - " procedure = bm.procedures.Pat4_C_B_1_DailyMean\r\n", - " \r\n", - " if var == \"F\":\r\n", - " prop = bm.properties.Water_Course_Discharge\r\n", - "\r\n", - " t_begin = start_time_user.strftime(\"%Y-%m-%dT%H:%M:%S%z\")\r\n", - " t_end = end_time_user.strftime(\"%Y-%m-%dT%H:%M:%S%z\")\r\n", - "\r\n", - " # t_begin = \"1800-01-01T00:00:00+10\"\r\n", - " # t_end = \"2030-12-31T00:00:00+10\"\r\n", - " collect=[]\r\n", - " for gauge in gauge_numbers:\r\n", - " \r\n", - " response = bm.request(bm.actions.GetObservation, gauge, prop, procedure, t_begin, t_end)\r\n", - "\r\n", - " response_json = bm.xml_to_json(response.text) \r\n", - " ts = bm.parse_get_data(response)\r\n", - " # print(ts)\r\n", - " if ts.empty:\r\n", - " ts = pd.DataFrame(columns=[\"DATASOURCEID\",\"SITEID\",\t\"SUBJECTID\", \"DATETIME\", \"VALUE\", \"QUALITYCODE\"])\r\n", - " collect.append(ts)\r\n", - " else:\r\n", - " # move to format DATASOURCEID\tSITEID\tSUBJECTID\tDATETIME\tVALUE\tQUALITYCODE\r\n", - " ts[\"DATASOURCEID\"] = \"BOM\"\r\n", - " ts[\"SITEID\"] = gauge\r\n", - " ts[\"SUBJECTID\"] = \"WATER\"\r\n", - " ts[\"DATETIME\"] = ts.index.to_pydatetime()\r\n", - " ts[\"DATETIME\"] = pd.to_datetime(ts[\"DATETIME\"])\r\n", - " ts[\"DATETIME\"] = ts[\"DATETIME\"].apply(fixdate)\r\n", - " ts[\"VALUE\"] = ts[\"Value[cumec]\"]\r\n", - " ts[\"QUALITYCODE\"] = ts[\"Quality\"]\r\n", - " ts.reset_index(drop=True, inplace=True)\r\n", - "\r\n", - " collect.append(ts[[\"DATASOURCEID\",\"SITEID\",\t\"SUBJECTID\", \"DATETIME\", \"VALUE\", \"QUALITYCODE\"]])\r\n", - " output = pd.concat(collect)\r\n", - " return output\r\n" + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
DATASOURCEIDSITEIDSUBJECTIDDATETIMEVALUEQUALITYCODE
0BOMA4260902WATER2009-12-3127511.95210
1BOMA4260902WATER2010-01-0127427.92710
2BOMA4260902WATER2010-01-0227637.78510
3BOMA4260902WATER2010-01-0327736.98410
4BOMA4260902WATER2010-01-0427541.41410
.....................
4731BOMA4260902WATER2022-12-1468036.643140
4732BOMA4260902WATER2022-12-1569085.746140
4733BOMA4260902WATER2022-12-1670014.974140
4734BOMA4260902WATER2022-12-1770838.281140
4735BOMA4260902WATER2022-12-18NaN-1
\n", + "

4736 rows × 6 columns

\n", + "
" ], - "outputs": [], - "execution_count": 26, - "metadata": { - "jupyter": { - "source_hidden": false, - "outputs_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - }, - "gather": { - "logged": 1669344979378 - } - } + "text/plain": [ + " DATASOURCEID SITEID SUBJECTID DATETIME VALUE QUALITYCODE\n", + "0 BOM A4260902 WATER 2009-12-31 27511.952 10\n", + "1 BOM A4260902 WATER 2010-01-01 27427.927 10\n", + "2 BOM A4260902 WATER 2010-01-02 27637.785 10\n", + "3 BOM A4260902 WATER 2010-01-03 27736.984 10\n", + "4 BOM A4260902 WATER 2010-01-04 27541.414 10\n", + "... ... ... ... ... ... ...\n", + "4731 BOM A4260902 WATER 2022-12-14 68036.643 140\n", + "4732 BOM A4260902 WATER 2022-12-15 69085.746 140\n", + "4733 BOM A4260902 WATER 2022-12-16 70014.974 140\n", + "4734 BOM A4260902 WATER 2022-12-17 70838.281 140\n", + "4735 BOM A4260902 WATER 2022-12-18 NaN -1\n", + "\n", + "[4736 rows x 6 columns]" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "gg.gauge_pull(SA_Gauges, start_time_user = SA_Interval[0], end_time_user= SA_Interval[1], data_source = 'BOM', var='SV' )" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " Value[m] Quality Interpolation\n", + "Timestamp[UTC] \n", + "1979-12-31 14:00:00 59.852 110 ConstSucc\n", + "1980-01-01 14:00:00 59.861 110 ConstSucc\n", + "1980-01-02 14:00:00 59.867 110 ConstSucc\n", + "1980-01-03 14:00:00 59.861 110 ConstSucc\n", + "1980-01-04 14:00:00 59.856 110 ConstSucc\n", + "... ... ... ...\n", + "2022-12-13 14:00:00 60.182 140 ConstSucc\n", + "2022-12-14 14:00:00 60.166 140 ConstSucc\n", + "2022-12-15 14:00:00 60.179 140 ConstSucc\n", + "2022-12-16 14:00:00 60.184 140 ConstSucc\n", + "2022-12-17 14:00:00 NaN -1 ConstSucc\n", + "\n", + "[15693 rows x 3 columns]\n" + ] }, { - "cell_type": "code", - "source": [ - "gg.gauge_getter.gauge_pull_bom([\"410777\", \"412189\"], start_time_user = datetime.date(2022, 11, 1), end_time_user= datetime.date(2022, 11, 25), var='F' )" - ], - "outputs": [ - { - "output_type": "execute_result", - "execution_count": 6, - "data": { - "text/plain": "[['BOM', '410777', 'WATER', datetime.date(2022, 10, 31), nan, -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 1), nan, -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 2), nan, -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 3), nan, -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 4), nan, -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 5), nan, -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 6), nan, -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 7), nan, -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 8), nan, -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 9), nan, -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 10), nan, -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 11), nan, -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 12), nan, -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 13), nan, -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 14), nan, -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 15), nan, -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 16), nan, -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 17), nan, -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 18), nan, -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 10, 31), 44104.176, -1],\n ['BOM',\n '410777',\n 'WATER',\n datetime.date(2022, 11, 1),\n 30493.584000000003,\n -1],\n ['BOM',\n '410777',\n 'WATER',\n datetime.date(2022, 11, 2),\n 21748.176000000003,\n -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 3), 16050.096, -1],\n ['BOM',\n '410777',\n 'WATER',\n datetime.date(2022, 11, 4),\n 13022.985600000002,\n -1],\n ['BOM',\n '410777',\n 'WATER',\n datetime.date(2022, 11, 5),\n 11120.371200000001,\n -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 6), 10379.4048, -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 7), 11848.6368, -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 8), 9429.264000000001, -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 9), 9240.1344, -1],\n ['BOM',\n '410777',\n 'WATER',\n datetime.date(2022, 11, 10),\n 8013.168000000001,\n -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 11), 7179.5808, -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 12), 8835.0912, -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 13), 16183.3248, -1],\n ['BOM',\n '410777',\n 'WATER',\n datetime.date(2022, 11, 14),\n 15531.523200000001,\n -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 15), 11910.7584, -1],\n ['BOM',\n '410777',\n 'WATER',\n datetime.date(2022, 11, 16),\n 9049.190400000001,\n -1],\n ['BOM',\n '410777',\n 'WATER',\n datetime.date(2022, 11, 17),\n 7616.160000000001,\n -1],\n ['BOM', '410777', 'WATER', datetime.date(2022, 11, 18), nan, -1],\n ['BOM', '412189', 'WATER', datetime.date(2022, 10, 31), nan, -1],\n ['BOM', '412189', 'WATER', datetime.date(2022, 11, 1), nan, -1],\n ['BOM', '412189', 'WATER', datetime.date(2022, 11, 2), nan, -1],\n ['BOM', '412189', 'WATER', datetime.date(2022, 11, 3), nan, -1],\n ['BOM', '412189', 'WATER', datetime.date(2022, 11, 4), nan, -1],\n ['BOM', '412189', 'WATER', datetime.date(2022, 11, 5), nan, -1],\n ['BOM', '412189', 'WATER', datetime.date(2022, 11, 6), nan, -1],\n ['BOM', '412189', 'WATER', datetime.date(2022, 11, 7), nan, -1],\n ['BOM', '412189', 'WATER', datetime.date(2022, 11, 8), nan, -1],\n ['BOM', '412189', 'WATER', datetime.date(2022, 11, 9), nan, -1],\n ['BOM', '412189', 'WATER', datetime.date(2022, 11, 10), nan, -1],\n ['BOM', '412189', 'WATER', datetime.date(2022, 11, 11), nan, -1],\n ['BOM', '412189', 'WATER', datetime.date(2022, 11, 12), nan, -1],\n ['BOM', '412189', 'WATER', datetime.date(2022, 11, 13), nan, -1],\n ['BOM', '412189', 'WATER', datetime.date(2022, 11, 14), nan, -1],\n ['BOM', '412189', 'WATER', datetime.date(2022, 11, 15), nan, -1],\n ['BOM', '412189', 'WATER', datetime.date(2022, 11, 16), nan, -1],\n ['BOM', '412189', 'WATER', datetime.date(2022, 11, 17), nan, -1],\n ['BOM', '412189', 'WATER', datetime.date(2022, 11, 18), nan, -1],\n ['BOM', '412189', 'WATER', datetime.date(2022, 11, 19), nan, -1],\n ['BOM', '412189', 'WATER', datetime.date(2022, 11, 20), nan, -1],\n ['BOM', '412189', 'WATER', datetime.date(2022, 11, 21), nan, -1],\n ['BOM', '412189', 'WATER', datetime.date(2022, 11, 22), nan, -1],\n ['BOM', '412189', 'WATER', datetime.date(2022, 11, 23), nan, -1],\n ['BOM', '412189', 'WATER', datetime.date(2022, 11, 24), nan, -1]]" - }, - "metadata": {} - } + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
DATASOURCEIDSITEIDSUBJECTIDDATETIMEVALUEQUALITYCODE
0BOM425023WATER1979-12-3159.852110
1BOM425023WATER1980-01-0159.861110
2BOM425023WATER1980-01-0259.867110
3BOM425023WATER1980-01-0359.861110
4BOM425023WATER1980-01-0459.856110
.....................
15688BOM425023WATER2022-12-1360.182140
15689BOM425023WATER2022-12-1460.166140
15690BOM425023WATER2022-12-1560.179140
15691BOM425023WATER2022-12-1660.184140
15692BOM425023WATER2022-12-17NaN-1
\n", + "

15693 rows × 6 columns

\n", + "
" ], - "execution_count": 6, - "metadata": { - "jupyter": { - "source_hidden": false, - "outputs_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - }, - "gather": { - "logged": 1669960320102 - } - } + "text/plain": [ + " DATASOURCEID SITEID SUBJECTID DATETIME VALUE QUALITYCODE\n", + "0 BOM 425023 WATER 1979-12-31 59.852 110\n", + "1 BOM 425023 WATER 1980-01-01 59.861 110\n", + "2 BOM 425023 WATER 1980-01-02 59.867 110\n", + "3 BOM 425023 WATER 1980-01-03 59.861 110\n", + "4 BOM 425023 WATER 1980-01-04 59.856 110\n", + "... ... ... ... ... ... ...\n", + "15688 BOM 425023 WATER 2022-12-13 60.182 140\n", + "15689 BOM 425023 WATER 2022-12-14 60.166 140\n", + "15690 BOM 425023 WATER 2022-12-15 60.179 140\n", + "15691 BOM 425023 WATER 2022-12-16 60.184 140\n", + "15692 BOM 425023 WATER 2022-12-17 NaN -1\n", + "\n", + "[15693 rows x 6 columns]" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "gg.gauge_pull(NSW_Gauges, start_time_user = NSW_Interval[0], end_time_user= NSW_Interval[1], data_source = 'BOM', var='SL' )" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " Value[Ml] Quality Interpolation\n", + "Timestamp[UTC] \n", + "1979-12-31 14:00:00 632486.140 140 ConstSucc\n", + "1980-01-01 14:00:00 633602.913 140 ConstSucc\n", + "1980-01-02 14:00:00 634335.478 140 ConstSucc\n", + "1980-01-03 14:00:00 633537.066 140 ConstSucc\n", + "1980-01-04 14:00:00 632983.675 140 ConstSucc\n", + "... ... ... ...\n", + "2022-12-13 14:00:00 672499.138 140 ConstSucc\n", + "2022-12-14 14:00:00 670539.623 140 ConstSucc\n", + "2022-12-15 14:00:00 672077.372 140 ConstSucc\n", + "2022-12-16 14:00:00 672727.960 140 ConstSucc\n", + "2022-12-17 14:00:00 NaN -1 ConstSucc\n", + "\n", + "[15693 rows x 3 columns]\n" + ] }, { - "cell_type": "code", - "source": [ - "gg.gauge_pull([\"425010\"], start_time_user = datetime.date(2012, 11, 1), end_time_user= datetime.date(2022, 11, 25) ,data_source= \"bom\")" - ], - "outputs": [ - { - "output_type": "execute_result", - "execution_count": 16, - "data": { - "text/plain": " DATASOURCEID SITEID SUBJECTID DATETIME VALUE QUALITYCODE\n0 BOM 425010 WATER 2012-10-31 20550.8448 140\n1 BOM 425010 WATER 2012-11-01 20626.6176 140\n2 BOM 425010 WATER 2012-11-02 20499.0912 140\n3 BOM 425010 WATER 2012-11-03 19806.3360 140\n4 BOM 425010 WATER 2012-11-04 19042.3872 140\n... ... ... ... ... ... ...\n3672 BOM 425010 WATER 2022-11-20 NaN -1\n3673 BOM 425010 WATER 2022-11-21 NaN -1\n3674 BOM 425010 WATER 2022-11-22 NaN -1\n3675 BOM 425010 WATER 2022-11-23 NaN -1\n3676 BOM 425010 WATER 2022-11-24 NaN -1\n\n[3677 rows x 6 columns]", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
DATASOURCEIDSITEIDSUBJECTIDDATETIMEVALUEQUALITYCODE
0BOM425010WATER2012-10-3120550.8448140
1BOM425010WATER2012-11-0120626.6176140
2BOM425010WATER2012-11-0220499.0912140
3BOM425010WATER2012-11-0319806.3360140
4BOM425010WATER2012-11-0419042.3872140
.....................
3672BOM425010WATER2022-11-20NaN-1
3673BOM425010WATER2022-11-21NaN-1
3674BOM425010WATER2022-11-22NaN-1
3675BOM425010WATER2022-11-23NaN-1
3676BOM425010WATER2022-11-24NaN-1
\n

3677 rows × 6 columns

\n
" - }, - "metadata": {} - } + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
DATASOURCEIDSITEIDSUBJECTIDDATETIMEVALUEQUALITYCODE
0BOM425023WATER1979-12-31632486.140140
1BOM425023WATER1980-01-01633602.913140
2BOM425023WATER1980-01-02634335.478140
3BOM425023WATER1980-01-03633537.066140
4BOM425023WATER1980-01-04632983.675140
.....................
15688BOM425023WATER2022-12-13672499.138140
15689BOM425023WATER2022-12-14670539.623140
15690BOM425023WATER2022-12-15672077.372140
15691BOM425023WATER2022-12-16672727.960140
15692BOM425023WATER2022-12-17NaN-1
\n", + "

15693 rows × 6 columns

\n", + "
" ], - "execution_count": 16, - "metadata": { - "jupyter": { - "source_hidden": false, - "outputs_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - }, - "gather": { - "logged": 1669960600928 - } - } + "text/plain": [ + " DATASOURCEID SITEID SUBJECTID DATETIME VALUE QUALITYCODE\n", + "0 BOM 425023 WATER 1979-12-31 632486.140 140\n", + "1 BOM 425023 WATER 1980-01-01 633602.913 140\n", + "2 BOM 425023 WATER 1980-01-02 634335.478 140\n", + "3 BOM 425023 WATER 1980-01-03 633537.066 140\n", + "4 BOM 425023 WATER 1980-01-04 632983.675 140\n", + "... ... ... ... ... ... ...\n", + "15688 BOM 425023 WATER 2022-12-13 672499.138 140\n", + "15689 BOM 425023 WATER 2022-12-14 670539.623 140\n", + "15690 BOM 425023 WATER 2022-12-15 672077.372 140\n", + "15691 BOM 425023 WATER 2022-12-16 672727.960 140\n", + "15692 BOM 425023 WATER 2022-12-17 NaN -1\n", + "\n", + "[15693 rows x 6 columns]" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "gg.gauge_pull(NSW_Gauges, start_time_user = NSW_Interval[0], end_time_user= NSW_Interval[1], data_source = 'BOM',var='SV' )" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": { + "collapsed": false, + "gather": { + "logged": 1637292539125 + }, + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'NSW': [], 'QLD': [], 'VIC': ['407244'], 'rest': []}" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "gg.gauge_getter.sort_gauges_by_state([\"407244\"])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "nteract": { + "transient": { + "deleting": false + } + } + }, + "source": [ + "To use bom_water to return data in the same format see below" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": { + "collapsed": false, + "gather": { + "logged": 1669344979378 + }, + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [ + "from typing import Tuple, List, Dict, TypeVar, Set, Optional, Any\n", + "import bom_water\n", + "import requests\n", + "import pandas as pd\n", + "\n", + "import datetime\n", + "\n", + "def fixdate(timestamp):\n", + " date = timestamp.to_pydatetime()\n", + " date = date.date()\n", + " #datetime.astimzone('Australia/Sydney',date) #date.tz_localize('Australia/Sydney') #datetime.datetime.strptime(date, '%Y-%m-%dT%H:%M:%S')\n", + " return date\n", + "\n", + "def gauge_pull_bom(gauge_numbers: List[str], start_time_user: datetime.date, end_time_user: datetime.date,\n", + " var: str = 'F',\n", + " interval: str = 'day', data_type: str = 'mean') -> pd.DataFrame:\n", + " '''\n", + " Given a list of gauge numbers, breaks the list into individual gauges, and uses BomWater to get data, \n", + " returning as a Pandas dataframe object in a gauge getter format.\n", + " '''\n", + " \n", + " bm = bom_water.BomWater()\n", + "\n", + " if (interval == 'day') & (data_type == 'mean'):\n", + " procedure = bm.procedures.Pat4_C_B_1_DailyMean\n", + " \n", + " if var == \"F\":\n", + " prop = bm.properties.Water_Course_Discharge\n", + "\n", + " t_begin = start_time_user.strftime(\"%Y-%m-%dT%H:%M:%S%z\")\n", + " t_end = end_time_user.strftime(\"%Y-%m-%dT%H:%M:%S%z\")\n", + "\n", + " # t_begin = \"1800-01-01T00:00:00+10\"\n", + " # t_end = \"2030-12-31T00:00:00+10\"\n", + " collect=[]\n", + " for gauge in gauge_numbers:\n", + " \n", + " response = bm.request(bm.actions.GetObservation, gauge, prop, procedure, t_begin, t_end)\n", + "\n", + " response_json = bm.xml_to_json(response.text) \n", + " ts = bm.parse_get_data(response)\n", + " # print(ts)\n", + " if ts.empty:\n", + " ts = pd.DataFrame(columns=[\"DATASOURCEID\",\"SITEID\",\t\"SUBJECTID\", \"DATETIME\", \"VALUE\", \"QUALITYCODE\"])\n", + " collect.append(ts)\n", + " else:\n", + " # move to format DATASOURCEID\tSITEID\tSUBJECTID\tDATETIME\tVALUE\tQUALITYCODE\n", + " ts[\"DATASOURCEID\"] = \"BOM\"\n", + " ts[\"SITEID\"] = gauge\n", + " ts[\"SUBJECTID\"] = \"WATER\"\n", + " ts[\"DATETIME\"] = ts.index.to_pydatetime()\n", + " ts[\"DATETIME\"] = pd.to_datetime(ts[\"DATETIME\"])\n", + " ts[\"DATETIME\"] = ts[\"DATETIME\"].apply(fixdate)\n", + " ts[\"VALUE\"] = ts[\"Value[cumec]\"]\n", + " ts[\"QUALITYCODE\"] = ts[\"Quality\"]\n", + " ts.reset_index(drop=True, inplace=True)\n", + "\n", + " collect.append(ts[[\"DATASOURCEID\",\"SITEID\",\t\"SUBJECTID\", \"DATETIME\", \"VALUE\", \"QUALITYCODE\"]])\n", + " output = pd.concat(collect)\n", + " return output\n" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "collapsed": false, + "gather": { + "logged": 1669960320102 + }, + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "[['BOM', '410777', 'WATER', datetime.date(2022, 10, 31), nan, -1],\n", + " ['BOM', '410777', 'WATER', datetime.date(2022, 11, 1), nan, -1],\n", + " ['BOM', '410777', 'WATER', datetime.date(2022, 11, 2), nan, -1],\n", + " ['BOM', '410777', 'WATER', datetime.date(2022, 11, 3), nan, -1],\n", + " ['BOM', '410777', 'WATER', datetime.date(2022, 11, 4), nan, -1],\n", + " ['BOM', '410777', 'WATER', datetime.date(2022, 11, 5), nan, -1],\n", + " ['BOM', '410777', 'WATER', datetime.date(2022, 11, 6), nan, -1],\n", + " ['BOM', '410777', 'WATER', datetime.date(2022, 11, 7), nan, -1],\n", + " ['BOM', '410777', 'WATER', datetime.date(2022, 11, 8), nan, -1],\n", + " ['BOM', '410777', 'WATER', datetime.date(2022, 11, 9), nan, -1],\n", + " ['BOM', '410777', 'WATER', datetime.date(2022, 11, 10), nan, -1],\n", + " ['BOM', '410777', 'WATER', datetime.date(2022, 11, 11), nan, -1],\n", + " ['BOM', '410777', 'WATER', datetime.date(2022, 11, 12), nan, -1],\n", + " ['BOM', '410777', 'WATER', datetime.date(2022, 11, 13), nan, -1],\n", + " ['BOM', '410777', 'WATER', datetime.date(2022, 11, 14), nan, -1],\n", + " ['BOM', '410777', 'WATER', datetime.date(2022, 11, 15), nan, -1],\n", + " ['BOM', '410777', 'WATER', datetime.date(2022, 11, 16), nan, -1],\n", + " ['BOM', '410777', 'WATER', datetime.date(2022, 11, 17), nan, -1],\n", + " ['BOM', '410777', 'WATER', datetime.date(2022, 11, 18), nan, -1],\n", + " ['BOM', '410777', 'WATER', datetime.date(2022, 10, 31), 44104.176, -1],\n", + " ['BOM',\n", + " '410777',\n", + " 'WATER',\n", + " datetime.date(2022, 11, 1),\n", + " 30493.584000000003,\n", + " -1],\n", + " ['BOM',\n", + " '410777',\n", + " 'WATER',\n", + " datetime.date(2022, 11, 2),\n", + " 21748.176000000003,\n", + " -1],\n", + " ['BOM', '410777', 'WATER', datetime.date(2022, 11, 3), 16050.096, -1],\n", + " ['BOM',\n", + " '410777',\n", + " 'WATER',\n", + " datetime.date(2022, 11, 4),\n", + " 13022.985600000002,\n", + " -1],\n", + " ['BOM',\n", + " '410777',\n", + " 'WATER',\n", + " datetime.date(2022, 11, 5),\n", + " 11120.371200000001,\n", + " -1],\n", + " ['BOM', '410777', 'WATER', datetime.date(2022, 11, 6), 10379.4048, -1],\n", + " ['BOM', '410777', 'WATER', datetime.date(2022, 11, 7), 11848.6368, -1],\n", + " ['BOM', '410777', 'WATER', datetime.date(2022, 11, 8), 9429.264000000001, -1],\n", + " ['BOM', '410777', 'WATER', datetime.date(2022, 11, 9), 9240.1344, -1],\n", + " ['BOM',\n", + " '410777',\n", + " 'WATER',\n", + " datetime.date(2022, 11, 10),\n", + " 8013.168000000001,\n", + " -1],\n", + " ['BOM', '410777', 'WATER', datetime.date(2022, 11, 11), 7179.5808, -1],\n", + " ['BOM', '410777', 'WATER', datetime.date(2022, 11, 12), 8835.0912, -1],\n", + " ['BOM', '410777', 'WATER', datetime.date(2022, 11, 13), 16183.3248, -1],\n", + " ['BOM',\n", + " '410777',\n", + " 'WATER',\n", + " datetime.date(2022, 11, 14),\n", + " 15531.523200000001,\n", + " -1],\n", + " ['BOM', '410777', 'WATER', datetime.date(2022, 11, 15), 11910.7584, -1],\n", + " ['BOM',\n", + " '410777',\n", + " 'WATER',\n", + " datetime.date(2022, 11, 16),\n", + " 9049.190400000001,\n", + " -1],\n", + " ['BOM',\n", + " '410777',\n", + " 'WATER',\n", + " datetime.date(2022, 11, 17),\n", + " 7616.160000000001,\n", + " -1],\n", + " ['BOM', '410777', 'WATER', datetime.date(2022, 11, 18), nan, -1],\n", + " ['BOM', '412189', 'WATER', datetime.date(2022, 10, 31), nan, -1],\n", + " ['BOM', '412189', 'WATER', datetime.date(2022, 11, 1), nan, -1],\n", + " ['BOM', '412189', 'WATER', datetime.date(2022, 11, 2), nan, -1],\n", + " ['BOM', '412189', 'WATER', datetime.date(2022, 11, 3), nan, -1],\n", + " ['BOM', '412189', 'WATER', datetime.date(2022, 11, 4), nan, -1],\n", + " ['BOM', '412189', 'WATER', datetime.date(2022, 11, 5), nan, -1],\n", + " ['BOM', '412189', 'WATER', datetime.date(2022, 11, 6), nan, -1],\n", + " ['BOM', '412189', 'WATER', datetime.date(2022, 11, 7), nan, -1],\n", + " ['BOM', '412189', 'WATER', datetime.date(2022, 11, 8), nan, -1],\n", + " ['BOM', '412189', 'WATER', datetime.date(2022, 11, 9), nan, -1],\n", + " ['BOM', '412189', 'WATER', datetime.date(2022, 11, 10), nan, -1],\n", + " ['BOM', '412189', 'WATER', datetime.date(2022, 11, 11), nan, -1],\n", + " ['BOM', '412189', 'WATER', datetime.date(2022, 11, 12), nan, -1],\n", + " ['BOM', '412189', 'WATER', datetime.date(2022, 11, 13), nan, -1],\n", + " ['BOM', '412189', 'WATER', datetime.date(2022, 11, 14), nan, -1],\n", + " ['BOM', '412189', 'WATER', datetime.date(2022, 11, 15), nan, -1],\n", + " ['BOM', '412189', 'WATER', datetime.date(2022, 11, 16), nan, -1],\n", + " ['BOM', '412189', 'WATER', datetime.date(2022, 11, 17), nan, -1],\n", + " ['BOM', '412189', 'WATER', datetime.date(2022, 11, 18), nan, -1],\n", + " ['BOM', '412189', 'WATER', datetime.date(2022, 11, 19), nan, -1],\n", + " ['BOM', '412189', 'WATER', datetime.date(2022, 11, 20), nan, -1],\n", + " ['BOM', '412189', 'WATER', datetime.date(2022, 11, 21), nan, -1],\n", + " ['BOM', '412189', 'WATER', datetime.date(2022, 11, 22), nan, -1],\n", + " ['BOM', '412189', 'WATER', datetime.date(2022, 11, 23), nan, -1],\n", + " ['BOM', '412189', 'WATER', datetime.date(2022, 11, 24), nan, -1]]" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "gg.gauge_getter.gauge_pull_bom([\"410777\", \"412189\"], start_time_user = datetime.date(2022, 11, 1), end_time_user= datetime.date(2022, 11, 25), var='F' )" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "collapsed": false, + "gather": { + "logged": 1669960600928 }, + "jupyter": { + "outputs_hidden": false, + "source_hidden": false + }, + "nteract": { + "transient": { + "deleting": false + } + } + }, + "outputs": [ { - "cell_type": "code", - "source": [], - "outputs": [], - "execution_count": null, - "metadata": { - "jupyter": { - "source_hidden": false, - "outputs_hidden": false - }, - "nteract": { - "transient": { - "deleting": false - } - } - } + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
DATASOURCEIDSITEIDSUBJECTIDDATETIMEVALUEQUALITYCODE
0BOM425010WATER2012-10-3120550.8448140
1BOM425010WATER2012-11-0120626.6176140
2BOM425010WATER2012-11-0220499.0912140
3BOM425010WATER2012-11-0319806.3360140
4BOM425010WATER2012-11-0419042.3872140
.....................
3672BOM425010WATER2022-11-20NaN-1
3673BOM425010WATER2022-11-21NaN-1
3674BOM425010WATER2022-11-22NaN-1
3675BOM425010WATER2022-11-23NaN-1
3676BOM425010WATER2022-11-24NaN-1
\n", + "

3677 rows × 6 columns

\n", + "
" + ], + "text/plain": [ + " DATASOURCEID SITEID SUBJECTID DATETIME VALUE QUALITYCODE\n", + "0 BOM 425010 WATER 2012-10-31 20550.8448 140\n", + "1 BOM 425010 WATER 2012-11-01 20626.6176 140\n", + "2 BOM 425010 WATER 2012-11-02 20499.0912 140\n", + "3 BOM 425010 WATER 2012-11-03 19806.3360 140\n", + "4 BOM 425010 WATER 2012-11-04 19042.3872 140\n", + "... ... ... ... ... ... ...\n", + "3672 BOM 425010 WATER 2022-11-20 NaN -1\n", + "3673 BOM 425010 WATER 2022-11-21 NaN -1\n", + "3674 BOM 425010 WATER 2022-11-22 NaN -1\n", + "3675 BOM 425010 WATER 2022-11-23 NaN -1\n", + "3676 BOM 425010 WATER 2022-11-24 NaN -1\n", + "\n", + "[3677 rows x 6 columns]" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" } - ], - "metadata": { - "kernelspec": { - "name": "python3", - "language": "python", - "display_name": "Python 3" - }, - "language_info": { - "name": "python", - "version": "3.6.9", - "mimetype": "text/x-python", - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "pygments_lexer": "ipython3", - "nbconvert_exporter": "python", - "file_extension": ".py" - }, - "kernel_info": { - "name": "python3" + ], + "source": [ + "gg.gauge_pull([\"425010\"], start_time_user = datetime.date(2012, 11, 1), end_time_user= datetime.date(2022, 11, 25) ,data_source= \"bom\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false, + "source_hidden": false }, "nteract": { - "version": "nteract-front-end@1.0.0" - }, - "microsoft": { - "host": { - "AzureML": { - "notebookHasBeenCompleted": true - } - } + "transient": { + "deleting": false + } + } + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernel_info": { + "name": "python3" + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.5" + }, + "microsoft": { + "host": { + "AzureML": { + "notebookHasBeenCompleted": true } + } }, - "nbformat": 4, - "nbformat_minor": 2 -} \ No newline at end of file + "nteract": { + "version": "nteract-front-end@1.0.0" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/README.md b/README.md index 5679ef5..c7e8aa7 100644 --- a/README.md +++ b/README.md @@ -15,25 +15,60 @@ ## Description MDBA Gauge Getter provides a unified and simple interface to collect surface water data from the following state water portals: - |State|Site|Source| | --- | --- | --- | | NSW | realtimedata.waternsw.com.au |CP| | QLD | water-monitoring.information.qld.gov.au |PUBLISH| | VIC | data.water.vic.gov.au |AT| +| SA | bom.gov.au/waterdata/ | BOM Water Data Online| -The tool is configured to abstract away the details specific to each state water portal and return a consistent structure. -By default it will return a daily mean of a flow in ML/day for a given gauge number, but level and other intervals and aggregations are available +The tool is configured to abstract away the details specific to each state water portal and return a consistent structure. By default it will return a daily mean of a flow in ML/day for a given gauge number, but storage level, storage volume, other intervals and aggregations are available. Example of its use can be seen in Gauge_getter_example.ipynb notebook, contact ben.bradshaw@mdba.gov.au for more details. -## Installation +## Local Installation via git +- Clone the repo in your local folder: for example, inside `~./users/john.doe` git clone `https://github.com/MDBAuth/MDBA_Gauge_Getter.git` which will create a git tracked project repository inside `~./users/john.doe/mdba_gauge_getter`. +- Create a virtual environment: for example, `conda create --name gauge_getter_env`. +- Activate the virtual environment: `conda activate gauge_getter_env`. +- Go inside the project folder i.e., `cd ~./users/john.doe/mdba_gauge_getter`. +- Check `git status` and `git branch`. - Install dependencies with `pip3 install -r requirements.txt` - (Optionally) install dev dependencies with `pip3 install -r requirements-dev.txt` - Run `python3 setup.py install` to install the module +## Quick Start + +- Install via pip with the command: `pip install mdba-gauge-getter` +- After installation, import the package with the command: `import mdba_gauge_getter.gauge_getter as gg` +- Import datetime for converting your intervals into python datetime object: `import datetime as dt` + +## Usage + +There are several options to call Gauge Getter which are as follows: +- `gauge_numbers` denotes the gauge(s) for which the parameters such as flow, lake/storage level, storage volume etc. will be obtained. It takes a list of strings (gauge numbers) as input. +- `start_time_user` denotes the start time of the userdefined interval. It takes a datetime python object as input. +- `end_time_user` denotes the end time of the userdefined interval. It takes a datetime python object as input. +- `data_source` denotes which state the gauge(s) belong(s) to which API to fetch the data from. Please note that SA does not currently have an API to obtain the data from; hence, the data is fetched from the BOM API. Different *data_source* options are: + - i.e., 'NSW', 'VIC', 'QLD', 'SA'/'BOM' + - 'BOM' +- `var` denotes the parameter to retreieve such as flow, lake/storage level, storage volume etc. It takes a string indicating the parameter type as input. Different string notation for different *var* options are: + - 'F' for flow (default). + - 'L' for water level used for flow calculation. + - 'LL'/'SL' for lake/storage level. + - 'SV' for storage volume. Please note that this is exclusively a BOM API parameter. Please specify the data source as 'BOM' if you would like to retrieve this parameter. +- `interval` indicates the duration the parameter data are collected for aggregation. Different *interval* options are: + - 'day'. Alternate options for BOM API call is: 'd'. + - 'hour'. Alternate options for BOM API call is: 'h'. + - 'month'. Alternate options for BOM API call is: 'm'. + - 'year'. Alternate options for BOM API call is: 'y'. +- `data_type` inidcates the aggregation method. Different *data_type* options are: + - 'mean' (default). Alternate options for BOM API call are: 'avg', 'average', 'av' and 'a'. + - 'min'. Alternate options for BOM API call is: 'minimum'. Only available when obtaining *daily* interval data. + - 'max'. Alternate options for BOM API call is: 'maximum'. Only available when obtaining *daily* interval data. + ## Support -For issues relating to the script, a tutorial, or feedback please contact Ben Bradshaw ben.bradshaw@mdba.gov.au -For data issues please see the corresponding state water portals +For issues relating to the script, a tutorial, or feedback please contact Ben Bradshaw (ben.bradshaw@mdba.gov.au) or Ahsanul Habib (ahsanul.habib@mdba.gov.au). + +For data issues please see the corresponding state water portals. diff --git a/mdba_gauge_getter/gauge_getter.py b/mdba_gauge_getter/gauge_getter.py index d8b1d16..bba2c84 100644 --- a/mdba_gauge_getter/gauge_getter.py +++ b/mdba_gauge_getter/gauge_getter.py @@ -377,7 +377,6 @@ def gauge_pull_bom(gauge_numbers: List[str], start_time_user: datetime.date, end response = bm.request(bm.actions.GetObservation, gauge, prop, procedure, t_begin, t_end) # response_json = bm.xml_to_json(response.text) ts = bm.parse_get_data(response) - print(ts) if ts.empty: ts = pd.DataFrame(columns=["DATASOURCEID","SITEID", "SUBJECTID", "DATETIME", "VALUE", "QUALITYCODE"]) collect.append(ts) diff --git a/mdba_gauge_getter/version.py b/mdba_gauge_getter/version.py index 68eb9b6..ab45471 100644 --- a/mdba_gauge_getter/version.py +++ b/mdba_gauge_getter/version.py @@ -1 +1 @@ -__version__ = '0.4.5' +__version__ = '0.4.6' diff --git a/tests/test_gauge_getter.py b/tests/test_gauge_getter.py index 68516e0..55ed6e6 100644 --- a/tests/test_gauge_getter.py +++ b/tests/test_gauge_getter.py @@ -373,7 +373,7 @@ def dict_matches(d1: Dict[str, Any], d2: Dict[str, Any]): def test_sort_gauges_by_state(): gauge_getter.gauge_data_uri = StringIO(MOCK_CSV) - print(gauge_getter.gauge_data_uri) + # print(gauge_getter.gauge_data_uri) gauge_getter.init() ret = gauge_getter.sort_gauges_by_state(['1', '2', '3', '4', '5', '6', '10']) expect = {'NSW': ['1', '3'], 'QLD': ['2', '3', '4'], 'VIC': ['4', '5'], 'SA': ['6'], 'rest': ['10']}