|
| 1 | +from pygeotemporal.sensors import SensorsApi |
| 2 | +from pygeotemporal.streams import StreamsApi |
| 3 | +from pygeotemporal.datapoints import DatapointsApi |
| 4 | +from pygeotemporal.cache import CacheApi |
| 5 | +from pygeotemporal.time_transformers import time2utc |
| 6 | +from datetime import date, datetime |
| 7 | +from dateutil.relativedelta import relativedelta |
| 8 | +import smtplib |
| 9 | +from email.mime.text import MIMEText |
| 10 | +import yaml |
| 11 | +import sys |
| 12 | +import copy |
| 13 | +import os |
| 14 | +import csv |
| 15 | +import logging |
| 16 | +import subprocess |
| 17 | +import json |
| 18 | +from pprint import pprint |
| 19 | +import time |
| 20 | + |
| 21 | +timer = time.time() |
| 22 | + |
| 23 | +# Append path to hucfinder as needed |
| 24 | +# TODO add to config.yml |
| 25 | +sys.path.append('../../..') |
| 26 | +from huc_finder.huc import HucFinder |
| 27 | +hucfinder = HucFinder('../../../huc_finder/huc-all.shp') |
| 28 | + |
| 29 | +# get configuration |
| 30 | +config = yaml.load(open("config.yml", 'r'), Loader=yaml.FullLoader) |
| 31 | + |
| 32 | +# configure logging |
| 33 | +logging.getLogger("requests").setLevel(logging.WARNING) |
| 34 | +logging.getLogger("urllib3").setLevel(logging.WARNING) |
| 35 | +logging.getLogger("fiona").setLevel(logging.WARNING) |
| 36 | +logging.getLogger("geopandas").setLevel(logging.ERROR) |
| 37 | +logging.getLogger("pyproj").setLevel(logging.ERROR) |
| 38 | +logging.basicConfig(filename=config['log_filename'], level=logging.DEBUG) |
| 39 | + |
| 40 | +host = config['host'] |
| 41 | +username = config['username'] |
| 42 | +password = config['password'] |
| 43 | + |
| 44 | +logging.info(" ") |
| 45 | +logging.info("[greon.py] Starting GREON Parser for %s" % host) |
| 46 | + |
| 47 | +binning_sensor_id_list = [] |
| 48 | + |
| 49 | +# Authenticate |
| 50 | +sensorclient = SensorsApi(host=host, username=username, password=password) |
| 51 | +streamclient = StreamsApi(host=host, username=username, password=password) |
| 52 | +datapointclient = DatapointsApi(host=host, username=username, password=password) |
| 53 | +cacheclient = CacheApi(host=host, username=username, password=password) |
| 54 | + |
| 55 | +def create_sensor_stream(site): |
| 56 | + # get or create sensor |
| 57 | + geocode = config['geocodes'][site] |
| 58 | + huc_json = hucfinder.getHuc(lat=geocode[1], lon=geocode[0]) |
| 59 | + logging.debug("[greon.py] Getting sensor") |
| 60 | + sensor_json = sensorclient.sensor_create_json(site, |
| 61 | + geocode[0], |
| 62 | + geocode[1], |
| 63 | + 0, |
| 64 | + site, |
| 65 | + huc_json['huc4']['code'], |
| 66 | + huc=huc_json, |
| 67 | + network='greon', |
| 68 | + organization_id='greon', |
| 69 | + title="Great Rivers Ecological Observation Network") |
| 70 | + |
| 71 | + sensor = sensorclient.sensor_post_json(sensor_json) |
| 72 | + |
| 73 | + latest_datapoint = sensor['max_end_time'] |
| 74 | + |
| 75 | + logging.debug("[greon.py] Latest datapoint:" + str(latest_datapoint)) |
| 76 | + |
| 77 | + # create or get stream - need to add _WQ to name of sensor for stream |
| 78 | + # then romove before returning |
| 79 | + sensor_name = copy.copy(sensor['name']) |
| 80 | + sensor['name'] = sensor_name + "_WQ" |
| 81 | + sensor['properties']["name"] = sensor_name + "_WQ" |
| 82 | + stream_json = streamclient.stream_create_json_from_sensor(sensor) |
| 83 | + stream_wq = streamclient.stream_post_json(stream_json) |
| 84 | + sensor['name'] = sensor_name |
| 85 | + sensor['properties']['name'] = sensor_name |
| 86 | + |
| 87 | + return sensor, stream_wq, geocode, latest_datapoint |
| 88 | + |
| 89 | +for site in config['gltg-sites']: |
| 90 | + |
| 91 | + logging.debug("[greon.py] ------ Parsing %s" % site) |
| 92 | + |
| 93 | + sensor, stream_wq, geocode, latest_datapoint = create_sensor_stream(site) |
| 94 | + |
| 95 | + # create list of paths for site and data type |
| 96 | + file_list = [] |
| 97 | + |
| 98 | + logging.debug("[greon.py] Getting data files for %s" % site) |
| 99 | + for path, subdirs, files in os.walk("%s/%s%s" % (config['loggernet_dir'], str(site), "/WQData")): |
| 100 | + # don't add files to list that are previous to latest_datapoint |
| 101 | + for name in files: |
| 102 | + if name[-3:] == "dat": |
| 103 | + file_list.append(os.path.join(path, name)) |
| 104 | + logging.debug("[greon.py] Done getting data files for %s" % site) |
| 105 | + |
| 106 | + count = 0 |
| 107 | + greon1_to_greon2 = False |
| 108 | + datapoints_bulk_list = [] |
| 109 | + good_datapoint_count = 0 |
| 110 | + rows_total = 0 |
| 111 | + rows_count = 0 |
| 112 | + |
| 113 | + while len(file_list) > 0: |
| 114 | + file_rows = open(file_list[0]).read().splitlines() |
| 115 | + file_rows.pop(0) |
| 116 | + file_rows.pop(1) |
| 117 | + file_rows.pop(1) |
| 118 | + data = csv.DictReader(file_rows) |
| 119 | + |
| 120 | + for row in data: |
| 121 | + rows_count += 1 |
| 122 | + |
| 123 | + if site == "GREON-01" and row['TIMESTAMP'] > "2017-07-02 02:52:00" and not greon1_to_greon2: |
| 124 | + if greon1_to_greon2 is False: |
| 125 | + logging.debug("[greon.py] Switching GREON-01 to GREON-02") |
| 126 | + sensorclient.sensor_statistics_post(sensor['id']) |
| 127 | + # sensor, stream_wq, geocode = create_sensor_stream("GREON-02") |
| 128 | + sensor, stream_wq, geocode, latest_datapoint = create_sensor_stream("GREON-02") |
| 129 | + greon1_to_greon2 = True |
| 130 | + continue |
| 131 | + |
| 132 | + if latest_datapoint is None or time2utc(row['TIMESTAMP']) > str(latest_datapoint): |
| 133 | + |
| 134 | + count += 1 |
| 135 | + properties = {} |
| 136 | + for key in row.keys(): |
| 137 | + if key in config['map_parameters'] and row[key] not in ['NAN']: |
| 138 | + properties[config['map_parameters'][key]] = float(row[key]) |
| 139 | + if properties: |
| 140 | + |
| 141 | + datapoint_json = { |
| 142 | + 'start_time': time2utc(row['TIMESTAMP']), |
| 143 | + 'end_time': time2utc(row['TIMESTAMP']), |
| 144 | + 'type': 'Feature', |
| 145 | + 'geometry': { |
| 146 | + 'type': "Point", |
| 147 | + 'coordinates': [ |
| 148 | + geocode[0], |
| 149 | + geocode[1], |
| 150 | + 0 |
| 151 | + ] |
| 152 | + }, |
| 153 | + 'stream_id': stream_wq['id'], |
| 154 | + 'sensor_id': sensor['id'], |
| 155 | + 'sensor_name': str(stream_wq['name']), |
| 156 | + 'properties': properties |
| 157 | + } |
| 158 | + |
| 159 | + good_datapoint_count += 1 |
| 160 | + |
| 161 | + datapoints_bulk_list.append(datapoint_json) |
| 162 | + |
| 163 | + if good_datapoint_count % 100 == 0 or len(file_list) == 1: |
| 164 | + logging.debug("[greon.py] Posting datapoints up to: " + str(count)+" of sensor " + str(site)) |
| 165 | + |
| 166 | + response = datapointclient.datapoint_create_bulk(datapoints_bulk_list) |
| 167 | + logging.debug("[greon.py] %s" % response) |
| 168 | + |
| 169 | + |
| 170 | + datapoints_bulk_list = [] |
| 171 | + if str(sensor['id']) not in binning_sensor_id_list: |
| 172 | + binning_sensor_id_list.append(str(sensor['id'])) |
| 173 | + else: |
| 174 | + logging.debug("[greon.py] Not posting data for row - all NANs ") |
| 175 | + file_list.pop(0) |
| 176 | + |
| 177 | + if len(datapoints_bulk_list) > 0: |
| 178 | + logging.debug("[greon.py] creating datapoints up to (end of sensor loop) of sensor " + str(site)) |
| 179 | + response = datapointclient.datapoint_create_bulk(datapoints_bulk_list) |
| 180 | + logging.debug(response) |
| 181 | + datapoints_bulk_list = [] |
| 182 | + if str(sensor['id']) not in binning_sensor_id_list: |
| 183 | + binning_sensor_id_list.append(str(sensor['id'])) |
| 184 | + |
| 185 | + logging.debug("[greon.py] updating sensor with id " + str(sensor['id'])) |
| 186 | + sensorclient.sensor_statistics_post(sensor['id']) |
| 187 | + |
| 188 | + sensor_for_update = sensorclient.sensor_get(sensor['id']) #.json()['sensor'] |
| 189 | + |
| 190 | + logging.debug("[greon.py] Setting online status for sensor " + str(sensor['id'])) |
| 191 | + |
| 192 | + if sensor_for_update["max_end_time"][:10] > str(date.today() - relativedelta(days=5)): |
| 193 | + if "online_status" not in sensor_for_update["properties"] or sensor_for_update["properties"][ |
| 194 | + "online_status"] != "online": |
| 195 | + logging.debug("changing online status to online for " + str(site)) |
| 196 | + sensor_for_update["properties"]["online_status"] = "online" |
| 197 | + response = sensorclient.update_sensor_metadata(sensor_for_update) |
| 198 | + |
| 199 | + else: |
| 200 | + if "online_status" not in sensor_for_update["properties"] or sensor_for_update["properties"][ |
| 201 | + "online_status"] != "offline": |
| 202 | + logging.debug("changing online status to offline for " + str(site)) |
| 203 | + sensor_for_update["properties"]["online_status"] = "offline" |
| 204 | + response = sensorclient.update_sensor_metadata(sensor_for_update) |
| 205 | + |
| 206 | + logging.debug("[greon.py] Done with sensor " + str(sensor['id'])) |
| 207 | + |
| 208 | +if len(binning_sensor_id_list) > 0: |
| 209 | + logging.info("[greon.py] Start binning") |
| 210 | + subprocess.Popen( |
| 211 | + ["python", config['binning_path'], |
| 212 | + "--host", host, |
| 213 | + "-u", username, |
| 214 | + "-p", password, |
| 215 | + "-s", " ".join(binning_sensor_id_list), |
| 216 | + "-l", config['log_filename'], |
| 217 | + "-w", str(config['binning_wait']), |
| 218 | + "-n", str(config['n_binning_threads']) |
| 219 | + ] |
| 220 | + ).wait() |
| 221 | +else: |
| 222 | + logging.info("[greon.py] There are no new datapoints to parser - skipping binning") |
| 223 | + |
| 224 | +logging.debug("[greon.py] Done Parsing GREON. Total time %sm" % ((time.time()-timer)/60)) |
| 225 | +logging.debug(" ") |
0 commit comments