From 6c5d68b938d91da6e3491cea26ed1c96fa6eed2b Mon Sep 17 00:00:00 2001 From: Miguel Leon Date: Thu, 1 Nov 2018 14:20:12 -0400 Subject: [PATCH 1/8] Update wof/examples/flask/odm2/measurement/odm2_measurement_dao.py DAO changes for ANSP --- .../odm2/measurement/odm2_measurement_dao.py | 45 +++++++++++++------ 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/wof/examples/flask/odm2/measurement/odm2_measurement_dao.py b/wof/examples/flask/odm2/measurement/odm2_measurement_dao.py index 0dd20f4..ed6bc3f 100644 --- a/wof/examples/flask/odm2/measurement/odm2_measurement_dao.py +++ b/wof/examples/flask/odm2/measurement/odm2_measurement_dao.py @@ -32,18 +32,24 @@ def __init__(self, db_connection_string): def __del__(self): self.db_session.close() - +# def get_all_sites(self): s_Arr = [] - s_rArr = self.db_session.query(odm2_models.MeasurementResults,odm2_models.Sites).\ - distinct(odm2_models.Sites.SamplingFeatureID).\ + # s_rArr = self.db_session.query(odm2_models.MeasurementResults,odm2_models.Sites).\ + # distinct(odm2_models.Sites.SamplingFeatureID).\ + # join(odm2_models.FeatureActions).\ + # join(odm2_models.Specimens).\ + # filter(odm2_models.Specimens.SamplingFeatureID == odm2_models.RelatedFeatures.SamplingFeatureID, + # odm2_models.MeasurementResults.FeatureActionID == odm2_models.FeatureActions.FeatureActionID, + # odm2_models.Sites.SamplingFeatureID == odm2_models.RelatedFeatures.RelatedFeatureID).all() + s_rArr = self.db_session.query(odm2_models.Sites).\ join(odm2_models.FeatureActions).\ - join(odm2_models.Specimens).\ - filter(odm2_models.Specimens.SamplingFeatureID == odm2_models.RelatedFeatures.SamplingFeatureID, - odm2_models.MeasurementResults.FeatureActionID == odm2_models.FeatureActions.FeatureActionID, - odm2_models.Sites.SamplingFeatureID == odm2_models.RelatedFeatures.RelatedFeatureID).all() + join(odm2_models.MeasurementResults).\ + filter(odm2_models.FeatureActions.SamplingFeatureID == odm2_models.Sites.SamplingFeatureID, + odm2_models.MeasurementResults.FeatureActionID == odm2_models.FeatureActions.FeatureActionID).distinct() + # print('site count: ' + str(len(s_rArr))) for s_r in s_rArr: - s = model.Site(s_r.Sites) + s = model.Site(s_r) s_Arr.append(s) return s_Arr @@ -275,7 +281,7 @@ def get_specimen_data(self): q = self.db_session.query(odm2_models.MeasurementResultValues).\ join(odm2_models.MeasurementResults).\ join(odm2_models.FeatureActions).\ - join(odm2_models.Specimens).\ + join(odm2_models.SamplingFeatures).\ join(odm2_models.Variables) return q @@ -289,11 +295,22 @@ def get_datavalues(self, site_code, var_code, begin_date_time=None, if (not begin_date_time or not end_date_time): try: + # print('HERE HERE') q = self.get_specimen_data() - q = q.filter(odm2_models.Specimens.SamplingFeatureID == odm2_models.RelatedFeatures.SamplingFeatureID, - odm2_models.MeasurementResults.FeatureActionID == odm2_models.FeatureActions.FeatureActionID, - odm2_models.RelatedFeatures.RelatedFeatureID == site.SiteID, - odm2_models.Variables.VariableCode == var_code) + # print(len(q.all())) + # print('all spcimen values') + q = q.filter( + odm2_models.MeasurementResults.FeatureActionID == odm2_models.FeatureActions.FeatureActionID, # noqa + odm2_models.MeasurementResults.VariableID == odm2_models.Variables.VariableID, + odm2_models.SamplingFeatures.SamplingFeatureCode == site_code, + odm2_models.Variables.VariableCode == var_code). \ + order_by(odm2_models.MeasurementResultValues.ValueDateTime) + # print(len(q.all())) + # print('number of values') + # q.filter(odm2_models.Specimens.SamplingFeatureID == odm2_models.RelatedFeatures.SamplingFeatureID, + # odm2_models.MeasurementResults.FeatureActionID == odm2_models.FeatureActions.FeatureActionID, + # odm2_models.RelatedFeatures.RelatedFeatureID == site.SiteID, + # odm2_models.Variables.VariableCode == var_code) if unitid is not None: q = q.filter(odm2_models.MeasurementResults.UnitsID == int(unitid)) if samplemedium is not None: @@ -308,7 +325,7 @@ def get_datavalues(self, site_code, var_code, begin_date_time=None, q = self.get_specimen_data() q = q.filter(odm2_models.Specimens.SamplingFeatureID == odm2_models.RelatedFeatures.SamplingFeatureID, odm2_models.MeasurementResults.FeatureActionID == odm2_models.FeatureActions.FeatureActionID, - odm2_models.RelatedFeatures.RelatedFeatureID == site.SiteID, + # odm2_models.RelatedFeatures.RelatedFeatureID == site.SiteID, odm2_models.Variables.VariableCode == var_code, odm2_models.MeasurementResultValues.ValueDateTime >= begin_date_time, odm2_models.MeasurementResultValues.ValueDateTime <= end_date_time) From c2a885d84be5869c1d259a462cad00210ada18b9 Mon Sep 17 00:00:00 2001 From: Miguel Leon Date: Fri, 2 Nov 2018 09:51:54 -0400 Subject: [PATCH 2/8] changes to DAO for CUAHSI Hydroclient compatability changes to DAO for CUAHSI Hydroclient compatability --- .../odm2/measurement/odm2_measurement_dao.py | 21 +++++++++++++++++-- .../odm2/measurement/sqlalch_odm2_models.py | 9 +++++++- 2 files changed, 27 insertions(+), 3 deletions(-) diff --git a/wof/examples/flask/odm2/measurement/odm2_measurement_dao.py b/wof/examples/flask/odm2/measurement/odm2_measurement_dao.py index ed6bc3f..26530b2 100644 --- a/wof/examples/flask/odm2/measurement/odm2_measurement_dao.py +++ b/wof/examples/flask/odm2/measurement/odm2_measurement_dao.py @@ -1,4 +1,6 @@ from __future__ import (absolute_import, division, print_function) +import os +import yaml from datetime import datetime from sqlalchemy import create_engine, distinct, func @@ -29,10 +31,17 @@ def __init__(self, db_connection_string): self.db_session = scoped_session(sessionmaker( autocommit=False, autoflush=False, bind=self.engine)) #odm2_models.Base.query = self.db_session.query_property() - + # Read in WaterML -> ODM2 CV Mapping + with open(os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'cvmap_wml_1_1.yml'))) as yml: + self.yml_dict = yaml.load(yml) def __del__(self): self.db_session.close() # + def get_match(self, cvkey, term): + for k, v in self.yml_dict[cvkey].items(): + if term in v: + return k + return term def get_all_sites(self): s_Arr = [] # s_rArr = self.db_session.query(odm2_models.MeasurementResults,odm2_models.Sites).\ @@ -163,7 +172,15 @@ def get_variables_from_results(self,var_codes=None): s = result.SampledMediumCV t = result.TimeAggregationIntervalUnitsObj ti = result.TimeAggregationInterval - w_v = model.Variable(v,s,u,t,ti) + ag = result.AggregationStatisticCV + print('ag') + print(ag) + at = result.FeatureActionObj.ActionObj.ActionTypeCV + w_v = model.Variable(v, s, u, t, ti, ag, at) + # w_v = model.Variable(v,s,u,t,ti) + # w_v.AggregationStatisticCV = self.get_match('datatype', w_v.DataType) + w_v.DataType = self.get_match('datatype', w_v.DataType) + w_v.SampleMedium = self.get_match('samplemedium', w_v.SampleMedium) v_arr.append(w_v) return v_arr diff --git a/wof/examples/flask/odm2/measurement/sqlalch_odm2_models.py b/wof/examples/flask/odm2/measurement/sqlalch_odm2_models.py index 1ec1678..09cba0c 100644 --- a/wof/examples/flask/odm2/measurement/sqlalch_odm2_models.py +++ b/wof/examples/flask/odm2/measurement/sqlalch_odm2_models.py @@ -5,7 +5,8 @@ class Variable(wof_base.BaseVariable): def __init__(self, v=None, - VarSampleMedium=None, v_unit=None, v_tunit=None, v_timeinterval=None): + VarSampleMedium=None, v_unit=None, v_tunit=None, v_timeinterval=None, + aggregationstatisticCV=None, actiontypeCV=None): variable_key = '%s::%s-%s' % (v.VariableCode, v_unit.UnitsID, VarSampleMedium) @@ -18,6 +19,12 @@ def __init__(self, v=None, self.DataType = v.VariableTypeCV self.Speciation = v.SpeciationCV self.VariableUnitsID = v_unit.UnitsID + self.DataType = aggregationstatisticCV + self.ValueType = actiontypeCV + self.GeneralCategory = v.VariableTypeCV + + self.GeneralCategoryValidate = False + self.ValueTypeValidate = False if v_unit is not None: self.VariableUnits = Unit(v_unit) else: From f267ca76a8ece7c1d51f11d00b65b5f3059df172 Mon Sep 17 00:00:00 2001 From: Miguel Leon Date: Mon, 5 Nov 2018 18:27:32 -0500 Subject: [PATCH 3/8] fixes measurement DAO updates --- wof/examples/flask/odm2/measurement/core.py | 449 +++++++++ .../flask/odm2/measurement/core_1_0.py | 891 +++++++++++++++++ .../flask/odm2/measurement/core_1_1.py | 945 ++++++++++++++++++ wof/examples/flask/odm2/measurement/dao.py | 129 +++ .../odm2/measurement/odm2_measurement_dao.py | 117 ++- .../measurement/odm2_measurement_dao_off.py | 413 ++++++++ .../flask/odm2/measurement/spyned_1_1.py | 384 +++++++ .../odm2/measurement/sqlalch_odm2_models.py | 49 +- .../odm2/timeseries/odm2_timeseries_dao.py | 4 +- 9 files changed, 3320 insertions(+), 61 deletions(-) create mode 100644 wof/examples/flask/odm2/measurement/core.py create mode 100644 wof/examples/flask/odm2/measurement/core_1_0.py create mode 100644 wof/examples/flask/odm2/measurement/core_1_1.py create mode 100644 wof/examples/flask/odm2/measurement/dao.py create mode 100644 wof/examples/flask/odm2/measurement/odm2_measurement_dao_off.py create mode 100644 wof/examples/flask/odm2/measurement/spyned_1_1.py diff --git a/wof/examples/flask/odm2/measurement/core.py b/wof/examples/flask/odm2/measurement/core.py new file mode 100644 index 0000000..2945add --- /dev/null +++ b/wof/examples/flask/odm2/measurement/core.py @@ -0,0 +1,449 @@ +from __future__ import (absolute_import, division, print_function) + +import cgi +import configparser +import datetime +import logging +import os +import urllib + +from dateutil.parser import parse + +from lxml import etree +from lxml.etree import XMLParser +from lxml.etree import XMLSyntaxError + +import pytz + +from spyne.application import Application +from spyne.const.http import HTTP_405 +from spyne.error import RequestNotAllowed +from spyne.model.fault import Fault +from spyne.model.primitive import AnyXml, Unicode +from spyne.protocol.http import HttpRpc +from spyne.protocol.soap import Soap11 +from spyne.protocol.soap.mime import collapse_swa +from spyne.protocol.xml import XmlDocument +from spyne.server.http import HttpTransportContext +from spyne.server.wsgi import WsgiApplication + +from wof.WofWsdls import WofWSDL_1_0, WofWSDL_1_1 +from wof.apps.spyned_1_0 import TWOFService as wml10 +from wof.apps.spyned_1_1 import TWOFService as wml11 +from wof.apps.waterml2 import TWOFService as wml2 + + +logging.getLogger('werkzeug').setLevel(logging.CRITICAL) +logging.basicConfig(level=logging.ERROR) +# logging.basicConfig(filename='/home/azureadmin/test/log.log', filemore='w', level=logging.DEBUG) +logging.getLogger('spyne.model.complex').setLevel(logging.ERROR) +logging.getLogger('spyne.interface._base').setLevel(logging.ERROR) +logging.getLogger('spyne.util.appreg').setLevel(logging.ERROR) +logging.getLogger('spyne.interface.xml_schema').setLevel(logging.ERROR) +logging.getLogger('spyne.protocol.dictdoc.simple').setLevel(logging.ERROR) +logger = logging.getLogger(__name__) +logger_invalid = logging.getLogger(__name__ + ".invalid") +# print(logging.getLogger(__name__)) +try: + from wof import __version__ + version = __version__ +except ImportError: + version = 'dev' + +_root_dir = os.path.abspath(os.path.dirname(__file__)) +_TEMPLATES = os.path.join(_root_dir, 'flask', 'templates') + +_SERVICE_PARAMS = { + "r_type": "rest", + "s_type": "soap", + "wml10_tns": "http://www.cuahsi.org/his/1.0/ws/", + "wml11_tns": "http://www.cuahsi.org/his/1.1/ws/", + "wml10_rest_name": "WaterOneFlow_rest_1_0", + "wml10_soap_name": "WaterOneFlow_soap_1_0", + "wml11_rest_name": "WaterOneFlow_rest_1_1", + "wml11_soap_name": "WaterOneFlow_soap_1_1", +} +# utc_time_zone = tz(None,0) +UTC_TZ = pytz.utc + + +def site_map(app): + output = [] + for rule in app.url_map.iter_rules(): + # methods = ','.join(rule.methods) + # line = urllib.unquote("{:50s} {:20s} {}".format( + # rule.endpoint, methods, rule) + # ) + line = urllib.unquote("{}".format(rule)) + if '/static/' in line or '/rest/' in line: + continue + output.append(line) + # print("Access Service Path at ") + # for line in sorted(output): + # print(line) + return sorted(output) + + +class WOFConfig(object): + network = 'NETWORK' + vocabulary = 'VOCABULARY' + menu_group_name = 'MENU_GROUP_NAME' + service_wsdl = 'SERVICE_WSDL' + timezone = None + timezone_abbr = None + + default_site = None + default_variable = None + default_start_date = None + default_end_date = None + default_unitid = None + default_samplemedium = None + + default_west = None + default_south = None + default_north = None + default_east = None + + + organization = 'MYORGANIZATION' + address = '1234 DriveIn' + city = 'Seattle' + state = 'Washington' + zipcode = '98064' + contactname = 'John Smith' + contactemail = 'johnsmith@example.com' + phone = '555-555-555' + link = 'http://www.example.com/' + + def __init__(self, file_name, templates=None): + config = configparser.RawConfigParser() + config.read(file_name) + #print(logging.ERROR) + if config.has_option('WOF_1_1', 'Network'): + self.network = config.get('WOF_1_1', 'Network') + else: + self.network = config.get('WOF', 'Network') + if config.has_option('WOF_1_1', 'Vocabulary'): + self.vocabulary = config.get('WOF_1_1', 'Vocabulary') + else: + self.vocabulary = config.get('WOF', 'Vocabulary') + if config.has_option('WOF_1_1', 'URLPATH'): + self.urlpath = config.get('WOF_1_1', 'URLPATH') + else: + self.urlpath = config.get('WOF', 'URLPATH') + if config.has_option('WOF_1_1', 'Menu_Group_Name'): + self.menu_group_name = config.get('WOF_1_1', 'Menu_Group_Name') + else: + self.menu_group_name = config.get('WOF', 'Menu_Group_Name') + if config.has_option('WOF_1_1', 'Service_WSDL'): + self.service_wsdl = config.get('WOF_1_1', 'Service_WSDL') + print(self.service_wsdl) + else: + self.service_wsdl = config.get('WOF', 'Service_WSDL') + if config.has_option('WOF_1_1', 'Timezone'): + self.timezone = config.get('WOF_1_1', 'Timezone') + + else: + self.timezone = config.get('WOF', 'Timezone') + if config.has_option('WOF_1_1', 'TimezoneAbbreviation'): + self.timezone_abbr = config.get('WOF_1_1', + 'TimezoneAbbreviation') + else: + self.timezone_abbr = config.get('WOF', 'TimezoneAbbreviation') + + if config.has_section('Default_Params'): + if config.has_option('Default_Params', 'UnitID'): + self.default_unitid = config.get('Default_Params', + 'UnitID') + if config.has_option('Default_Params', 'SampleMedium'): + self.default_samplemedium = config.get('Default_Params', + 'SampleMedium') + + if config.has_option('Default_Params_1_1', 'Site'): + self.default_site = config.get('Default_Params_1_1', + 'Site') + else: + self.default_site = config.get('Default_Params', 'Site') + if config.has_option('Default_Params_1_1', 'Variable'): + self.default_variable = config.get('Default_Params_1_1', + 'Variable') + else: + self.default_variable = config.get('Default_Params', + 'Variable') + if config.has_option('Default_Params_1_1', 'StartDate'): + self.default_start_date = config.get('Default_Params_1_1', + 'StartDate') + else: + self.default_start_date = config.get('Default_Params', + 'StartDate') + if config.has_option('Default_Params_1_1', 'EndDate'): + self.default_end_date = config.get('Default_Params_1_1', + 'EndDate') + else: + self.default_end_date = config.get('Default_Params', + 'EndDate') + if config.has_option('Default_Params_1_1', 'East'): + self.default_east = config.get('Default_Params_1_1', + 'East') + else: + if config.has_option('Default_Params', 'East'): + self.default_east = config.get('Default_Params', + 'East') + else: + self.default_east = 180 + if config.has_option('Default_Params_1_1', 'North'): + self.default_north = config.get('Default_Params_1_1', + 'North') + else: + if config.has_option('Default_Params', 'North'): + self.default_north = config.get('Default_Params', + 'North') + else: + self.default_north = 90 + if config.has_option('Default_Params_1_1', 'South'): + self.default_south = config.get('Default_Params_1_1', + 'South') + else: + if config.has_option('Default_Params', 'South'): + self.default_south = config.get('Default_Params', + 'South') + else: + self.default_south = -90 + if config.has_option('Default_Params_1_1', 'West'): + self.default_west = config.get('Default_Params_1_1', + 'West') + else: + if config.has_option('Default_Params', 'West'): + self.default_west = config.get('Default_Params', + 'West') + else: + self.default_west = -180 + + if config.has_section('Contact'): + if config.has_option('Contact', 'Organization'): + self.organization = config.get('Contact', 'Organization') + if config.has_option('Contact', 'Address'): + self.address = config.get('Contact', 'Address') + if config.has_option('Contact', 'City'): + self.city = config.get('Contact', 'City') + if config.has_option('Contact', 'State'): + self.state = config.get('Contact', 'State') + if config.has_option('Contact', 'ZipCode'): + self.zipcode = config.get('Contact', 'ZipCode') + if config.has_option('Contact', 'Name'): + self.contactname = config.get('Contact', 'Name') + if config.has_option('Contact', 'Email'): + self.contactemail = config.get('Contact', 'Email') + if config.has_option('Contact', 'Phone'): + self.phone = config.get('Contact', 'Phone') + if config.has_option('Contact', 'Link'): + self.link = config.get('Contact', 'Link') + + if templates is not None: + self.TEMPLATES = templates + elif config.has_option('WOFPY', 'Templates'): + self.TEMPLATES = config.get('WOFPY', 'Templates') + else: + self.TEMPLATES = _TEMPLATES + + +class wofSoap11(Soap11): + def _wof_parse_xml_string(self, xml_string, parser, charset=None): + string = b''.join(xml_string) + if charset: + # string = string.decode(charset) + string = string.decode('utf-8-sig') # Remove BOM. + string = string.encode(charset) # Back to original encoding type. + try: + try: + root, xmlids = etree.XMLID(string, parser) + except ValueError as e: + logging.debug( + 'ValueError: de-serializing from unicode strings with ' + 'encoding declaration is not supported by lxml.' + ) + root, xmlids = etree.XMLID(string.encode(charset), parser) + + except XMLSyntaxError as e: + logger_invalid.error("%r in string %r", e, string) + raise Fault('Client.XMLSyntaxError', str(e)) + + return root, xmlids + + def create_in_document(self, ctx, charset=None): + if isinstance(ctx.transport, HttpTransportContext): + # according to the soap via http standard, soap requests must only + # work with proper POST requests. + content_type = ctx.transport.get_request_content_type() + http_verb = ctx.transport.get_request_method() + if content_type is None or http_verb != "POST": + ctx.transport.resp_code = HTTP_405 + raise RequestNotAllowed( + "You must issue a POST request with the Content-Type " + "header properly set.") + + content_type = cgi.parse_header(content_type) + collapse_swa(content_type, ctx.in_string) + + ctx.in_document = self._wof_parse_xml_string( + ctx.in_string, + XMLParser(**self.parser_kwargs), + charset + ) + + +""" returns an array of the applications """ + + +def getSpyneApplications(wof_obj_1_0, wof_obj_1_1, templates=None): + + # wof_obj_1_0 = wof_1_0.WOF(dao, config_file) + # wof_obj_1_1 = wof_1_1.WOF_1_1(dao,config_file) + print(wof_obj_1_0.urlpath) + sensorNetwork = wof_obj_1_0.urlpath.replace('/', '').lower() + + soap_app_1_0 = Application( + [wml10(wof_obj_1_0, Unicode, _SERVICE_PARAMS["s_type"])], + tns=_SERVICE_PARAMS["wml10_tns"], + name=sensorNetwork+'_svc_' + _SERVICE_PARAMS["wml10_soap_name"], + in_protocol=wofSoap11(validator='lxml'), + out_protocol=Soap11(), + ) + + rest_app_1_0 = Application( + [wml10(wof_obj_1_0, AnyXml, _SERVICE_PARAMS["r_type"])], + tns=_SERVICE_PARAMS["wml10_tns"], + name=sensorNetwork+'_svc_' + _SERVICE_PARAMS["wml10_rest_name"], + in_protocol=HttpRpc(validator='soft'), + out_protocol=XmlDocument(), + ) + + soap_app_1_1 = Application( + [wml11(wof_obj_1_1, Unicode, _SERVICE_PARAMS["s_type"])], + tns=_SERVICE_PARAMS["wml11_tns"], + name=sensorNetwork+'_svc_' + _SERVICE_PARAMS["wml11_soap_name"], + in_protocol=wofSoap11(validator='lxml'), + out_protocol=Soap11(), + ) + + rest_app_1_1 = Application( + [wml11(wof_obj_1_1, AnyXml, _SERVICE_PARAMS["r_type"])], + tns=_SERVICE_PARAMS["wml11_tns"], + name=sensorNetwork + '_svc_' + _SERVICE_PARAMS["wml11_rest_name"], + in_protocol=HttpRpc(validator='soft'), + out_protocol=XmlDocument(), + ) + # need to update template to 1_1 object. + # + # File "\lib\site-packages\jinja2\environment.py", line 408, in getattr + # return getattr(obj, attribute) + # UndefinedError: 'method_result' is undefined + rest_app_2 = Application( + [wml2(wof_obj_1_0, Unicode, _SERVICE_PARAMS["r_type"])], + tns=_SERVICE_PARAMS["wml11_tns"], + name=sensorNetwork + '_svc_' + _SERVICE_PARAMS["wml11_rest_name"], + in_protocol=HttpRpc(validator='soft'), + # out_protocol=XmlDocument(), + out_protocol=HttpRpc(mime_type='text/xml'), + + ) + + rest_wsgi_wrapper_1_0 = WsgiApplication(rest_app_1_0) + soap_wsgi_wrapper_1_0 = WsgiApplication(soap_app_1_0) + rest_wsgi_wrapper_1_1 = WsgiApplication(rest_app_1_1) + soap_wsgi_wrapper_1_1 = WsgiApplication(soap_app_1_1) + rest_wsgi_wrapper_2_0 = WsgiApplication(rest_app_2) + + spyneApps = { + '/' + sensorNetwork+'/rest/1_0': rest_wsgi_wrapper_1_0, + '/' + sensorNetwork+'/rest/1_1': rest_wsgi_wrapper_1_1, + '/' + sensorNetwork+'/soap/cuahsi_1_0': soap_wsgi_wrapper_1_0, + '/' + sensorNetwork+'/soap/cuahsi_1_1': soap_wsgi_wrapper_1_1, + '/' + sensorNetwork+'/rest/2': rest_wsgi_wrapper_2_0, + } + + templatesPath = None + if templates is None: + if wof_obj_1_1._config is not None: + templatesPath = os.path.abspath(wof_obj_1_1._config.TEMPLATES) + else: + templatesPath = os.path.abspath(templates) + + if templatesPath: + print(templatesPath) + if not os.path.exists(templatesPath): + logging.info('Templates path: {} NOT exists {}'.format( + templatesPath, + os.path.exists(templatesPath)) + ) + templatesPath = _TEMPLATES + logging.info('default temnplate path: %s' % templatesPath) + # needs to be service_baseURL. in config wof_obj_1_0.service_wsdl + wsdl10 = WofWSDL_1_0( + soap_wsgi_wrapper_1_0.doc.wsdl11.interface, + templates=templatesPath, + network=sensorNetwork, + version=version + ) + + # soap_wsgi_wrapper_1_0._wsdl = wsdl10.build_interface_document('/'+ sensorNetwork+'/soap/wateroneflow',templatesPath) #.get_wsdl_1_0('/'+ sensorNetwork+'/soap/wateroneflow') # noqa + soap_wsgi_wrapper_1_0.event_manager.add_listener( + 'wsdl', + wsdl10.on_get_wsdl_1_0_ + ) + # path: /{sensorNetwork}/soap/wateroneflow_1_1/.wsdl returns the WSDL. + wsdl11 = WofWSDL_1_1( + soap_wsgi_wrapper_1_1.doc.wsdl11.interface, + templates=templatesPath, + network=sensorNetwork, + version=version + ) + # soap_wsgi_wrapper_1_1._wsdl = wsdl11.build_interface_document('/'+ sensorNetwork+'/soap/wateroneflow_1_1',templatesPath) #.get_wsdl_1_0('/'+ sensorNetwork+'/soap/wateroneflow') # noqa + soap_wsgi_wrapper_1_1.event_manager.add_listener( + 'wsdl', + wsdl11.on_get_wsdl_1_1_ + ) + + return spyneApps + + +class wofConfig(object): + dao = None + config = None + configObject = None + + def __init__(self, dao=None, wofConfigFile=None): + self.config = wofConfigFile + self.dao = dao + if (self.config is not None): + try: + print(self.config) + self.configObject = WOFConfig(self.config) + except: + pass + + +def _get_datavalues_datetime(obj, local_datetime_attr, + utc_datetime_attr): + """ + Returns a datetime given an object and the names of the + attributes for local time and utc date time + """ + local_datetime = getattr(obj, local_datetime_attr, None) + if local_datetime: + if type(local_datetime) == datetime.datetime: + if not local_datetime.tzinfo: + raise ValueError("local times must be timezone-aware") + # return local_datetime.isoformat() + return local_datetime + else: + ldt = parse(local_datetime) + return ldt + else: + utc_datetime = getattr(obj, utc_datetime_attr) + if type(utc_datetime) == datetime.datetime: + utcdt = UTC_TZ.localize(utc_datetime) + return utcdt + else: + utcdt = parse(utc_datetime) + return utcdt diff --git a/wof/examples/flask/odm2/measurement/core_1_0.py b/wof/examples/flask/odm2/measurement/core_1_0.py new file mode 100644 index 0000000..e2a719d --- /dev/null +++ b/wof/examples/flask/odm2/measurement/core_1_0.py @@ -0,0 +1,891 @@ +from __future__ import (absolute_import, division, print_function) + +import datetime +import logging + +from xml.sax.saxutils import escape + +from wof import WaterML +from wof import core + + +class WOF(object): + + network = 'NETWORK' + vocabulary = 'VOCABULARY' + menu_group_name = 'MENU_GROUP_NAME' + service_wsdl = 'SERVICE_WSDL' + timezone = None + timezone_abbr = None + + dao = None + + default_site = None + default_variable = None + default_start_date = None + default_end_date = None + default_unitid = None + default_samplemedium = None + + organization = 'MYORGANIZATION' + address = '1234 DriveIn' + city = 'Seattle' + state = 'Washington' + zipcode = '98064' + contactname = 'John Smith' + contactemail = 'johnsmith@example.com' + phone = '555-555-555' + link = 'http://www.example.com/' + + _config = None + _templates = None + + def __init__(self, dao, config_file=None, templates=None): + self.dao = dao + if templates: + self._templates = templates + if config_file: + self.config_from_file(config_file) + + def config_from_file(self, file_name): + if self._templates: + config = core.WOFConfig(file_name, templates=self._templates) + else: + config = core.WOFConfig(file_name) + self._config = config + + self.network = config.network.lower() + self.vocabulary = config.vocabulary.lower() + self.urlpath = config.urlpath.lower() + self.menu_group_name = config.menu_group_name + self.service_wsdl = config.service_wsdl + self.timezone = config.timezone + self.timezone_abbr = config.timezone_abbr + + self.default_site = config.default_site + self.default_variable = config.default_variable + self.default_start_date = config.default_start_date + self.default_end_date = config.default_end_date + self.default_unitid = config.default_unitid + self.default_samplemedium = config.default_samplemedium + + self.organization = config.organization + self.address = config.address + self.city = config.city + self.state = config.state + self.zipcode = config.zipcode + self.contactname = config.contactname + self.contactemail = config.contactemail + self.phone = config.phone + self.link = config.link + + ''' + For WML 1.0 many terms were embedded in the Schemas, + which made them not expandable. This provides a standard warning message + ''' + def invalid_enum_message(self, name, value, default=None): + message = ('WaterML 1.0 schema enum issue "{}" not in {}. ' + 'Use the cuashi_1_1 endpoint ').format(value, name) + if default is not None: + message = 'Substituted "' + default + '". ' + message + return message + + def get_site_code(self, siteArg): + + if ':' in siteArg: + networkname, siteCode = siteArg.split(':', 1) + networkname = networkname.lower() + if self.network == networkname: + return siteCode + else: + return None + return siteArg + + def get_variable_code(self, varArg): + + if ':' in varArg: + vocabname, varCode = varArg.split(':', 1) + vocabname = vocabname.lower() + if self.vocabulary == vocabname: + return varCode + else: + return None + return varArg + + def create_get_site_response(self, siteArg=None): + print('here create_get_site_response') + if siteArg is None or siteArg == '': + siteResultArr = self.dao.get_all_sites() + else: + siteCodesArr = siteArg.split(',') + siteCodesArr = [self.get_site_code(s) + for s in siteCodesArr] + siteResultArr = self.dao.get_sites_by_codes(siteCodesArr) + + # if len(siteResultArr) == 0: + # return None + + siteInfoResponse = WaterML.SiteInfoResponseType() + + queryInfo = WaterML.QueryInfoType() + # TODO: check on how this should be done for multiple sites. + criteria = WaterML.criteria(locationParam=siteArg) + queryInfo.set_criteria(criteria) + queryInfoNote = WaterML.NoteType() + queryInfo.add_note(queryInfoNote) + queryInfo.set_extension('') + siteInfoResponse.set_queryInfo(queryInfo) + + if siteResultArr: + for siteResult in siteResultArr: + s = self.create_site_element(siteResult) + siteInfoResponse.add_site(s) + else: + # site = WaterML.site() + # siteInfoResponse.add_site(site) + raise Exception("Site,'%s', Not Found" % siteArg) + + return siteInfoResponse + + def create_get_site_info_response(self, siteArg, varArg=None): + if siteArg is None: + raise Exception("Site Not Found") + siteCode = self.get_site_code(siteArg) + siteResult = self.dao.get_site_by_code(siteCode) + + if (varArg is None or varArg == ''): + seriesResultArr = self.dao.get_series_by_sitecode(siteCode) + else: + varCode = self.get_variable_code(varArg) + seriesResultArr = self.dao.get_series_by_sitecode_and_varcode( + siteCode, varCode) + + if seriesResultArr is None or len(seriesResultArr) == 0: + raise Exception("Site,'%s', Not Found" % siteArg) + + siteInfoResponse = WaterML.SiteInfoResponseType() + + queryInfo = WaterML.QueryInfoType() + criteria = WaterML.criteria(locationParam=siteArg, + variableParam=varArg) + queryInfo.set_criteria(criteria) + queryInfoNote = WaterML.NoteType() + queryInfo.add_note(queryInfoNote) + queryInfo.set_extension('') + siteInfoResponse.set_queryInfo(queryInfo) + + if siteResult: + s = self.create_site_element(siteResult, seriesResultArr) + siteInfoResponse.add_site(s) + else: + # site = WaterML.site() + # siteInfoResponse.add_site(site) + raise Exception("Site,'%s', Not Found" % siteArg) + + return siteInfoResponse + + def create_get_variable_info_response(self, varArg=None): + + if (varArg is None or varArg == ''): + variableResultArr = self.dao.get_all_variables() + else: + varCodesArr = varArg.split(',') + varCodesArr = [self.get_variable_code(v) + for v in varCodesArr] + variableResultArr = self.dao.get_variables_by_codes(varCodesArr) + + if not variableResultArr: + raise Exception("Variable,'%s', Not Found" % varArg) + + variableInfoResponse = WaterML.VariablesResponseType() + + # TODO: Should queryInfo be in thois response? Suds doesn't + # like when it is. If it should be in the response, then the + # WSDL needs to be updated + + # queryInfo = WaterML.QueryInfoType() + # criteria = WaterML.criteria(variableParam=varArg) + # queryInfo.set_criteria(criteria) + # queryInfoNote = WaterML.NoteType() + # queryInfo.add_note(queryInfoNote) + # queryInfo.set_extension('') + # variableInfoResponse.set_queryInfo(queryInfo) + + variables = WaterML.variables() + for variableResult in variableResultArr: + v = self.create_variable_element(variableResult) + variables.add_variable(v) + variableInfoResponse.set_variables(variables) + return variableInfoResponse + + def create_get_values_response(self, siteArg, varArg, startDateTime=None, + endDateTime=None): + + # TODO: Tim thinks the DAO should handle network and vocab parsing, + # not WOF + + siteCode = self.get_site_code(siteArg) + varCode = self.get_variable_code(varArg) + + valueResultArr = self.dao.get_datavalues(siteCode, varCode, + startDateTime, endDateTime) + + timeSeriesResponse = WaterML.TimeSeriesResponseType() + + queryInfo = WaterML.QueryInfoType() + timeParam = WaterML.timeParam( + beginDateTime=startDateTime, endDateTime=endDateTime) + criteria = WaterML.criteria( + locationParam=siteArg, variableParam=varArg, timeParam=timeParam) + queryInfo.set_criteria(criteria) + queryInfoNote = WaterML.NoteType() + queryInfo.add_note(queryInfoNote) + queryInfo.set_extension('') + timeSeriesResponse.set_queryInfo(queryInfo) + + # if not valueResultArr: + # timeSeries = WaterML.TimeSeriesType() + # timeSeriesResponse.set_timeSeries(timeSeries) + # return timeSeriesResponse + + if not valueResultArr: + raise Exception("Values Not Found for %s:%s for dates %s - %s" % ( + siteCode, varCode, startDateTime, endDateTime)) + + if isinstance(valueResultArr, dict): + for key in valueResultArr.keys(): + valueResultArr = valueResultArr[key] + break + + timeSeries = WaterML.TimeSeriesType() + + # sourceInfo (which is a siteInfo) element + siteResult = self.dao.get_site_by_code(siteCode) + + # TODO: Exception? + if not siteResult: + pass + + sourceInfo = self.create_site_info_element(siteResult) + timeSeries.sourceInfo = sourceInfo + + # variable element + varResult = self.dao.get_variable_by_code(varCode) + + # TODO: Exception? + if not varResult: + pass + + variable = self.create_variable_element(varResult) + timeSeries.variable = variable + + # TODO: fill in some more of the attributes in this element. + values = WaterML.TsValuesSingleVariableType() + + values.count = len(valueResultArr) + + if varResult.VariableUnits: + values.unitsAbbreviation = varResult.VariableUnits.UnitsAbbreviation # noqa + values.unitsCode = varResult.VariableUnits.UnitsID + + # Need to keep track of unique methodIDs and sourceIDs. + methodIdSet = set() + sourceIdSet = set() + qualifierIdSet = set() + offsetTypeIdSet = set() + qualitycontrollevelIdSet = set() + + for valueResult in valueResultArr: + if valueResult.QualityControlLevelID is not None: + qualitycontrollevelIdSet.add(valueResult.QualityControlLevelID) # noqa + qlevelResult = self.dao.get_qualcontrollvl_by_id(valueResult.QualityControlLevelID) # noqa + if hasattr(qlevelResult, 'Definition'): + valueResult.QualityControlLevel = qlevelResult.Definition + # else: + # if hasattr(valueResult,'QualityControlLevel'): + # valueResult.QualityControlLevel = qlevelResult.QualityControlLevelCode # noqa + v = self.create_value_element(valueResult) + values.add_value(v) + + if valueResult.MethodID is not None: + methodIdSet.add(valueResult.MethodID) + + if valueResult.SourceID is not None: + sourceIdSet.add(valueResult.SourceID) + + if valueResult.QualifierID is not None: + qualifierIdSet.add(valueResult.QualifierID) + + if valueResult.OffsetTypeID is not None: + offsetTypeIdSet.add(valueResult.OffsetTypeID) + + if valueResult.QualityControlLevelID is not None: + qualitycontrollevelIdSet.add(valueResult.QualityControlLevelID) + + # Add method elements for each unique methodID + if methodIdSet: + methodIdArr = list(methodIdSet) + methodResultArr = self.dao.get_methods_by_ids(methodIdArr) + for methodResult in methodResultArr: + method = self.create_method_element(methodResult) + values.add_method(method) + + # Add source elements for each unique sourceID + if sourceIdSet: + sourceIdArr = list(sourceIdSet) + sourceResultArr = self.dao.get_sources_by_ids(sourceIdArr) + for sourceResult in sourceResultArr: + source = self.create_source_element(sourceResult) + values.add_source(source) + + # Add qualifier elements + if qualifierIdSet: + qualIdArr = list(qualifierIdSet) + qualResultArr = self.dao.get_qualifiers_by_ids(qualIdArr) + for qualifierResult in qualResultArr: + q = WaterML.qualifier( + qualifierID=qualifierResult.QualifierID, + default=None, + network=self.network, + vocabulary=self.vocabulary, + qualifierCode=qualifierResult.QualifierCode) + values.add_qualifier(q) + + # Add offset elements + if offsetTypeIdSet: + offsetTypeIdArr = list(offsetTypeIdSet) + offsetTypeResultArr = self.dao.get_offsettypes_by_ids( + offsetTypeIdArr) + for offsetTypeResult in offsetTypeResultArr: + offset = self.create_offset_element(offsetTypeResult) + values.add_offset(offset) + + # Add qualitycontrollevel elements + if qualitycontrollevelIdSet: + qlevelIdIdArr = list(qualitycontrollevelIdSet) + + try: + qlevelResultArr = self.dao.get_qualcontrollvls_by_ids(qlevelIdIdArr) # noqa + for qlevelResult in qlevelResultArr: + qlevel = self.create_qlevel_element(qlevelResult) + values.add_qualityControlLevel(qlevel) + except: + logging.warn("WofPy: DOA has no get_qualcontrollvls_by_ids method (added for 2.x)") # noqa + for qlevelID in qlevelIdIdArr: + qlevel = WaterML.QualityControlLevelType( + qualityControlLevelID=qlevelID) + values.add_qualityControlLevel(qlevel) + + timeSeries.values = values + timeSeriesResponse.set_timeSeries(timeSeries) + return timeSeriesResponse + + def create_qlevel_element(self, qlevelResult): + # qlevel = WaterML.QualityControlLevelType( + # qualityControlLevelID=qlevelResult.QualityControlLevelID, + # valueOf_=qlevelResult.QualityControlLevelCode) + qcode = WaterML.qualityControlLevel( + qualityControlLevelCode=qlevelResult.QualityControlLevelCode, + qualityControlLevelID=str(qlevelResult.QualityControlLevelID) + ) + return qcode + + def create_offset_element(self, offsetTypeResult): + # TODO: where does offsetIsVertical come from. + # TODO: where does offsetHorizDirectionDegrees come from? + offset = WaterML.OffsetType( + offsetTypeID=offsetTypeResult.OffsetTypeID, + offsetValue=None, + offsetDescription=offsetTypeResult.OffsetDescription, + offsetIsVertical='true', + offsetHorizDirectionDegrees=None) + + if offsetTypeResult.OffsetUnits: + units = WaterML.UnitsType( + UnitID=offsetTypeResult.OffsetUnits.UnitsID, + UnitAbbreviation=offsetTypeResult.OffsetUnits.UnitsAbbreviation, # noqa + UnitName=offsetTypeResult.OffsetUnits.UnitsName, + UnitType=offsetTypeResult.OffsetUnits.UnitsType) + + offset.units = units + + return offset + + def create_method_element(self, methodResult): + method = WaterML.MethodType( + methodID=methodResult.MethodID, + MethodDescription=methodResult.MethodDescription, + MethodLink=methodResult.MethodLink) + + # need at least one MethodLink element to meet WaterML 1.0 + # schema validation + if method.MethodLink is None: + method.MethodLink = '' + return method + + def create_source_element(self, sourceResult): + source = WaterML.SourceType( + sourceID=sourceResult.SourceID, + Organization=sourceResult.Organization, + SourceDescription=sourceResult.SourceDescription, + SourceLink=sourceResult.SourceLink) + + contactInfo = self.create_contact_info_element(sourceResult) + + source.ContactInformation = contactInfo + + if sourceResult.Metadata: + metadata = WaterML.MetaDataType( + TopicCategory=sourceResult.Metadata.TopicCategory, + Title=sourceResult.Metadata.Title, + Abstract=sourceResult.Metadata.Abstract, + ProfileVersion=sourceResult.Metadata.ProfileVersion, + MetadataLink=sourceResult.Metadata.MetadataLink) + + source.Metadata = metadata + + return source + + def create_contact_info_element(self, sourceResult): + + if (sourceResult.Address and + sourceResult.City and + sourceResult.State and + sourceResult.ZipCode): + addressString = ", ".join([sourceResult.Address, + sourceResult.City, + sourceResult.State, + sourceResult.ZipCode]) + contactInfo = WaterML.ContactInformationType( + Email=sourceResult.Email, + ContactName=sourceResult.ContactName, + Phone=sourceResult.Phone, + Address=addressString) + return contactInfo + return None + + def check_censorCode(self, censorCode): + default = "nc" + valueList = [ + "lt", + "gt", + "nc", + "nd", + "pnq", + ] + if (censorCode in valueList): + return censorCode + else: + logging.info(self.invalid_enum_message('censorCode', censorCode)) + return default + + def check_QualityControlLevel(self, QualityControlLevel): + default = "Unknown" + valueList = [ + "Raw data", + "Quality controlled data", + "Derived products", + "Interpreted products", + "Knowledge products", + "Unknown", + ] + if (QualityControlLevel in valueList): + return QualityControlLevel + else: + logging.info(self.invalid_enum_message( + 'QualityControlLevel', + QualityControlLevel) + ) + return default + + # TODO: lots more stuff to fill out here. + def create_value_element(self, valueResult): + adate = core._get_datavalues_datetime( + valueResult, "LocalDateTime", "DateTimeUTC").isoformat() + clean_censorCode = self.check_censorCode(valueResult.CensorCode) + clean_qcl = self.check_QualityControlLevel(valueResult.QualityControlLevel) # noqa + + value = WaterML.ValueSingleVariable( + qualityControlLevel=clean_qcl, + # qualityControlLevel=valueResult.QualityControlLevel, + methodID=valueResult.MethodID, + sourceID=valueResult.SourceID, + # censorCode=valueResult.CensorCode, + censorCode=clean_censorCode, + sampleID=valueResult.SampleID, + offsetTypeID=valueResult.OffsetTypeID, + accuracyStdDev=valueResult.ValueAccuracy, + offsetValue=valueResult.OffsetValue, + dateTime=adate, + qualifiers=valueResult.QualifierID, + valueOf_=str(valueResult.DataValue) + ) + + # TODO: value.offset stuff? Why does value element have all + # this offset stuff. + # offsetTypeResult = valueResult.OffsetType + # if offsetTypeResult != None: + # value.offsetDescription = offsetTypeResult.OffsetDescription + # value.offsetUnitsAbbreviation = offsetTypeResult.OffsetUnits.UnitsAbbreviation # noqa + # value.offsetUnitsCode = offsetTypeResult.OffsetUnits.UnitsID + return value + + def create_site_element(self, siteResult, seriesResultArr=None): + site = WaterML.site() + siteInfo = self.create_site_info_element(siteResult) + + site.set_siteInfo(siteInfo) + + # need at least one note element to meet WaterML 1.0 schema validation + if (not siteResult.County + and not siteResult.State + and not siteResult.Comments): + + siteInfo.add_note(WaterML.NoteType()) + else: + if siteResult.County: + countyNote = WaterML.NoteType(title="County", + valueOf_=siteResult.County) + siteInfo.add_note(countyNote) + + if siteResult.State: + stateNote = WaterML.NoteType(title="State", + valueOf_=siteResult.State) + siteInfo.add_note(stateNote) + + if siteResult.Comments: + commentsNote = WaterML.NoteType( + title="Site Comments", + valueOf_=escape(siteResult.Comments)) + siteInfo.add_note(commentsNote) + + seriesCatalog = WaterML.seriesCatalogType() + if seriesResultArr is not None: + seriesCatalog.menuGroupName = self.menu_group_name + # TODO: Make sure this is set properly in config filename. + seriesCatalog.serviceWsdl = self.service_wsdl + + for seriesResult in seriesResultArr: + series = self.create_series_element(seriesResult) + + seriesCatalog.add_series(series) + + site.add_seriesCatalog(seriesCatalog) + + # need at least one extension element to meet WaterML 1.0 + # schema validation + site.set_extension('') + return site + + def create_site_info_element(self, siteResult): + siteInfo = WaterML.SiteInfoType() + siteInfo.set_siteName(siteResult.SiteName) + + # TODO: agencyName + siteCode = WaterML.siteCode(network=self.network, + siteID=siteResult.SiteID, + valueOf_=siteResult.SiteCode, + agencyName=None, + defaultId=None) + + siteInfo.add_siteCode(siteCode) + + # TODO: Maybe remove this? None of the other WOF services + # return this info probably because it is not that useful + timeZoneInfo = WaterML.timeZoneInfo(siteUsesDaylightSavingsTime=False, + daylightSavingsTimeZone=None) + timeZoneInfo.defaultTimeZone = WaterML.defaultTimeZone( + ZoneOffset=self.timezone, + ZoneAbbreviation=self.timezone_abbr) + + siteInfo.set_timeZoneInfo(timeZoneInfo) + geoLocation = WaterML.geoLocation() + geogLocation = WaterML.LatLonPointType( + srs="EPSG:{0}".format(siteResult.LatLongDatum.SRSID), + latitude=siteResult.Latitude, + longitude=siteResult.Longitude) + geoLocation.set_geogLocation(geogLocation) + + if (siteResult.LocalX and siteResult.LocalY): + localSiteXY = WaterML.localSiteXY() + localSiteXY.projectionInformation = siteResult.LocalProjection.SRSName # noqa + localSiteXY.X = siteResult.LocalX + localSiteXY.Y = siteResult.LocalY + geoLocation.add_localSiteXY(localSiteXY) + + siteInfo.set_geoLocation(geoLocation) + siteInfo.set_verticalDatum(siteResult.VerticalDatum) + + # need at least one extension element to meet WaterML 1.0 + # schema validation + + siteInfo.set_extension('') + + # need at least one altname element to meet WaterML 1.0 schema + # validation + siteInfo.set_altname('') + return siteInfo + + def create_series_element(self, seriesResult): + series = WaterML.series() + + # Variable + variable = self.create_variable_element(seriesResult.Variable) + series.set_variable(variable) + + series.valueCount = WaterML.valueCount( + valueOf_=str(seriesResult.ValueCount)) + +# in WML1_0, these are strings. + beginDateTime = core._get_datavalues_datetime( + seriesResult, "BeginDateTime", "BeginDateTimeUTC").isoformat() + endDateTime = core._get_datavalues_datetime( + seriesResult, "EndDateTime", "EndDateTimeUTC").isoformat() + + # TimeInterval + if beginDateTime is None: + beginDateTime = datetime.datetime.now().isoformat() + if endDateTime is None: + endDateTime = datetime.datetime.now().isoformat() + variableTimeInt = WaterML.TimeIntervalType( + beginDateTime=beginDateTime, + endDateTime=endDateTime) + series.variableTimeInterval = variableTimeInt + + # Method + if seriesResult.Method: + method = self.create_method_element(seriesResult.Method) + series.Method = method + + # Source + if seriesResult.Source: + source = self.create_source_element(seriesResult.Source) + series.Source = source + + # QualityControlLevel + qualityControlLevel = WaterML.QualityControlLevelType( + qualityControlLevelID=seriesResult.QualityControlLevelID, + valueOf_=seriesResult.QualityControlLevelCode) + + series.QualityControlLevel = qualityControlLevel + + return series + + def check_dataTypeEnum(self, datatype): + default = "Unknown" + valueList = [ + "Continuous", + "Instantaneous", + "Cumulative", + "Incremental", + "Average", + "Maximum", + "Minimum", + "Constant Over Interval", + "Categorical", + "Best Easy Systematic Estimator ", + "Unknown", + "Variance", + "Median", + "Mode", + "Best Easy Systematic Estimator", + "Standard Deviation", + "Skewness", + "Equivalent Mean", + "Sporadic", + "Unknown", + ] + if datatype is None: + logging.warn('Datatype is not specified') + return default + if (datatype in valueList): + return datatype + else: + logging.info(self.invalid_enum_message('datatype', datatype)) + return default + + def check_UnitsType(self, UnitsType): + default = "Dimensionless" + valueList = [ + "Angle", + "Area", + "Dimensionless", + "Energy", + "Energy Flux", + "Flow", + "Force", + "Frequency", + "Length", + "Light", + "Mass", + "Permeability", + "Power", + "Pressure/Stress", + "Resolution", + "Scale", + "Temperature", + "Time", + "Velocity", + "Volume", + ] + if UnitsType is None: + logging.warn('UnitsType is not specified ') + return default + if (UnitsType in valueList): + return UnitsType + else: + logging.info( + self.invalid_enum_message( + 'UnitsType', + UnitsType, + default=default + ) + ) + return default + + def check_SampleMedium(self, SampleMedium): + default = "Unknown" + valueList = [ + "Surface Water", + "Ground Water", + "Sediment", + "Soil", + "Air", + "Tissue", + "Precipitation", + "Unknown", + "Other", + "Snow", + "Not Relevant", + ] + if SampleMedium is None: + logging.warn('SampleMedium is not specified') + return default + if (SampleMedium in valueList): + return SampleMedium + else: + logging.info( + self.invalid_enum_message( + 'SampleMedium', + SampleMedium) + ) + return default + + def check_generalCategory(self, generalCategory): + default = "Unknown" + valueList = [ + "Water Quality", + "Climate", + "Hydrology", + "Geology", + "Biota", + "Unknown", + "Instrumentation", + ] + if generalCategory is None: + logging.warn('GeneralCategory is not specified') + return default + if (generalCategory in valueList): + return generalCategory + else: + logging.info( + self.invalid_enum_message( + 'generalCategory', + generalCategory + ) + ) + return default + + def check_valueType(self, valueType): + default = "Unknown" + valueList = [ + "Field Observation", + "Sample", + "Model Simulation Result", + "Derived Value", + "Unknown", + ] + if valueType is None: + logging.warn('ValueType is not specified') + return default + if (valueType in valueList): + return valueType + else: + logging.info(self.invalid_enum_message('valueType', valueType)) + return default + + def create_variable_element(self, variableResult): + clean_datatype = self.check_dataTypeEnum(variableResult.DataType) + clean_medium = self.check_SampleMedium(variableResult.SampleMedium) + clean_category = self.check_generalCategory(variableResult.GeneralCategory) # noqa + clean_valuetype = self.check_valueType(variableResult.ValueType) + variable = WaterML.VariableInfoType( + variableName=variableResult.VariableName, + # valueType=variableResult.ValueType, + valueType=clean_valuetype, + # dataType=variableResult.DataType, + dataType=clean_datatype, + # generalCategory=variableResult.GeneralCategory, + generalCategory=clean_category, + # sampleMedium=variableResult.SampleMedium, + sampleMedium=clean_medium, + NoDataValue=variableResult.NoDataValue, + variableDescription=variableResult.VariableDescription + ) + + # For specimen data. + v_code = variableResult.VariableCode + v_code_i = v_code.find('::') + if v_code_i != -1: + v_code = v_code[0:v_code_i] + + variableCode = WaterML.variableCode() + variableCode.vocabulary = self.vocabulary + # TODO: What is this, should it always be true? + variableCode.default = "true" + variableCode.variableID = variableResult.VariableID + variableCode.valueOf_ = v_code + + variable.add_variableCode(variableCode) + clean_variableUnits = self.check_UnitsType(variableResult.VariableUnits.UnitsType) # noqa + + if variableResult.VariableUnits: + units = WaterML.units( + unitsAbbreviation=variableResult.VariableUnits.UnitsAbbreviation, # noqa + unitsCode=variableResult.VariableUnitsID, + # unitsType=variableResult.VariableUnits.UnitsType, + unitsType=clean_variableUnits, + valueOf_=variableResult.VariableUnits.UnitsName) + + variable.set_units(units) + + timeSupport = WaterML.timeSupport() + timeSupport.isRegular = variableResult.IsRegular + + if variableResult.TimeUnits: + timeUnits = WaterML.UnitsType( + UnitID=variableResult.TimeUnits.UnitsID, + UnitName=variableResult.TimeUnits.UnitsName, + UnitDescription=variableResult.TimeUnits.UnitsName, + # UnitType=variableResult.TimeUnits.UnitsType, + UnitType="Time", + UnitAbbreviation=variableResult.TimeUnits.UnitsAbbreviation) + + timeSupport.set_unit(timeUnits) + + # TODO: time interval is not the same as time support. + # Time interval refers to a spacing between values for regular data, + # which isn't stored in ODM. + if variableResult.TimeSupport: + # integer in WaterML 1.0 + timeSupport.timeInterval = str(int(variableResult.TimeSupport)) + variable.set_timeSupport(timeSupport) + return variable + + def create_wml2_values_object(self, siteArg, varArg, startDateTime=None, + endDateTime=None): + siteCode = self.get_site_code(siteArg) + varCode = self.get_variable_code(varArg) + valueResultArr = self.dao.get_datavalues(siteCode, varCode, + startDateTime, endDateTime) + return valueResultArr diff --git a/wof/examples/flask/odm2/measurement/core_1_1.py b/wof/examples/flask/odm2/measurement/core_1_1.py new file mode 100644 index 0000000..de075b4 --- /dev/null +++ b/wof/examples/flask/odm2/measurement/core_1_1.py @@ -0,0 +1,945 @@ +from __future__ import (absolute_import, division, print_function) + +import datetime +import logging +from xml.sax.saxutils import escape + +import dateutil.parser + +from wof import WaterML_1_1 as WaterML +from wof import core +from wof import vocabularies as voc + + +class WOF_1_1(object): + + network = 'NETWORK' + vocabulary = 'VOCABULARY' + menu_group_name = 'MENU_GROUP_NAME' + service_wsdl = 'SERVICE_WSDL' + timezone = None + timezone_abbr = None + + dao = None + + default_site = None + default_variable = None + default_start_date = None + default_end_date = None + default_west = None + default_south = None + default_north = None + default_east = None + + organization = 'MYORGANIZATION' + address = '1234 DriveIn' + city = 'Seattle' + state = 'Washington' + zipcode = '98064' + contactname = 'John Smith' + contactemail = 'johnsmith@example.com' + phone = '555-555-555' + link = 'http://www.example.com/' + + _config = None + _templates = None + + def __init__(self, dao, config_file=None, templates=None): + self.dao = dao + if templates is not None: + self._templates = templates + if config_file: + self.config_from_file(config_file) + + def config_from_file(self, file_name): + if self._templates: + config = core.WOFConfig(file_name, self._templates) + else: + config = core.WOFConfig(file_name) + self._config = config + + self.network = config.network.lower() + self.vocabulary = config.vocabulary.lower() + self.urlpath = config.urlpath.lower() + self.menu_group_name = config.menu_group_name + self.service_wsdl = config.service_wsdl + self.timezone = config.timezone + self.timezone_abbr = config.timezone_abbr + + self.default_site = config.default_site + self.default_variable = config.default_variable + self.default_start_date = config.default_start_date + self.default_end_date = config.default_end_date + self.default_unitid = config.default_unitid + self.default_samplemedium = config.default_samplemedium + + self.default_east = config.default_east + self.default_north = config.default_north + self.default_south = config.default_south + self.default_west = config.default_west + + self.organization = config.organization + self.address = config.address + self.city = config.city + self.state = config.state + self.zipcode = config.zipcode + self.contactname = config.contactname + self.contactemail = config.contactemail + self.phone = config.phone + self.link = config.link + + def get_site_code(self, siteArg): + + if ':' in siteArg: + networkname, siteCode = siteArg.split(':', 1) + networkname = networkname.lower() + # print('networkname!!') + # print(networkname) + # print(siteCode) + # print(self.network) + if self.network == networkname: + return siteCode + else: + return None + return siteArg + + def get_variable_code(self, varArg): + + if ':' in varArg: + vocabname, varCode = varArg.split(':', 1) + vocabname = vocabname.lower() + if self.vocabulary == vocabname: + return varCode + else: + return None + return varArg + + def create_get_site_response(self, siteArg=None): + print('here create_get_site_response') + if siteArg is None or siteArg == '': + siteResultArr = self.dao.get_all_sites() + else: + siteCodesArr = siteArg.split(',') + siteCodesArr = [self.get_site_code(s) + for s in siteCodesArr] + siteResultArr = self.dao.get_sites_by_codes(siteCodesArr) + + # if len(siteResultArr) == 0: + # return None + + siteInfoResponse = WaterML.SiteInfoResponseType() + queryInfo = WaterML.QueryInfoType(creationTime=datetime.datetime.now()) + # TODO: check on how this should be done for multiple sites. + pType = WaterML.parameterType(name='site', value=siteArg) + criteria = WaterML.criteriaType(MethodCalled='GetSites') + criteria.add_parameter(pType) + queryInfo.set_criteria(criteria) + # queryInfoNote = WaterML.NoteType() + # queryInfo.add_note(queryInfoNote) + # queryInfo.set_extension('') + siteInfoResponse.set_queryInfo(queryInfo) + + if siteResultArr: + for siteResult in siteResultArr: + s = self.create_site_element(siteResult) + siteInfoResponse.add_site(s) + else: + # site = WaterML.site() + # siteInfoResponse.add_site(site) + raise Exception("Site {} Not Found".format(siteArg)) + + return siteInfoResponse + + def create_get_site_info_response(self, siteArg, varArg=None): + print('here create_get_site_info_response') + siteCode = self.get_site_code(siteArg) + siteResult = self.dao.get_site_by_code(siteCode) + print('siteCode!!') + print(siteCode) + if (varArg is None or varArg == 'get_series_by_sitecode'): + print('get_series_by_sitecode') + seriesResultArr = self.dao.get_series_by_sitecode(siteCode) + else: + print('get_series_by_sitecode_and_varcode') + varCode = self.get_variable_code(varArg) + seriesResultArr = self.dao.get_series_by_sitecode_and_varcode( + siteCode, varCode) + + # if len(seriesResultArr) == 0: + # return None + # print(' variable code') + # print(varCode) + siteInfoResponse = WaterML.SiteInfoResponseType() + + queryInfo = WaterML.QueryInfoType(creationTime=datetime.datetime.now()) + criteria = WaterML.criteriaType(MethodCalled='GetSiteInfo') + pType_site = WaterML.parameterType(name='site', value=siteArg) + criteria.add_parameter(pType_site) + if varArg is not None: + pType_var = WaterML.parameterType(name='variable', value=varArg) + criteria.add_parameter(pType_var) + queryInfo.set_criteria(criteria) + # queryInfoNote = WaterML.NoteType() + # queryInfo.add_note(queryInfoNote) + # queryInfo.set_extension('') + siteInfoResponse.set_queryInfo(queryInfo) + + s = self.create_site_element(siteResult, seriesResultArr) + siteInfoResponse.add_site(s) + print('site info response!') + print(siteInfoResponse) + return siteInfoResponse + + def create_get_site_info_multiple_response(self, siteArg): + siteCodesArr = siteArg.split(',') + siteCodesArr = [self.get_site_code(s) + for s in siteCodesArr] + + siteInfoResponse = WaterML.SiteInfoResponseType() + + queryInfo = WaterML.QueryInfoType(creationTime=datetime.datetime.now()) + criteria = WaterML.criteriaType(MethodCalled='GetSiteInfo') + pType_site = WaterML.parameterType(name='site', value=siteArg) + criteria.add_parameter(pType_site) + queryInfo.set_criteria(criteria) + # queryInfoNote = WaterML.NoteType() + # queryInfo.add_note(queryInfoNote) + # queryInfo.set_extension('') + siteInfoResponse.set_queryInfo(queryInfo) + + for siteArg in siteCodesArr: + siteCode = self.get_site_code(siteArg) + siteResult = self.dao.get_site_by_code(siteCode) + seriesResultArr = self.dao.get_series_by_sitecode(siteCode) + + # if len(seriesResultArr) == 0: + # return None + s = self.create_site_element(siteResult, seriesResultArr) + siteInfoResponse.add_site(s) + + return siteInfoResponse + + def to_bool(self, value): + valid = { + 'true': True, + 't': True, + '1': True, + 'false': False, + 'f': False, + '0': False, + } + + if isinstance(value, bool): + return value + + if not isinstance(value, basestring): + value = False + return value + + lower_value = value.lower() + if lower_value in valid: + return valid[lower_value] + else: + value = False + return value + + def create_get_site_box_response(self, west, south, east, north, + IncludeSeries): + + IncludeSeries = self.to_bool(IncludeSeries) + siteResultArr = self.dao.get_sites_by_box(west, south, east, north) + + siteInfoResponse = WaterML.SiteInfoResponseType() + + queryInfo = WaterML.QueryInfoType(creationTime=datetime.datetime.now()) + criteria = WaterML.criteriaType(MethodCalled='GetSitesByBox') + pType_west = WaterML.parameterType(name='west', value=west) + pType_west = WaterML.parameterType(name='south', value=south) + pType_west = WaterML.parameterType(name='east', value=east) + pType_west = WaterML.parameterType(name='north', value=north) + pType_west = WaterML.parameterType(name='IncludeSeries', value=IncludeSeries) # noqa + criteria.add_parameter(pType_west) + queryInfo.set_criteria(criteria) + # queryInfoNote = WaterML.NoteType() + # queryInfo.add_note(queryInfoNote) + # queryInfo.set_extension('') + siteInfoResponse.set_queryInfo(queryInfo) + + for siteResult in siteResultArr: + seriesResultArr = None + if IncludeSeries: + seriesResultArr = self.dao.get_series_by_sitecode(siteResult.SiteCode) # noqa + + # if len(seriesResultArr) == 0: + # return None + s = self.create_site_element( + siteResult, + seriesResultArr, + IncludeSeries + ) + siteInfoResponse.add_site(s) + + return siteInfoResponse + + def create_get_variable_info_response(self, varArg=None): + + if (varArg is None or varArg == ''): + variableResultArr = self.dao.get_all_variables() + else: + varCodesArr = varArg.split(',') + varCodesArr = [self.get_variable_code(v) + for v in varCodesArr] + variableResultArr = self.dao.get_variables_by_codes(varCodesArr) + + variableInfoResponse = WaterML.VariablesResponseType() + + # TODO: Should queryInfo be in thois response? Suds doesn't + # like when it is. If it should be in the response, then the + # WSDL needs to be updated + + queryInfo = WaterML.QueryInfoType() + criteria = WaterML.criteriaType(MethodCalled='GetVariableInfo') + if varArg is not None: + pType_var = WaterML.parameterType(name='variable', value=varArg) + criteria.add_parameter(pType_var) + queryInfo.set_criteria(criteria) + queryInfoNote = WaterML.NoteType('Web Service') + queryInfo.add_note(queryInfoNote) + # queryInfo.set_extension('') + variableInfoResponse.set_queryInfo(queryInfo) + + variables = WaterML.variablesType() + for variableResult in variableResultArr: + v = self.create_variable_element(variableResult) + variables.add_variable(v) + variableInfoResponse.set_variables(variables) + return variableInfoResponse + + def create_get_values_response(self, siteArg, varArg, startDateTime=None, + endDateTime=None): + + # TODO: Tim thinks the DAO should handle network and vocab parsing, + # not WOF + siteCode = self.get_site_code(siteArg) + varCode = self.get_variable_code(varArg) + + valueResultArr = self.dao.get_datavalues(siteCode, varCode, + startDateTime, endDateTime) + # if not valueResultArr: + # raise Exception( + # "ERROR: No data found for {}:{} for dates {} - {}".format( + # siteCode, varCode, startDateTime, endDateTime + # ) + # ) + + timeSeriesResponse = WaterML.TimeSeriesResponseType() + + queryInfo = WaterML.QueryInfoType(creationTime=datetime.datetime.now()) + criteria = WaterML.criteriaType(MethodCalled='GetValues') + pType_site = WaterML.parameterType(name='site', value=siteArg) + criteria.add_parameter(pType_site) + pType_var = WaterML.parameterType(name='variable', value=varArg) + criteria.add_parameter(pType_var) + if startDateTime is not None: + pType_sdate = WaterML.parameterType( + name='startDate', + value=startDateTime + ) + criteria.add_parameter(pType_sdate) + if endDateTime is not None: + pType_edate = WaterML.parameterType( + name='endDate', + value=endDateTime + ) + criteria.add_parameter(pType_edate) + queryInfo.set_criteria(criteria) + # queryInfoNote = WaterML.NoteType() + # queryInfo.add_note(queryInfoNote) + # queryInfo.set_extension('') + timeSeriesResponse.set_queryInfo(queryInfo) + + # if not valueResultArr: + # timeSeries = WaterML.TimeSeriesType() + # timeSeriesResponse.add_timeSeries(timeSeries) + # return timeSeriesResponse + + if not valueResultArr: + raise Exception( + "Values Not Found for {}:{} for dates {} - {}".format( + siteCode, varCode, startDateTime, endDateTime) + ) + + if isinstance(valueResultArr, dict): + for key in valueResultArr.keys(): + timeSeries = self.create_timeseries( + siteCode, + key, + valueResultArr[key] + ) + timeSeriesResponse.add_timeSeries(timeSeries) + else: + timeSeries = self.create_timeseries( + siteCode, + varCode, + valueResultArr + ) + timeSeriesResponse.add_timeSeries(timeSeries) + return timeSeriesResponse + + def create_timeseries(self, siteCode, varCode, valueResultArr): + + timeSeries = WaterML.TimeSeriesType() + + # sourceInfo (which is a siteInfo) element + siteResult = self.dao.get_site_by_code(siteCode) + + # TODO: Exception? + if not siteResult: + pass + + sourceInfo = self.create_site_info_element(siteResult) + timeSeries.sourceInfo = sourceInfo + + # Variable element. + varResult = self.dao.get_variable_by_code(varCode) + + # TODO: Exception? + if not varResult: + pass + + variable = self.create_variable_element(varResult) + timeSeries.variable = variable + + # TODO: fill in some more of the attributes in this element. + values = WaterML.TsValuesSingleVariableType() + + # waterML 1.0 + # values.count = len(valueResultArr) + # if varResult.VariableUnits: + # values.unitsAbbreviation = varResult.VariableUnits.UnitsAbbreviation # noqa + # values.unitsCode = varResult.VariableUnits.UnitsID + + # Need to keep track of unique methodIDs and sourceIDs. + methodIdSet = set() + sourceIdSet = set() + qualifierIdSet = set() + offsetTypeIdSet = set() + qualitycontrollevelIdSet = set() + censorcodeset = {} + + for valueResult in valueResultArr: + v = self.create_value_element(valueResult) + values.add_value(v) + if valueResult.MethodID is not None and valueResult.MethodCode: + methodIdSet.add(valueResult.MethodID) + + if valueResult.SourceID is not None: + sourceIdSet.add(valueResult.SourceID) + + if valueResult.QualifierID is not None: + qualifierIdSet.add(valueResult.QualifierID) + + if valueResult.OffsetTypeID is not None: + offsetTypeIdSet.add(valueResult.OffsetTypeID) + + if valueResult.QualityControlLevelID is not None: + qualitycontrollevelIdSet.add(valueResult.QualityControlLevelID) + + if valueResult.CensorCode is not None: + censorcode = v.get_censorCode() + censorcodeset[censorcode] = valueResult.CensorCode + + # Add method elements for each unique methodID. + if methodIdSet: + methodIdArr = list(methodIdSet) + methodResultArr = self.dao.get_methods_by_ids(methodIdArr) + for methodResult in methodResultArr: + method = self.create_method_element(methodResult) + values.add_method(method) + + # Add source elements for each unique sourceID. + if sourceIdSet: + sourceIdArr = list(sourceIdSet) + sourceResultArr = self.dao.get_sources_by_ids(sourceIdArr) + for sourceResult in sourceResultArr: + source = self.create_source_element(sourceResult) + values.add_source(source) + + # Add qualifier elements. + if qualifierIdSet: + qualIdArr = list(qualifierIdSet) + qualResultArr = self.dao.get_qualifiers_by_ids(qualIdArr) + for qualifierResult in qualResultArr: + q = WaterML.QualifierType( + qualifierID=qualifierResult.QualifierID, + default=None, + network=self.network, + vocabulary=self.vocabulary, + qualifierCode=qualifierResult.QualifierCode) + values.add_qualifier(q) + + # Add offset elements + if offsetTypeIdSet: + offsetTypeIdArr = list(offsetTypeIdSet) + offsetTypeResultArr = self.dao.get_offsettypes_by_ids( + offsetTypeIdArr) + for offsetTypeResult in offsetTypeResultArr: + offset = self.create_offset_element(offsetTypeResult) + values.add_offset(offset) + + # Add qualitycontrollevel elements. + if qualitycontrollevelIdSet: + qlevelIdIdArr = list(qualitycontrollevelIdSet) + try: + qlevelResultArr = self.dao.get_qualcontrollvls_by_ids(qlevelIdIdArr) # noqa + for qlevelResult in qlevelResultArr: + qlevel = self.create_qlevel_element(qlevelResult) + values.add_qualityControlLevel(qlevel) + except: + for qlevelID in qlevelIdIdArr: + qlevel = WaterML.QualityControlLevelType( + qualityControlLevelID=qlevelID) + values.add_qualityControlLevel(qlevel) + + # Add censorcode element. + if censorcodeset: + for key in censorcodeset: + cCode = WaterML.CensorCodeType( + censorCode=key, + censorCodeDescription=censorcodeset[key] + ) + values.add_censorCode(cCode) + + timeSeries.add_values(values) + # timeSeriesResponse.set_timeSeries(timeSeries) + return timeSeries + + def create_get_values_site_response(self, site, + startDateTime, + endDateTime): + + timeSeriesResponse = WaterML.TimeSeriesResponseType() + queryInfo = WaterML.QueryInfoType(creationTime=datetime.datetime.now()) + criteria = WaterML.criteriaType(MethodCalled='GetValuesForASite') + pType_site = WaterML.parameterType(name='site', value=site) + criteria.add_parameter(pType_site) + if startDateTime is not None: + pType_sdate = WaterML.parameterType( + name='startDate', + value=startDateTime + ) + criteria.add_parameter(pType_sdate) + if endDateTime is not None: + pType_edate = WaterML.parameterType( + name='endDate', + value=endDateTime + ) + criteria.add_parameter(pType_edate) + queryInfo.set_criteria(criteria) + # queryInfoNote = WaterML.NoteType() + # queryInfo.add_note(queryInfoNote) + # queryInfo.set_extension('') + timeSeriesResponse.set_queryInfo(queryInfo) + + siteCode = self.get_site_code(site) + seriesResultArr = self.dao.get_series_by_sitecode(siteCode) + if seriesResultArr: + for seriesResult in seriesResultArr: + valueResultArr = self.dao.get_datavalues( + siteCode, + seriesResult.Variable.VariableCode, + startDateTime, + endDateTime + ) + # if not valueResultArr: + # raise Exception( + # ('ERROR: No data found for {} for dates ' + # '{} - {}').format(site, startDateTime, endDateTime)) + if not valueResultArr: + continue + + timeSeries = self.create_timeseries( + siteCode, + seriesResult.Variable.VariableCode, + valueResultArr + ) + timeSeriesResponse.add_timeSeries(timeSeries) + + return timeSeriesResponse + + def create_qlevel_element(self, qlevelResult): + qlevel = WaterML.QualityControlLevelType( + qualityControlLevelID=qlevelResult.QualityControlLevelID, + qualityControlLevelCode=qlevelResult.QualityControlLevelID, + definition=qlevelResult.Definition, + explanation=qlevelResult.Explanation + ) + return qlevel + + def create_offset_element(self, offsetTypeResult): + # TODO: where does offsetIsVertical come from. + # TODO: where does offsetHorizDirectionDegrees come from? + offset = WaterML.OffsetType( + offsetTypeID=offsetTypeResult.OffsetTypeID, + offsetValue=None, + offsetDescription=offsetTypeResult.OffsetDescription, + offsetIsVertical=True) + + if offsetTypeResult.OffsetUnits: + units = WaterML.UnitsType( + unitID=offsetTypeResult.OffsetUnits.UnitsID, + unitAbbreviation=offsetTypeResult.OffsetUnits.UnitsAbbreviation, # noqa + unitName=offsetTypeResult.OffsetUnits.UnitsName, + unitType=offsetTypeResult.OffsetUnits.UnitsType, + unitCode=offsetTypeResult.OffsetUnits.UnitsID + ) + + offset.units = units + + return offset + + def create_method_element(self, methodResult): + method = WaterML.MethodType( + methodID=methodResult.MethodID, + methodCode=methodResult.MethodID, + methodDescription=methodResult.MethodDescription, + methodLink=methodResult.MethodLink) + + # Need at least one MethodLink element to meet WaterML 1.0 + # schema validation. + if method.methodLink is None: + method.methodLink = '' + + return method + + def create_source_element(self, sourceResult): + source = WaterML.SourceType( + sourceID=sourceResult.SourceID, + sourceCode=sourceResult.SourceCode, + organization=sourceResult.Organization, + sourceDescription=sourceResult.SourceDescription, + sourceLink=sourceResult.SourceLink) + + contactInfo = self.create_contact_info_element(sourceResult) + + source.contactInformation = [contactInfo] + + if sourceResult.Metadata: + metadata = WaterML.MetaDataType( + topicCategory=sourceResult.Metadata.TopicCategory, + title=sourceResult.Metadata.Title, + abstract=sourceResult.Metadata.Abstract, + profileVersion=sourceResult.Metadata.ProfileVersion, + metadataLink=sourceResult.Metadata.MetadataLink) + + source.Metadata = metadata + + return source + + def create_contact_info_element(self, sourceResult): + + if sourceResult.ContactName: + + addressString = '' + if sourceResult.Address: + addressString = '{}'.format(sourceResult.Address) + + contactInfo = WaterML.ContactInformationType( + email=[sourceResult.Email], + contactName=sourceResult.ContactName, + phone=[sourceResult.Phone], + address=[addressString]) + + return contactInfo + + return None + + # TODO: lots more stuff to fill out here. + def create_value_element(self, valueResult): + # datetime_string = core._get_datavalues_datetime( + # valueResult, "LocalDateTime", "DateTimeUTC") + aDate = core._get_datavalues_datetime( + valueResult, + "LocalDateTime", + "DateTimeUTC" + ) + # aDate= dateutil.parser.parse(datetime_string) + + if not hasattr(valueResult, 'MethodCode'): + setattr(valueResult, 'MethodCode', None) + if not hasattr(valueResult, 'SourceCode'): + setattr(valueResult, 'SourceCode', None) + + clean_censorCode = voc.check_censorCode(valueResult.CensorCode) + # clean_qcl = self.check_QualityControlLevel(valueResult.QualityControlLevel) # noqa + + value = WaterML.ValueSingleVariable( + qualityControlLevelCode=valueResult.QualityControlLevelID, + methodCode=valueResult.MethodID, + sourceCode=valueResult.SourceCode, + timeOffset=valueResult.UTCOffset, + censorCode=clean_censorCode, + sampleID=valueResult.SampleID, + offsetTypeID=valueResult.OffsetTypeID, + accuracyStdDev=valueResult.ValueAccuracy, + offsetValue=valueResult.OffsetValue, + # dateTime=datetime_string, + dateTime=aDate, + dateTimeUTC=valueResult.DateTimeUTC, + qualifiers=valueResult.QualifierID, + valueOf_=str(valueResult.DataValue) + ) + + # TODO: value.offset stuff? Why does value element have all + # this offset stuff + # offsetTypeResult = valueResult.OffsetType + # if offsetTypeResult != None: + # value.offsetDescription = offsetTypeResult.OffsetDescription + # value.offsetUnitsAbbreviation = offsetTypeResult.OffsetUnits.UnitsAbbreviation # noqa + # value.offsetUnitsCode = offsetTypeResult.OffsetUnits.UnitsID + return value + + def create_site_element(self, siteResult, + seriesResultArr=None, IncludeSeries=True): + site = WaterML.siteType() + siteInfo = self.create_site_info_element(siteResult) + + site.set_siteInfo(siteInfo) + + if any([siteResult.County, siteResult.State, siteResult.Comments]): + if siteResult.County: + countyNote = WaterML.PropertyType( + name="County", + valueOf_=siteResult.County + ) + siteInfo.add_siteProperty(countyNote) + + if siteResult.State: + stateNote = WaterML.PropertyType( + name="State", + valueOf_=siteResult.State + ) + siteInfo.add_siteProperty(stateNote) + + if siteResult.Comments: + commentsNote = WaterML.PropertyType( + name="Site Comments", + valueOf_=escape(siteResult.Comments)) + siteInfo.add_siteProperty(commentsNote) + + if IncludeSeries: + seriesCatalog = WaterML.seriesCatalogType() + if seriesResultArr is not None: + seriesCatalog.menuGroupName = self.menu_group_name + # TODO: Make sure this is set properly in config filename. + seriesCatalog.serviceWsdl = self.service_wsdl + + for seriesResult in seriesResultArr: + series = self.create_series_element(seriesResult) + seriesCatalog.add_series(series) + + site.add_seriesCatalog(seriesCatalog) + + # need at least one extension element to meet WaterML 1.0 + # schema validation + # site.set_extension('') + return site + + def create_site_info_element(self, siteResult): + siteInfo = WaterML.SiteInfoType() + siteInfo.set_siteName(siteResult.SiteName) + + # TODO: agencyName + siteCode = WaterML.siteCodeType( + network=self.network, + siteID=siteResult.SiteID, + valueOf_=siteResult.SiteCode, + agencyCode=siteResult.AgencyCode, + agencyName=siteResult.AgencyName + ) + + siteInfo.add_siteCode(siteCode) + + # TODO: Maybe remove this? None of the other WOF services + # return this info probably because it is not that useful + # timeZoneInfo = WaterML.timeZoneInfoType( + # siteUsesDaylightSavingsTime=False, + # daylightSavingsTimeZone=None + # ) + # timeZoneInfo.defaultTimeZone = WaterML.defaultTimeZoneType( + # zoneOffset=self.timezone, + # zoneAbbreviation=self.timezone_abbr) + + # siteInfo.set_timeZoneInfo(timeZoneInfo) + geoLocation = WaterML.geoLocationType() + geogLocation = WaterML.LatLonPointType( + srs="EPSG:{0}".format(siteResult.LatLongDatum.SRSID), + latitude=siteResult.Latitude, + longitude=siteResult.Longitude) + geoLocation.set_geogLocation(geogLocation) + + if (siteResult.LocalX and siteResult.LocalY): + localSiteXY = WaterML.localSiteXYType() + localSiteXY.projectionInformation = siteResult.LocalProjection.SRSName # noqa + localSiteXY.X = siteResult.LocalX + localSiteXY.Y = siteResult.LocalY + geoLocation.add_localSiteXY(localSiteXY) + + siteInfo.set_geoLocation(geoLocation) + siteInfo.set_verticalDatum(siteResult.VerticalDatum) + + # need at least one extension element to meet WaterML 1.0 + # schema validation + + # siteInfo.set_extension('') + + # need at least one altname element to meet WaterML 1.0 schema + # validation + # siteInfo.set_altname('') + return siteInfo + + def create_series_element(self, seriesResult): + series = WaterML.seriesType() + + # Variable + variable = self.create_variable_element(seriesResult.Variable) + series.set_variable(variable) + + series.valueCount = WaterML.valueCountType( + valueOf_=str(seriesResult.ValueCount)) + + beginDateTime = core._get_datavalues_datetime( + seriesResult, "BeginDateTime", "BeginDateTimeUTC") + endDateTime = core._get_datavalues_datetime( + seriesResult, "EndDateTime", "EndDateTimeUTC") + # WML1_1 wants a datetime, no zone. breaks the conversion + try: + beginDateTime = dateutil.parser.parse(beginDateTime) + except Exception as inst: + logging.warn('bad datetime conversion on beginDateTime:' + inst.message) # noqa + try: + endDateTime = dateutil.parser.parse(endDateTime) + except Exception as inst: + logging.warn('bad datetime conversion on endDateTime:' + inst.message) # noqa + + # TimeInterval. + if beginDateTime is None: + beginDateTime = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S") # noqa + if endDateTime is None: + endDateTime = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S") # noqa + + variableTimeInt = WaterML.TimeIntervalType( + beginDateTime=beginDateTime, + endDateTime=endDateTime) + series.variableTimeInterval = variableTimeInt + + # Method. + if seriesResult.Method: + method = self.create_method_element(seriesResult.Method) + series.method = method + + # Source. + if seriesResult.Source: + source = self.create_source_element(seriesResult.Source) + series.source = source + + # QualityControlLevel. + qualityControlLevel = WaterML.QualityControlLevelType( + qualityControlLevelID=seriesResult.QualityControlLevelID, + qualityControlLevelCode=seriesResult.QualityControlLevelID, + definition=seriesResult.Definition, + explanation=seriesResult.Explanation + ) + series.qualityControlLevel = qualityControlLevel + return series + + def create_variable_element(self, variableResult): + clean_datatype = voc.check_dataTypeEnum(variableResult.DataType) + clean_medium = voc.check_SampleMedium(variableResult.SampleMedium) + # GeneralCategoryValidate and ValueTypeValidate optional capability added to support ODM2 CV handling + _GeneralCategoryValidate = True + if hasattr(variableResult, 'GeneralCategoryValidate'): + _GeneralCategoryValidate = variableResult.GeneralCategoryValidate + clean_category = voc.check_generalCategory(variableResult.GeneralCategory, + validate=_GeneralCategoryValidate) + _ValueTypeValidate = True + if hasattr(variableResult, 'ValueTypeValidate'): + _ValueTypeValidate = variableResult.ValueTypeValidate + clean_valuetype = voc.check_valueType(variableResult.ValueType, + validate=_ValueTypeValidate) + + variable = WaterML.VariableInfoType( + variableName=variableResult.VariableName, + valueType=clean_valuetype, + dataType=clean_datatype, + generalCategory=clean_category, + sampleMedium=clean_medium, + noDataValue=variableResult.NoDataValue, + variableDescription=variableResult.VariableDescription, + speciation=variableResult.Speciation) + + # For specimen data. + v_code = variableResult.VariableCode + v_code_i = v_code.find('::') + if v_code_i != -1: + v_code = v_code[0:v_code_i] + + variableCode = WaterML.variableCodeType() + variableCode.vocabulary = self.vocabulary + # TODO: What is this, should it always be true? + variableCode.default = "true" + variableCode.variableID = variableResult.VariableID + variableCode.valueOf_ = v_code + + variable.add_variableCode(variableCode) + # UnitsTypeValidate optional capability added to support ODM2 CV handling + _UnitsTypeValidate = True + if hasattr(variableResult.VariableUnits, 'UnitsTypeValidate'): + _UnitsTypeValidate = variableResult.VariableUnits.UnitsTypeValidate + clean_variableUnitsType = voc.check_UnitsType(variableResult.VariableUnits.UnitsType, + validate=_UnitsTypeValidate) + + if variableResult.VariableUnits: + units = WaterML.UnitsType( + unitID=variableResult.VariableUnitsID, + unitAbbreviation=variableResult.VariableUnits.UnitsAbbreviation, # noqa + unitCode=variableResult.VariableUnitsID, + unitType=clean_variableUnitsType, + unitName=variableResult.VariableUnits.UnitsName) + + variable.set_unit(units) + + timeSupport = WaterML.timeScaleType() + timeSupport.isRegular = variableResult.IsRegular + + if variableResult.TimeUnits: + timeUnits = WaterML.UnitsType( + unitID=variableResult.TimeUnits.UnitsID, + unitName=variableResult.TimeUnits.UnitsName, + unitDescription=variableResult.TimeUnits.UnitsName, + unitType=variableResult.TimeUnits.UnitsType, + unitAbbreviation=variableResult.TimeUnits.UnitsAbbreviation, + unitCode=variableResult.TimeUnits.UnitsID) + + timeSupport.set_unit(timeUnits) + + # TODO: time interval is not the same as time support. + # Time interval refers to a spacing between values for regular data, + # which isn't stored in ODM. + if variableResult.TimeSupport: + # integer in WaterML 1.0 + timeSupport.timeSupport = float(variableResult.TimeSupport) + variable.set_timeScale(timeSupport) + return variable + + def create_wml2_values_object(self, siteArg, varArg, startDateTime=None, + endDateTime=None): + siteCode = self.get_site_code(siteArg) + varCode = self.get_variable_code(varArg) + valueResultArr = self.dao.get_datavalues( + siteCode, + varCode, + startDateTime, + endDateTime + ) + return valueResultArr diff --git a/wof/examples/flask/odm2/measurement/dao.py b/wof/examples/flask/odm2/measurement/dao.py new file mode 100644 index 0000000..2cb9e27 --- /dev/null +++ b/wof/examples/flask/odm2/measurement/dao.py @@ -0,0 +1,129 @@ +from __future__ import (absolute_import, division, print_function) + +msg = '{} Method not implemented by this service.'.format + + +class BaseDao(object): + + def get_all_sites(self): + """ + Returns a list of all the Sites in the data source. + """ + raise NotImplementedError(msg('get_all_sites')) + + def get_site_by_code(self, site_code): + """ + Returns a single Site identified by its code. + """ + raise NotImplementedError(msg('get_site_by_code')) + + def get_sites_by_codes(self, site_codes_arr): + """ + Returns a list of Sites identified by the given site code list. + """ + raise NotImplementedError(msg('get_sites_by_codes')) + + def get_all_variables(self): + """ + Returns a list of all Variables in the data source. + """ + raise NotImplementedError(msg('get_all_variables')) + + def get_variable_by_code(self, var_code): + """ + Returns a single Variable identified by its code. + """ + raise NotImplementedError(msg('get_variable_by_code')) + + def get_variables_by_codes(self, var_codes_arr): + """ + Returns a list of Variables identified by the given variable code list. + """ + raise NotImplementedError(msg('get_variables_by_codes')) + + def get_series_by_sitecode(self, site_code): + """ + Returns a list of Series for the given site code. + """ + raise NotImplementedError(msg('get_series_by_sitecode')) + + def get_series_by_sitecode_and_varcode(self, site_code, var_code): + """ + Returns a list of Series for the given site code and variable + code combination. + """ + raise NotImplementedError(msg('get_series_by_sitecode_and_varcode')) + + def get_datavalues(self, site_code, var_code, begin_date_time=None, + end_date_time=None): + """ + Returns a list of DataValues for the given site code and variable code, + filtered by the optional begin and end datetimes. + """ + raise NotImplementedError(msg('get_datavalues')) + + def get_method_by_id(self, method_id): + """ + Returns a single Method identified by the given id. + """ + raise NotImplementedError(msg('get_method_by_id')) + + def get_methods_by_ids(self, method_id_arr): + """ + Returns a list of Methods identified by the given id list. + """ + raise NotImplementedError(msg('get_methods_by_ids')) + + def get_source_by_id(self, source_id): + """ + Returns a single Source identified by the given id. + """ + raise NotImplementedError(msg('get_source_by_id')) + + def get_sources_by_ids(self, source_id_arr): + """ + Returns a list of Sources identified by the given id list. + """ + raise NotImplementedError(msg('get_sources_by_ids')) + + def get_qualifier_by_id(self, qualifier_id): + """ + Returns a single Qualifier identified by the given id. + """ + raise NotImplementedError(msg('get_qualifier_by_id')) + + def get_qualifiers_by_ids(self, qualifier_id_arr): + """ + Returns a list of Qualifiers identified by the given id list. + """ + raise NotImplementedError(msg('get_qualifiers_by_ids')) + + def get_qualcontrollvl_by_id(self, qual_control_lvl_id): + """ + Returns a single QualityControlLevel identified by the given id. + """ + raise NotImplementedError(msg('get_qualcontrollvl_by_id')) + + def get_qualcontrollvls_by_ids(self, qual_control_lvl_id_arr): + """ + Returns a list of QualityControlLevels identified by the given id list. + """ + raise NotImplementedError(msg('get_qualcontrollvls_by_ids')) + + def get_offsettype_by_id(self, offset_type_id): + """ + Returns a single OffsetType identified by the given id. + """ + raise NotImplementedError(msg('get_offsettype_by_id')) + + def get_offsettypes_by_ids(self, offset_type_id_arr): + """ + Returns a list of OffsetTypes identified by the given id list. + """ + raise NotImplementedError(msg('get_offsettypes_by_ids')) + + def get_sites_by_box(self, west, south, east, north): + """ + Returns a list of OffsetTypes identified by the given id list. + """ + raise NotImplementedError(msg('get_sites_by_box')) diff --git a/wof/examples/flask/odm2/measurement/odm2_measurement_dao.py b/wof/examples/flask/odm2/measurement/odm2_measurement_dao.py index 26530b2..d11122f 100644 --- a/wof/examples/flask/odm2/measurement/odm2_measurement_dao.py +++ b/wof/examples/flask/odm2/measurement/odm2_measurement_dao.py @@ -36,7 +36,15 @@ def __init__(self, db_connection_string): self.yml_dict = yaml.load(yml) def __del__(self): self.db_session.close() -# + + def db_check(self): + try: + self.db_session.query(odm2_models.SamplingFeatures).first() + except: + self.db_session.rollback() + finally: + pass + def get_match(self, cvkey, term): for k, v in self.yml_dict[cvkey].items(): if term in v: @@ -65,11 +73,14 @@ def get_all_sites(self): def get_site_by_code(self, site_code): w_s = None + print('HEREEEEEEE') try: s = self.db_session.query(odm2_models.Sites).\ filter(odm2_models.Sites.SamplingFeatureCode == site_code).one() except: s = None + print(site_code) + print('NO SITE') if s is not None: w_s = model.Site(s) @@ -203,45 +214,52 @@ def get_series_by_sitecode(self, site_code): site = self.get_site_by_code(site_code) if site is None: + print('here OMG') return None - r = self.db_session.query(odm2_models.MeasurementResults.VariableID.label('vid'), - odm2_models.MeasurementResults.UnitsID.label('unitid'), - odm2_models.MeasurementResults.SampledMediumCV.label('samplemedium'), - func.min(odm2_models.Actions.BeginDateTime).label("begindate"), - func.max(odm2_models.Actions.EndDateTime).label("enddate")).\ - group_by(odm2_models.MeasurementResults.VariableID, - odm2_models.MeasurementResults.UnitsID, - odm2_models.MeasurementResults.SampledMediumCV).\ - join(odm2_models.FeatureActions).\ - join(odm2_models.Actions).\ - join(odm2_models.Specimens).\ - filter(odm2_models.Specimens.SamplingFeatureID == odm2_models.RelatedFeatures.SamplingFeatureID, - odm2_models.MeasurementResults.FeatureActionID == odm2_models.FeatureActions.FeatureActionID, - odm2_models.RelatedFeatures.RelatedFeatureID == site.SiteID).all() + # odm2_models.MeasurementResults.SampledMediumCV + # odm2_models.RelatedFeatures.RelatedFeatureID == site.SiteID + # odm2_models.RelatedFeatures.RelatedFeatureID == site.SiteID + # , + # odm2_models.MeasurementResults.SampledMediumCV + self.db_check() result = self.db_session.query(odm2_models.MeasurementResults).\ - distinct(odm2_models.MeasurementResults.VariableID, - odm2_models.MeasurementResults.UnitsID, - odm2_models.MeasurementResults.SampledMediumCV).\ join(odm2_models.FeatureActions).\ - join(odm2_models.Specimens).\ - filter(odm2_models.Specimens.SamplingFeatureID == odm2_models.RelatedFeatures.SamplingFeatureID, - odm2_models.MeasurementResults.FeatureActionID == odm2_models.FeatureActions.FeatureActionID, - odm2_models.RelatedFeatures.RelatedFeatureID == site.SiteID).all() - + join(odm2_models.SamplingFeatures).\ + filter(odm2_models.SamplingFeatures.SamplingFeatureCode == site_code, + odm2_models.MeasurementResults.FeatureActionID == odm2_models.FeatureActions.FeatureActionID).\ + group_by(odm2_models.MeasurementResults.VariableID, + odm2_models.MeasurementResults.ResultID, + odm2_models.Results.ResultID) + print('get the results') + print(len(result.all())) + # print(str(result.statement.compile(dialect=postgresql.dialect()))) + result = result.all() + print('number of results') r_arr = [] aff = None first_flag = True - for series in result: - for item in r: - if item.vid == series.VariableID and item.unitid == series.UnitsID: - if first_flag: - first_flag = False - aff = self.db_session.query(odm2_models.Affiliations).\ - filter(odm2_models.Affiliations.OrganizationID == series.FeatureActionObj.ActionObj.MethodObj.OrganizationID).first() - w_r = model.Series(series,aff,item.begindate,item.enddate) - r_arr.append(w_r) - + # for series in result: + # for item in r: + print('about to it results') + for i in range(len(result)): + # print('it result ' + str(result.Resultid)) + if i == 0: + + aff = self.db_session.query(odm2_models.Affiliations). \ + join(odm2_models.ActionBy). \ + filter(odm2_models.ActionBy.ActionID == result[i].FeatureActionObj.ActionID).first() # noqa + if aff is not None: + w_r = model.Series(result[i],aff, '2000-01-01T00:00:00', '2018-12-01T00:00:00') #,result[i].BeginDateTime,result[i].EndDateTime + else: + w_r = model.Series(result[i],None, '2000-01-01T00:00:00', '2018-12-01T00:00:00') #,result[i].BeginDateTime,result[i].EndDateTime + w_r.Variable.DataType = self.get_match('datatype', w_r.Variable.DataType) + w_r.Variable.SampleMedium = self.get_match('samplemedium', w_r.Variable.SampleMedium) + w_r.SampleMedium = w_r.Variable.SampleMedium + print(w_r.Variable) + # print(w_r.Definition) + r_arr.append(w_r) + print('done') return r_arr def get_series_by_sitecode_and_varcode(self, site_code, var_code): @@ -249,6 +267,7 @@ def get_series_by_sitecode_and_varcode(self, site_code, var_code): site = self.get_site_by_code(site_code) if site is None: return None + print('HELLO HELLO') r = self.db_session.query(odm2_models.MeasurementResults.VariableID.label('vid'), odm2_models.MeasurementResults.UnitsID.label('unitid'), odm2_models.MeasurementResults.SampledMediumCV.label('samplemedium'), @@ -314,16 +333,18 @@ def get_datavalues(self, site_code, var_code, begin_date_time=None, try: # print('HERE HERE') q = self.get_specimen_data() - # print(len(q.all())) - # print('all spcimen values') + print(len(q.all())) + print('all spcimen values') + print(site_code) + print(var_code) q = q.filter( odm2_models.MeasurementResults.FeatureActionID == odm2_models.FeatureActions.FeatureActionID, # noqa odm2_models.MeasurementResults.VariableID == odm2_models.Variables.VariableID, odm2_models.SamplingFeatures.SamplingFeatureCode == site_code, odm2_models.Variables.VariableCode == var_code). \ order_by(odm2_models.MeasurementResultValues.ValueDateTime) - # print(len(q.all())) - # print('number of values') + print(len(q.all())) + print('number of values') # q.filter(odm2_models.Specimens.SamplingFeatureID == odm2_models.RelatedFeatures.SamplingFeatureID, # odm2_models.MeasurementResults.FeatureActionID == odm2_models.FeatureActions.FeatureActionID, # odm2_models.RelatedFeatures.RelatedFeatureID == site.SiteID, @@ -332,24 +353,36 @@ def get_datavalues(self, site_code, var_code, begin_date_time=None, q = q.filter(odm2_models.MeasurementResults.UnitsID == int(unitid)) if samplemedium is not None: q = q.filter(odm2_models.MeasurementResults.SampledMediumCV == samplemedium) + print(len(q.all())) + print('number of values more filtered') valueResultArr = q.all() except: valueResultArr = [] else: + print('parse dates') begin_date_time = parse(begin_date_time) end_date_time = parse(end_date_time) + print(begin_date_time) + print(end_date_time) try: + # print(len(q.all())) + # print('all spcimen values') + print(site_code) + print(var_code) q = self.get_specimen_data() - q = q.filter(odm2_models.Specimens.SamplingFeatureID == odm2_models.RelatedFeatures.SamplingFeatureID, - odm2_models.MeasurementResults.FeatureActionID == odm2_models.FeatureActions.FeatureActionID, - # odm2_models.RelatedFeatures.RelatedFeatureID == site.SiteID, - odm2_models.Variables.VariableCode == var_code, + q = q.filter(odm2_models.MeasurementResults.FeatureActionID == odm2_models.FeatureActions.FeatureActionID, # noqa + odm2_models.MeasurementResults.VariableID == odm2_models.Variables.VariableID, + odm2_models.SamplingFeatures.SamplingFeatureCode == site_code,odm2_models.Variables.VariableCode == var_code, odm2_models.MeasurementResultValues.ValueDateTime >= begin_date_time, - odm2_models.MeasurementResultValues.ValueDateTime <= end_date_time) + odm2_models.MeasurementResultValues.ValueDateTime <= end_date_time).order_by(odm2_models.MeasurementResultValues.ValueDateTime) + print(len(q.all())) + print('number of values') if unitid is not None: q = q.filter(odm2_models.MeasurementResults.UnitsID == int(unitid)) if samplemedium is not None: q = q.filter(odm2_models.MeasurementResults.SampledMediumCV == samplemedium) + print(len(q.all())) + print('number of filtered values') valueResultArr = q.all() except: valueResultArr = [] diff --git a/wof/examples/flask/odm2/measurement/odm2_measurement_dao_off.py b/wof/examples/flask/odm2/measurement/odm2_measurement_dao_off.py new file mode 100644 index 0000000..73e3d55 --- /dev/null +++ b/wof/examples/flask/odm2/measurement/odm2_measurement_dao_off.py @@ -0,0 +1,413 @@ +from __future__ import (absolute_import, division, print_function) + +from datetime import datetime +from sqlalchemy import create_engine, distinct, func +from sqlalchemy.orm import scoped_session, sessionmaker +from sqlalchemy.sql import and_ + +from dateutil.parser import parse +import os +import yaml +from wof.dao import BaseDao +import wof.examples.flask.odm2.measurement.sqlalch_odm2_models as model +import odm2api.ODM2.models as odm2_models +from sqlalchemy.orm import aliased +from sqlalchemy import or_ +from sqlalchemy import and_ +from sqlalchemy.sql import func + +from sqlalchemy.orm import with_polymorphic + +import re + +class Odm2Dao(BaseDao): + + def __init__(self, db_connection_string): + self.engine = create_engine(db_connection_string, convert_unicode=True, + pool_size=100) + odm2_models.setSchema(self.engine) + # Default application pool size is 5. Use 100 to improve performance. + self.db_session = scoped_session(sessionmaker( + autocommit=False, autoflush=False, bind=self.engine)) + #odm2_models.Base.query = self.db_session.query_property() + with open(os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'cvmap_wml_1_1.yml'))) as yml: + self.yml_dict = yaml.load(yml) + + def __del__(self): + self.db_session.close() + + def get_match(self, cvkey, term): + for k, v in self.yml_dict[cvkey].items(): + if term in v: + return k + return term + + def get_all_sites(self): + s_Arr = [] + s_rArr = self.db_session.query(odm2_models.MeasurementResults,odm2_models.Sites).\ + distinct(odm2_models.Sites.SamplingFeatureID).\ + join(odm2_models.FeatureActions).\ + join(odm2_models.Specimens).\ + filter(odm2_models.Specimens.SamplingFeatureID == odm2_models.RelatedFeatures.SamplingFeatureID, + odm2_models.MeasurementResults.FeatureActionID == odm2_models.FeatureActions.FeatureActionID, + odm2_models.Sites.SamplingFeatureID == odm2_models.RelatedFeatures.RelatedFeatureID).all() + for s_r in s_rArr: + s = model.Site(s_r.Sites) + s_Arr.append(s) + + return s_Arr + + def get_site_by_code(self, site_code): + w_s = None + try: + s = self.db_session.query(odm2_models.Sites).\ + filter(odm2_models.Sites.SamplingFeatureCode == site_code).one() + except: + s = None + if s is not None: + w_s = model.Site(s) + + return w_s + + def get_sites_by_codes(self, site_codes_arr): + s_arr = [] + for site_code in site_codes_arr: + w_s = self.get_site_by_code(site_code) + if w_s is not None: + s_arr.append(w_s) + return s_arr + + def get_sites_by_box(self, west,south,east,north): + """ + north - ymax - latitude + south - ymin - latitude + west - xmin - longitude + east - xmax - longitude + """ + s_Arr = [] + s_rArr = self.db_session.query(odm2_models.MeasurementResults, + odm2_models.Sites).\ + distinct(odm2_models.Sites.SamplingFeatureID).\ + join(odm2_models.FeatureActions).\ + join(odm2_models.Specimens).\ + filter(odm2_models.Specimens.SamplingFeatureID == odm2_models.RelatedFeatures.SamplingFeatureID, + odm2_models.MeasurementResults.FeatureActionID == odm2_models.FeatureActions.FeatureActionID, + odm2_models.Sites.SamplingFeatureID == odm2_models.RelatedFeatures.RelatedFeatureID, + odm2_models.Sites.Latitude >= south, + odm2_models.Sites.Latitude <= north, + odm2_models.Sites.Longitude >= west, + odm2_models.Sites.Longitude <= east).all() + for s_r in s_rArr: + s = model.Site(s_r.Sites) + s_Arr.append(s) + + return s_Arr + + def get_variable_params(self, var_code): + """ + For machataria timeseries data: Variable code::unit id-sample medium + """ + unitid = None + samplemedium = None + items = re.split('::|-',var_code) + numofparams = len(items) + if numofparams == 1: + var_code = items[0] + elif numofparams == 2: + var_code = items[0] + unitid = items[1] + elif numofparams == 3: + var_code = items[0] + unitid = items[1] + samplemedium = items[2] + return var_code,unitid,samplemedium + + def get_variables_from_results(self,var_codes=None): + l_var_codes = None + if var_codes is not None: + if not isinstance(var_codes, list): + l_var_codes = [] + l_var_codes.append(var_codes) + else: + l_var_codes = var_codes + + r_m_Arr = [] + if l_var_codes is None: + r_m = self.db_session.query(odm2_models.MeasurementResults).\ + distinct(odm2_models.MeasurementResults.VariableID, + odm2_models.MeasurementResults.UnitsID, + odm2_models.MeasurementResults.SampledMediumCV).all() + r_m_Arr.append(r_m) + else: + for item in l_var_codes: + var_code, unitid, samplemedium = self.get_variable_params(item) + + q = self.db_session.query(odm2_models.MeasurementResults).\ + distinct(odm2_models.MeasurementResults.VariableID, + odm2_models.MeasurementResults.UnitsID, + odm2_models.MeasurementResults.SampledMediumCV).\ + join(odm2_models.Variables) + q = q.filter(odm2_models.MeasurementResults.VariableID == odm2_models.Variables.VariableID, + odm2_models.Variables.VariableCode == var_code) + + if unitid is not None: + q = q.filter(odm2_models.MeasurementResults.UnitsID == int(unitid)) + if samplemedium is not None: + q = q.filter(odm2_models.MeasurementResults.SampledMediumCV == samplemedium) + r_m = q.all() + r_m_Arr.append(r_m) + + v_arr = [] + if len(r_m_Arr) is not 0: + for result_m in r_m_Arr: + for result in result_m: + v = result.VariableObj + u = result.UnitsObj + s = result.SampledMediumCV + t = result.TimeAggregationIntervalUnitsObj + ti = result.TimeAggregationInterval + w_v = model.Variable(v,s,u,t,ti) + v_arr.append(w_v) + return v_arr + + def get_all_variables(self): + v_arr = self.get_variables_from_results() + return v_arr + + def get_variable_by_code(self, var_code): + w_v = None + v_arr = self.get_variables_from_results(var_code) + if len(v_arr) is not 0: + w_v = v_arr.pop() + return w_v + + def get_variables_by_codes(self, var_codes_arr): + v_arr = self.get_variables_from_results(var_codes_arr) + return v_arr + + def get_series_by_sitecode(self, site_code): + + site = self.get_site_by_code(site_code) + if site is None: + return None + r = self.db_session.query(odm2_models.MeasurementResults.VariableID.label('vid'), + odm2_models.MeasurementResults.UnitsID.label('unitid'), + func.min(odm2_models.Actions.BeginDateTime).label("begindate"), + func.max(odm2_models.Actions.EndDateTime).label("enddate")).\ + group_by(odm2_models.MeasurementResults.VariableID, + odm2_models.MeasurementResults.UnitsID).\ + join(odm2_models.FeatureActions).\ + join(odm2_models.Actions).\ + join(odm2_models.SamplingFeatures).\ + filter(odm2_models.SamplingFeatures.SamplingFeatureCode == site_code, + odm2_models.MeasurementResults.FeatureActionID == odm2_models.FeatureActions.FeatureActionID, + ) + #print(len(r.all())) + #print('number of values') + r = r.all() + + result = self.db_session.query(odm2_models.MeasurementResults).\ + distinct(odm2_models.MeasurementResults.VariableID, + odm2_models.MeasurementResults.UnitsID).\ + join(odm2_models.FeatureActions).\ + join(odm2_models.SamplingFeatures).\ + filter(odm2_models.SamplingFeatures.SamplingFeatureCode == site_code, + odm2_models.MeasurementResults.FeatureActionID == odm2_models.FeatureActions.FeatureActionID) + # print(len(result.all())) + result = result.all() + + r_arr = [] + aff = None + first_flag = True + for series in result: + for item in r: + if item.vid == series.VariableID and item.unitid == series.UnitsID: + if first_flag: + first_flag = False + aff = self.db_session.query(odm2_models.Affiliations).\ + filter(odm2_models.Affiliations.OrganizationID == series.FeatureActionObj.ActionObj.MethodObj.OrganizationID).first() + w_r = model.Series(series,aff,item.begindate,item.enddate) + w_r.Variable.DataType = self.get_match('datatype', w_r.Variable.DataType) + w_r.Variable.SampleMedium = self.get_match('samplemedium', w_r.Variable.SampleMedium) + w_r.SampleMedium = w_r.Variable.SampleMedium + r_arr.append(w_r) + return r_arr + + def get_series_by_sitecode_and_varcode(self, site_code, var_code): + + site = self.get_site_by_code(site_code) + if site is None: + return None + r = self.db_session.query(odm2_models.MeasurementResults.VariableID.label('vid'), + odm2_models.MeasurementResults.UnitsID.label('unitid'), + odm2_models.MeasurementResults.SampledMediumCV.label('samplemedium'), + func.min(odm2_models.Actions.BeginDateTime).label("begindate"), + func.max(odm2_models.Actions.EndDateTime).label("enddate")).\ + group_by(odm2_models.MeasurementResults.VariableID, + odm2_models.MeasurementResults.UnitsID, + odm2_models.MeasurementResults.SampledMediumCV).\ + join(odm2_models.FeatureActions).\ + join(odm2_models.Actions).\ + join(odm2_models.Specimens).\ + join(odm2_models.Variables).\ + filter(odm2_models.Specimens.SamplingFeatureID == odm2_models.RelatedFeatures.SamplingFeatureID, + odm2_models.MeasurementResults.FeatureActionID == odm2_models.FeatureActions.FeatureActionID, + odm2_models.RelatedFeatures.RelatedFeatureID == site.SiteID, + odm2_models.Variables.VariableCode == var_code).all() + + result = self.db_session.query(odm2_models.MeasurementResults).\ + distinct(odm2_models.MeasurementResults.VariableID, + odm2_models.MeasurementResults.UnitsID, + odm2_models.MeasurementResults.SampledMediumCV).\ + join(odm2_models.FeatureActions).\ + join(odm2_models.Variables).\ + join(odm2_models.Specimens).\ + join(odm2_models.Variables).\ + filter(odm2_models.Specimens.SamplingFeatureID == odm2_models.RelatedFeatures.SamplingFeatureID, + odm2_models.MeasurementResults.FeatureActionID == odm2_models.FeatureActions.FeatureActionID, + odm2_models.RelatedFeatures.RelatedFeatureID == site.SiteID, + odm2_models.Variables.VariableCode == var_code).all() + + r_arr = [] + aff = None + first_flag = True + for series in result: + for item in r: + if item.vid == series.VariableID and item.unitid == series.UnitsID: + if first_flag: + first_flag = False + aff = self.db_session.query(odm2_models.Affiliations).\ + filter(odm2_models.Affiliations.OrganizationID == series.FeatureActionObj.ActionObj.MethodObj.OrganizationID).first() + w_r = model.Series(series,aff,item.begindate,item.enddate) + r_arr.append(w_r) + + return r_arr + + def get_specimen_data(self): + q = self.db_session.query(odm2_models.MeasurementResultValues).\ + join(odm2_models.MeasurementResults).\ + join(odm2_models.FeatureActions).\ + join(odm2_models.Specimens).\ + join(odm2_models.Variables) + return q + + def get_datavalues(self, site_code, var_code, begin_date_time=None, + end_date_time=None): + + site = self.get_site_by_code(site_code) + var_code, unitid, samplemedium = self.get_variable_params(var_code) + if site is None: + return None + + if (not begin_date_time or not end_date_time): + try: + q = self.get_specimen_data() + q = q.filter(odm2_models.Specimens.SamplingFeatureID == odm2_models.RelatedFeatures.SamplingFeatureID, + odm2_models.MeasurementResults.FeatureActionID == odm2_models.FeatureActions.FeatureActionID, + odm2_models.RelatedFeatures.RelatedFeatureID == site.SiteID, + odm2_models.Variables.VariableCode == var_code) + if unitid is not None: + q = q.filter(odm2_models.MeasurementResults.UnitsID == int(unitid)) + if samplemedium is not None: + q = q.filter(odm2_models.MeasurementResults.SampledMediumCV == samplemedium) + valueResultArr = q.all() + except: + valueResultArr = [] + else: + begin_date_time = parse(begin_date_time) + end_date_time = parse(end_date_time) + try: + q = self.get_specimen_data() + q = q.filter(odm2_models.Specimens.SamplingFeatureID == odm2_models.RelatedFeatures.SamplingFeatureID, + odm2_models.MeasurementResults.FeatureActionID == odm2_models.FeatureActions.FeatureActionID, + odm2_models.RelatedFeatures.RelatedFeatureID == site.SiteID, + odm2_models.Variables.VariableCode == var_code, + odm2_models.MeasurementResultValues.ValueDateTime >= begin_date_time, + odm2_models.MeasurementResultValues.ValueDateTime <= end_date_time) + if unitid is not None: + q = q.filter(odm2_models.MeasurementResults.UnitsID == int(unitid)) + if samplemedium is not None: + q = q.filter(odm2_models.MeasurementResults.SampledMediumCV == samplemedium) + valueResultArr = q.all() + except: + valueResultArr = [] + + v_dict = {} + v_arr = [] + if len(valueResultArr) is not 0: + aff = None + first_flag = True + if unitid is None and samplemedium is None: + for valueResult in valueResultArr: + variable_key = '%s::%s-%s' % (valueResult.ResultObj.VariableObj.VariableCode, + valueResult.ResultObj.UnitsID, + valueResult.ResultObj.SampledMediumCV) + if first_flag: + first_flag = False + org_id = valueResult.ResultObj.FeatureActionObj.ActionObj.MethodObj.OrganizationID + aff = self.db_session.query(odm2_models.Affiliations).\ + filter(odm2_models.Affiliations.OrganizationID == org_id).first() + w_v = model.DataValue(valueResult,aff) + + if variable_key in v_dict: + a_list = v_dict.get(variable_key) + a_list.append(w_v) + else: + v_dict[variable_key] = [] + v_dict[variable_key].append(w_v) + return v_dict + else: + for valueResult in valueResultArr: + if first_flag: + first_flag = False + org_id = valueResult.ResultObj.FeatureActionObj.ActionObj.MethodObj.OrganizationID + aff = self.db_session.query(odm2_models.Affiliations).\ + filter(odm2_models.Affiliations.OrganizationID == org_id).first() + w_v = model.DataValue(valueResult,aff) + v_arr.append(w_v) + return v_arr + + def get_method_by_id(self, method_id): + m = self.db_session.query(odm2_models.Methods).\ + filter(odm2_models.Methods.MethodID == method_id).first() + w_m = model.Method(m) + return w_m + + def get_methods_by_ids(self, method_id_arr): + m = self.db_session.query(odm2_models.Methods).\ + filter(odm2_models.Methods.MethodID.in_(method_id_arr)).all() + m_arr = [] + for i in range(len(m)): + w_m = model.Method(m[i]) + m_arr.append(w_m) + return m_arr + + def get_source_by_id(self, source_id): + aff = self.db_session.query(odm2_models.Affiliations).\ + filter(odm2_models.Affiliations.AffiliationID == source_id).one() + w_aff = model.Source(aff) + return w_aff + + def get_sources_by_ids(self, source_id_arr): + aff = self.db_session.query(odm2_models.Affiliations).\ + filter(odm2_models.Affiliations.AffiliationID.in_(source_id_arr)).all() + aff_arr = [] + for i in range(len(aff)): + w_a = model.Source(aff[i]) + aff_arr.append(w_a) + return aff_arr + + def get_qualcontrollvl_by_id(self, qual_control_lvl_id): + pl = self.db_session.query(odm2_models.ProcessingLevels)\ + .filter(odm2_models.ProcessingLevels.ProcessingLevelID == qual_control_lvl_id).first() + w_pl = model.QualityControlLevel(pl) + return w_pl + + def get_qualcontrollvls_by_ids(self, qual_control_lvl_id_arr): + pl = self.db_session.query(odm2_models.ProcessingLevels)\ + .filter(odm2_models.ProcessingLevels.ProcessingLevelID.in_(qual_control_lvl_id_arr)).all() + pl_arr = [] + for i in range(len(pl)): + w_pl = model.QualityControlLevel(pl[i]) + pl_arr.append(w_pl) + return pl_arr \ No newline at end of file diff --git a/wof/examples/flask/odm2/measurement/spyned_1_1.py b/wof/examples/flask/odm2/measurement/spyned_1_1.py new file mode 100644 index 0000000..0c90d30 --- /dev/null +++ b/wof/examples/flask/odm2/measurement/spyned_1_1.py @@ -0,0 +1,384 @@ +from __future__ import (absolute_import, division, print_function) + +import io +import logging + +from lxml import etree + +from spyne.decorator import rpc +from spyne.model.complex import Array +from spyne.model.fault import Fault +from spyne.model.primitive import AnyXml, Boolean, Float, Unicode +from spyne.service import ServiceBase +from spyne.util import memoize + +import wof + +__author__ = 'cyoun' + +logger = logging.getLogger(__name__) + +NSDEF = 'xmlns="http://www.cuahsi.org/waterML/1.1/"' + + +@memoize +def TWOFService(wof_inst, T, T_name): + class WOFService(ServiceBase): + + @rpc(Array(Unicode), Unicode, _returns=AnyXml) + def GetSitesObject(ctx, site=None, authToken=None): + try: + if site is not None: + siteArg = ','.join(str(s) for s in site) + else: + siteArg = None + logging.debug(site) + logging.debug(siteArg) + siteResponse = wof_inst.create_get_site_response(siteArg) + outStream = io.StringIO() + siteResponse.export(outStream, 0, name_="sitesResponse", + namespacedef_=NSDEF) + return outStream.getvalue() + except Exception as inst: + if type(inst) == Fault: + raise inst + else: + raise Fault(faultstring=str(inst)) + + # This is the one that returns WITH + @rpc(Array(Unicode), Unicode, _returns=T) + def GetSites(ctx, site=None, authToken=None): + siteResult = WOFService.GetSitesObject(ctx, site, authToken) + if T_name == wof._SERVICE_PARAMS["r_type"]: + return siteResult + return siteResult.replace('\n', '') + + @rpc(Float, Float, Float, Float, Boolean, Unicode, _returns=AnyXml) + def GetSitesByBoxObject(ctx, west, south, east, north, + IncludeSeries, authToken=None): + try: + siteResponse = wof_inst.create_get_site_box_response( + west, south, east, north, IncludeSeries + ) + outStream = io.StringIO() + siteResponse.export(outStream, 0, name_="sitesResponse", + namespacedef_=NSDEF) + return outStream.getvalue() + except Exception as inst: + if type(inst) == Fault: + raise inst + else: + raise Fault(faultstring=str(inst)) + + @rpc(Float, Float, Float, Float, Boolean, Unicode, _returns=T) + def GetSitesByBox(ctx, west, south, east, north, + IncludeSeries, authToken=None): + sitesResult = WOFService.GetSitesByBoxObject( + ctx, west, south, east, north, IncludeSeries, authToken + ) + if T_name == wof._SERVICE_PARAMS["r_type"]: + return sitesResult + return sitesResult.replace('\n', '') + + @rpc(Unicode, Unicode, _returns=AnyXml) + def GetSiteInfoObject(ctx, site, authToken=None): + print('site!!!!') + print(site) + # print(ctx) + # print(authToken) + try: + siteInfoResponse = wof_inst.create_get_site_info_response(site) + print('site in try no 44') + print(site) + outStream = io.StringIO() + siteInfoResponse.export( + outStream, 0, name_="sitesResponse", namespacedef_=NSDEF + ) + return outStream.getvalue() + except Exception as inst: + if type(inst) == Fault: + raise inst + else: + print('here44') + print(inst) + print(siteInfoResponse) + # print(ctx) + # raise Fault(faultstring=str(inst)) + # print(wof_inst) + # print(str(inst)) + # print(siteInfoResponse) + print(site) + + @rpc(Unicode, Unicode, _returns=T) + def GetSiteInfo(ctx, site, authToken=None): + # print('ctx') + # print(ctx) + print('site') + print(site) + # print('authToken') + # print(authToken) + siteinfoResult = WOFService.GetSiteInfoObject(ctx, site, authToken) + if T_name == wof._SERVICE_PARAMS["r_type"]: + return siteinfoResult + return siteinfoResult.replace('\n', '') + + @rpc(Array(Unicode), Unicode, _returns=AnyXml) + def GetSiteInfoMultpleObject(ctx, site, authToken=None): + if site is not None: + siteArg = ','.join(str(s) for s in site) + else: + siteArg = None + try: + siteInfoResponse = \ + wof_inst.create_get_site_info_multiple_response(siteArg) + outStream = io.StringIO() + siteInfoResponse.export( + outStream, 0, name_="sitesResponse", namespacedef_=NSDEF + ) + return outStream.getvalue() + except Exception as inst: + if type(inst) == Fault: + raise inst + else: + raise Fault(faultstring=str(inst)) + + @rpc(Array(Unicode), Unicode, _returns=T) + def GetSiteInfoMultple(ctx, site, authToken=None): + sitesinfoResult = WOFService.GetSiteInfoMultpleObject(ctx, site, authToken) # noqa + if T_name == wof._SERVICE_PARAMS["r_type"]: + return sitesinfoResult + return sitesinfoResult.replace('\n', '') + + @rpc(Unicode, Unicode, _returns=AnyXml) + def GetVariableInfoObject(ctx, variable, authToken=None): + try: + variableInfoResponse = \ + wof_inst.create_get_variable_info_response(variable) + outStream = io.StringIO() + variableInfoResponse.export(outStream, 0, + name_="variablesResponse", + namespacedef_=NSDEF) + return outStream.getvalue() + except Exception as inst: + if type(inst) == Fault: + raise inst + else: + raise Fault(faultstring=str(inst)) + + @rpc(Unicode, Unicode, _returns=T) + def GetVariableInfo(ctx, variable, authToken=None): + varinfoResult = WOFService.GetVariableInfoObject(ctx, variable, authToken) # noqa + if T_name == wof._SERVICE_PARAMS["r_type"]: + return varinfoResult + return varinfoResult.replace('\n', '') + + @rpc(Unicode, _returns=AnyXml) + def GetVariablesObject(ctx, authToken=None): + try: + variableInfoResponse = wof_inst.create_get_variable_info_response() # noqa + outStream = io.StringIO() + variableInfoResponse.export(outStream, 0, + name_="variablesResponse", + namespacedef_=NSDEF) + return outStream.getvalue() + except Exception as inst: + if type(inst) == Fault: + raise inst + else: + raise Fault(faultstring=str(inst)) + + @rpc(Unicode, _returns=T) + def GetVariables(ctx, authToken=None): + varsResult = WOFService.GetVariablesObject(ctx, authToken) + if T_name == wof._SERVICE_PARAMS["r_type"]: + return varsResult + return varsResult.replace('\n', '') + + @rpc(Unicode, Unicode, Unicode, Unicode, Unicode, _returns=AnyXml) + def GetValuesObject(ctx, location, variable, startDate=None, endDate=None, authToken=None): # noqa + try: + timeSeriesResponse = wof_inst.create_get_values_response( + location, variable, startDate, endDate) + outStream = io.StringIO() + timeSeriesResponse.export( + outStream, 0, name_="timeSeriesResponse", + namespacedef_=NSDEF) + return outStream.getvalue() + except Exception as inst: + if type(inst) == Fault: + raise inst + else: + raise Fault(faultstring=str(inst)) + + @rpc(Unicode, Unicode, Unicode, Unicode, Unicode, _returns=T) + def GetValues(ctx, location, variable, startDate=None, endDate=None, authToken=None): # noqa + valuesResult = WOFService.GetValuesObject( + ctx, + location, + variable, + startDate, + endDate, + authToken + ) + if T_name == wof._SERVICE_PARAMS["r_type"]: + return valuesResult + return valuesResult.replace('\n', '') + + @rpc(Unicode, Unicode, Unicode, Unicode, _returns=AnyXml) + def GetValuesForASiteObject(ctx, site, startDate=None, endDate=None, authToken=None): # noqa + try: + timeSeriesResponse = wof_inst.create_get_values_site_response( + site, startDate, endDate) + outStream = io.StringIO() + timeSeriesResponse.export( + outStream, 0, name_="timeSeriesResponse", + namespacedef_=NSDEF) + return outStream.getvalue() + except Exception as inst: + if type(inst) == Fault: + raise inst + else: + raise Fault(faultstring=str(inst)) + + @rpc(Unicode, Unicode, Unicode, Unicode, _returns=T) + def GetValuesForASite(ctx, site, startDate, endDate, authToken=None): + valuesResult = WOFService.GetValuesForASiteObject( + ctx, + site, + startDate, + endDate, + authToken + ) + if T_name == wof._SERVICE_PARAMS["r_type"]: + return valuesResult + return valuesResult.replace('\n', '') + + def _on_method_return_xml(ctx): + # whatever etree element you return is the final xml + # response, so to prevent the extraneous ("%sResult" % + # method_name) result element, we just need to return an + # element that has that element removed and replaced with + # its child, which was the original response + + # TODO: how do we determine which method is being returned from? + # Since I don't know, I am doing a dumb test for each one + print('_on_method_return_xml') + if 'xml' and 'soap' not in list(ctx.out_protocol.type): + logger.info("protocol types: %s" % list(ctx.out_protocol.type)) + modify_return_xml_object(ctx) + print("protocol types: %s" % list(ctx.out_protocol.type)) + print(ctx._MethodContext__descriptor.name) + return ctx + + if ctx._MethodContext__descriptor.name == 'GetValuesObject': + print('GetValuesObject') + modify_method_getValueObject(ctx) + return ctx + if ctx._MethodContext__descriptor.name == 'GetVariableInfoObject': + print('GetVariableInfoObject') + modify_method_getVariableInfoObject(ctx) + return ctx + + if wof._SERVICE_PARAMS["s_type"] in list(ctx.out_protocol.type): + print('wof._SERVICE_PARAMS["s_type"]') + result_element_name_list = [ + 'GetSitesObjectResult', + 'GetSiteInfoObjectResult', + 'GetVariableInfoObjectResult', + 'GetValuesObjectResult', + 'GetSitesByBoxObjectResult', + 'GetSiteInfoMultpleObjectResult', + 'GetVariablesObjectResult', + 'GetValuesForASiteObjectResult'] + element = ctx.out_document + for result_element_name in result_element_name_list: + print('result_element_name') + print(result_element_name) + result_element = element.find( + './/{%s}%s' % ( + element.nsmap['tns'], + result_element_name + ) + ) + + if result_element is not None: + parent = result_element.getparent() + children = result_element.getchildren() + parent.replace(result_element, children[0]) + print('result_element_name') + print(result_element) + return ctx + print('nothing returned') + return + + def modify_method_getValueObject(ctx): + logger.info("Modify WOF11 GetValuesObject request") + result_element_name = 'GetValuesObjectResult' + response_element_name = 'GetValuesObjectResponse' + WML_NAMESPACE = "http://www.cuahsi.org/waterML/1.1/" + WML = "{%s}" % WML_NAMESPACE + NSMAP = {None: WML_NAMESPACE} + wrappingElement = etree.Element(WML + "TimeSeriesResponse", nsmap=NSMAP) # noqa + + element = ctx.out_document + element.nsmap[None] = NSMAP + response_element = element.find('.//{%s}%s' % (element.nsmap['tns'], response_element_name)) # noqa + result_element=element.find('.//{%s}%s' % (element.nsmap['tns'], result_element_name)) # noqa + parent = response_element.getparent() + parent.append(wrappingElement) # Add to tree. + children = result_element.getchildren() + wrappingElement.append(children[0]) + # parent.replace(result_element, children[0]) + + parent.remove(response_element) + + return ctx + + def modify_method_getVariableInfoObject(ctx): + logger.info("Modify WOF11 VariableInfoObject request") + result_element_name = 'GetVariableInfoObjectResult' + response_element_name = 'GetVariableInfoObjectResponse' + WML_NAMESPACE = "http://www.cuahsi.org/waterML/1.1/" + WML = "{%s}" % WML_NAMESPACE + NSMAP = {None: WML_NAMESPACE} + wrappingElement = etree.Element(WML + "VariablesResponse", nsmap=NSMAP) # noqa + + element = ctx.out_document + element.nsmap[None] = NSMAP + response_element = element.find('.//{%s}%s' % (element.nsmap['tns'], response_element_name)) # noqa + result_element=element.find('.//{%s}%s' % (element.nsmap['tns'], result_element_name)) # noqa + parent = response_element.getparent() + parent.append(wrappingElement) # Add to tree. + children = result_element.getchildren() + wrappingElement.append(children[0]) + # parent.replace(result_element, children[0]) + parent.remove(response_element) + + return ctx + + def modify_return_xml_object(ctx): + print('modify_return_xml_object') + method_name = ctx._MethodContext__descriptor.name + logger.info("Modify WOF11 %s request" % method_name) + result_element_name = '%sResult' % method_name + print('method name') + print(method_name) + element = ctx.out_document + print('find result element') + print(element.nsmap['ns0']) + result_element = element.find('.//{%s}%s' % (element.nsmap['ns0'], result_element_name)) # noqa + if result_element is None: + result_element = element.find('.//{%s}%s' % (element.nsmap['ns0'], 'GetSiteInfoResponse')) + print(result_element) + if not result_element is None: + children = result_element.getchildren() + ctx.out_document = children[0] + # print(ctx) + return ctx + + WOFService.event_manager.add_listener( + 'method_return_document', + _on_method_return_xml + ) + + return WOFService diff --git a/wof/examples/flask/odm2/measurement/sqlalch_odm2_models.py b/wof/examples/flask/odm2/measurement/sqlalch_odm2_models.py index 09cba0c..e91f767 100644 --- a/wof/examples/flask/odm2/measurement/sqlalch_odm2_models.py +++ b/wof/examples/flask/odm2/measurement/sqlalch_odm2_models.py @@ -51,7 +51,7 @@ def __init__(self, v=None, class Site(wof_base.BaseSite): - def __init__(self, s=None): + def __init__(self, s=None, aff=None): self.SiteID = s.SamplingFeatureID self.Latitude = s.Latitude self.Longitude = s.Longitude @@ -66,7 +66,10 @@ def __init__(self, s=None): sr.SRSName = s.SpatialReferenceObj.SRSName sr.Notes = s.SpatialReferenceObj.SRSDescription self.LatLongDatum = sr - + if aff is not None: + if aff.OrganizationObj.OrganizationTypeCV in ['Government agency', 'Research agency']: + self.AgencyName = aff.OrganizationObj.OrganizationName + self.AgencyCode = aff.OrganizationObj.OrganizationCode class Series(wof_base.BaseSeries): def __init__(self, r=None, aff=None,bdate=None,edate=None): fa_obj = r.FeatureActionObj @@ -74,19 +77,22 @@ def __init__(self, r=None, aff=None,bdate=None,edate=None): v_obj = r.VariableObj p_obj = r.ProcessingLevelObj - sf_obj = fa_obj.SamplingFeatureObj + # sf_obj = fa_obj.SamplingFeatureObj a_obj = fa_obj.ActionObj m_obj = a_obj.MethodObj - o_obj = m_obj.OrganizationObj + # o_obj = m_obj.OrganizationObj self.SeriesID = r.ResultID #self.SiteID = fa_obj.SamplingFeatureID #self.SiteCode = sf_obj.SamplingFeatureCode #self.SiteName = sf_obj.SamplingFeatureName + self.Variable = Variable(v_obj, r.SampledMediumCV, - u_obj) + v_tunit= r.TimeAggregationIntervalUnitsObj, + v_timeinterval=r.TimeAggregationInterval, v_unit=u_obj, + aggregationstatisticCV=r.AggregationStatisticCV, actiontypeCV=a_obj.ActionTypeCV) #self.VariableID = r.VariableID #self.VariableCode = v_obj.VariableCode #self.VariableName = v_obj.VariableNameCV @@ -98,14 +104,14 @@ def __init__(self, r=None, aff=None,bdate=None,edate=None): #self.MethodID = m_obj.MethodID #self.MethodDescription = m_obj.MethodDescription self.Method = Method(m_obj) - self.Organization = o_obj.OrganizationName - self.BeginDateTimeUTC = bdate.isoformat() #a_obj.BeginDateTime.isoformat() - if a_obj.EndDateTime is not None: - self.EndDateTimeUTC = edate.isoformat() #a_obj.EndDateTime.isoformat() + self.BeginDateTimeUTC = bdate # .isoformat() #a_obj.BeginDateTime.isoformat() + #if a_obj.EndDateTime is not None: + self.EndDateTimeUTC = edate #.isoformat() #a_obj.EndDateTime.isoformat() self.ValueCount = r.ValueCount if aff is not None: + if aff.OrganizationObj is not None: + self.Organization = aff.OrganizationObj.OrganizationName self.Source = Source(aff) - class DataValue(wof_base.BaseDataValue): def __init__(self,v,aff_obj=None): @@ -142,19 +148,26 @@ def __init__(self,u_obj): self.UnitsName = u_obj.UnitsName self.UnitsType = u_obj.UnitsTypeCV self.UnitsAbbreviation = u_obj.UnitsAbbreviation + self.UnitsTypeValidate = False class Source(wof_base.BaseSource): def __init__(self,aff_obj): self.SourceID = aff_obj.AffiliationID - self.Organization = aff_obj.OrganizationObj.OrganizationName - self.OrganizationCode = aff_obj.OrganizationObj.OrganizationCode - self.SourceCode = aff_obj.OrganizationObj.OrganizationCode - self.SourceDescription = aff_obj.OrganizationObj.OrganizationDescription - self.SourceLink = aff_obj.OrganizationObj.OrganizationLink - self.ContactName = '%s %s' % (aff_obj.PersonObj.PersonFirstName,aff_obj.PersonObj.PersonLastName) - self.Phone = aff_obj.PrimaryPhone + + if aff_obj.OrganizationObj is not None: + self.Organization = aff_obj.OrganizationObj.OrganizationName + self.SourceCode = self.SourceID + self.SourceDescription = aff_obj.OrganizationObj.OrganizationDescription + self.SourceLink = aff_obj.OrganizationObj.OrganizationLink + + self.ContactName = '%s %s' % (aff_obj.PersonObj.PersonFirstName, + aff_obj.PersonObj.PersonLastName) + + if aff_obj.PrimaryPhone is not None: + self.Phone = aff_obj.PrimaryPhone self.Email = aff_obj.PrimaryEmail - self.Address = aff_obj.PrimaryAddress + if aff_obj.PrimaryAddress is not None: + self.Address = aff_obj.PrimaryAddress #self.City = 'San Diego' #self.State = 'CA' #self.ZipCode = '92122' diff --git a/wof/examples/flask/odm2/timeseries/odm2_timeseries_dao.py b/wof/examples/flask/odm2/timeseries/odm2_timeseries_dao.py index b58bc53..50a7d62 100644 --- a/wof/examples/flask/odm2/timeseries/odm2_timeseries_dao.py +++ b/wof/examples/flask/odm2/timeseries/odm2_timeseries_dao.py @@ -199,7 +199,7 @@ def get_variables_from_results(self, var_codes=None): s = result.ResultObj.SampledMediumCV t = result.TimeAggregationIntervalUnitsObj ti = result.TimeAggregationInterval - ag = result.ResultObj.AggregationStatisticCV + ag = result.AggregationStatisticCV at = result.ResultObj.FeatureActionObj.ActionObj.ActionTypeCV w_v = model.Variable(v, s, u, t, ti, ag, at) w_v.DataType = self.get_match('datatype', w_v.DataType) @@ -277,6 +277,8 @@ def get_series_by_sitecode(self, site_code): w_r.Variable.SampleMedium = self.get_match('samplemedium', w_r.Variable.SampleMedium) w_r.SampleMedium = w_r.Variable.SampleMedium r_arr.append(w_r) + print('HERE') h + print(r_arr) return r_arr def get_series_by_sitecode_and_varcode(self, site_code, var_code): From b36ee8de1b7ff5260108fcb8c1808df56cde5b9f Mon Sep 17 00:00:00 2001 From: Miguel Leon Date: Wed, 28 Nov 2018 09:38:27 -0500 Subject: [PATCH 4/8] updates for python3 updates for python3 --- wof/WaterML_1_1.py | 26 ++-- wof/apps/waterml2.py | 2 +- wof/core.py | 18 ++- .../odm2/measurement/odm2_measurement_dao.py | 133 +++++++++++++----- .../odm2/measurement/sqlalch_odm2_models.py | 20 ++- 5 files changed, 137 insertions(+), 62 deletions(-) diff --git a/wof/WaterML_1_1.py b/wof/WaterML_1_1.py index bca47ae..442629e 100644 --- a/wof/WaterML_1_1.py +++ b/wof/WaterML_1_1.py @@ -389,7 +389,7 @@ def quote_xml(inStr): "Escape markup chars, but do not modify CDATA sections." if not inStr: return '' - s1 = (isinstance(inStr, basestring) and inStr or + s1 = (isinstance(inStr, str) and inStr or '%s' % inStr) s2 = '' pos = 0 @@ -413,7 +413,7 @@ def quote_xml_aux(inStr): def quote_attrib(inStr): - s1 = (isinstance(inStr, basestring) and inStr or + s1 = (isinstance(inStr, str) and inStr or '%s' % inStr) s1 = s1.replace(u'&', u'&') s1 = s1.replace(u'<', u'<') @@ -826,7 +826,7 @@ class VariableInfoType(GeneratedsSuper): superclass = None def __init__(self, metadataTime=None, oid=None, variableCode=None, variableName=None, variableDescription=None, valueType=None, dataType=None, generalCategory=None, sampleMedium=None, unit=None, options=None, note=None, related=None, extension=None, noDataValue=None, timeScale=None, speciation=None, categories=None, variableProperty=None): self.original_tagname_ = None - if isinstance(metadataTime, basestring): + if isinstance(metadataTime, str): initvalue_ = datetime.datetime.strptime(metadataTime, '%Y-%m-%dT%H:%M:%S') else: initvalue_ = metadataTime @@ -1321,7 +1321,7 @@ class QueryInfoType(GeneratedsSuper): superclass = None def __init__(self, creationTime=None, queryURL=None, criteria=None, note=None, extension=None): self.original_tagname_ = None - if isinstance(creationTime, basestring): + if isinstance(creationTime, str): initvalue_ = datetime.datetime.strptime(creationTime, '%Y-%m-%dT%H:%M:%S') else: initvalue_ = creationTime @@ -1701,22 +1701,22 @@ class TimePeriodType(GeneratedsSuper): superclass = None def __init__(self, beginDateTime=None, endDateTime=None, beginDateTimeUTC=None, endDateTimeUTC=None, extensiontype_=None): self.original_tagname_ = None - if isinstance(beginDateTime, basestring): + if isinstance(beginDateTime, str): initvalue_ = datetime.datetime.strptime(beginDateTime, '%Y-%m-%dT%H:%M:%S') else: initvalue_ = beginDateTime self.beginDateTime = initvalue_ - if isinstance(endDateTime, basestring): + if isinstance(endDateTime, str): initvalue_ = datetime.datetime.strptime(endDateTime, '%Y-%m-%dT%H:%M:%S') else: initvalue_ = endDateTime self.endDateTime = initvalue_ - if isinstance(beginDateTimeUTC, basestring): + if isinstance(beginDateTimeUTC, str): initvalue_ = datetime.datetime.strptime(beginDateTimeUTC, '%Y-%m-%dT%H:%M:%S') else: initvalue_ = beginDateTimeUTC self.beginDateTimeUTC = initvalue_ - if isinstance(endDateTimeUTC, basestring): + if isinstance(endDateTimeUTC, str): initvalue_ = datetime.datetime.strptime(endDateTimeUTC, '%Y-%m-%dT%H:%M:%S') else: initvalue_ = endDateTimeUTC @@ -1883,7 +1883,7 @@ class TimeSingleType(TimePeriodType): def __init__(self, beginDateTime=None, endDateTime=None, beginDateTimeUTC=None, endDateTimeUTC=None, timeSingle=None): self.original_tagname_ = None super(TimeSingleType, self).__init__(beginDateTime, endDateTime, beginDateTimeUTC, endDateTimeUTC, ) - if isinstance(timeSingle, basestring): + if isinstance(timeSingle, str): initvalue_ = datetime.datetime.strptime(timeSingle, '%Y-%m-%dT%H:%M:%S') else: initvalue_ = timeSingle @@ -3038,7 +3038,7 @@ def __init__(self, codedVocabularyTerm=None, sampleID=None, methodCode=None, qua self.methodCode = _cast(None, methodCode) self.qualityControlLevelCode = _cast(None, qualityControlLevelCode) self.methodID = _cast(int, methodID) - if isinstance(metadataTime, basestring): + if isinstance(metadataTime, str): initvalue_ = datetime.datetime.strptime(metadataTime, '%Y-%m-%dT%H:%M:%S') else: initvalue_ = metadataTime @@ -3049,13 +3049,13 @@ def __init__(self, codedVocabularyTerm=None, sampleID=None, methodCode=None, qua self.censorCode = _cast(None, censorCode) self.accuracyStdDev = _cast(float, accuracyStdDev) self.offsetTypeID = _cast(int, offsetTypeID) - if isinstance(dateTime, basestring): + if isinstance(dateTime, str): initvalue_ = datetime.datetime.strptime(dateTime, '%Y-%m-%dT%H:%M:%S') else: initvalue_ = dateTime self.dateTime = initvalue_ self.offsetTypeCode = _cast(None, offsetTypeCode) - if isinstance(dateTimeUTC, basestring): + if isinstance(dateTimeUTC, str): initvalue_ = datetime.datetime.strptime(dateTimeUTC, '%Y-%m-%dT%H:%M:%S') else: initvalue_ = dateTimeUTC @@ -7221,7 +7221,7 @@ class SiteInfoType(SourceInfoType): def __init__(self, metadataTime=None, oid=None, siteName=None, siteCode=None, timeZoneInfo=None, geoLocation=None, elevation_m=None, verticalDatum=None, note=None, extension=None, altname=None, siteType=None, siteProperty=None): self.original_tagname_ = None super(SiteInfoType, self).__init__() - if isinstance(metadataTime, basestring): + if isinstance(metadataTime, str): initvalue_ = datetime.datetime.strptime(metadataTime, '%Y-%m-%dT%H:%M:%S') else: initvalue_ = metadataTime diff --git a/wof/apps/waterml2.py b/wof/apps/waterml2.py index 70cd1f8..4f01b0a 100644 --- a/wof/apps/waterml2.py +++ b/wof/apps/waterml2.py @@ -1,6 +1,6 @@ from __future__ import (absolute_import, division, print_function) -import StringIO +from io import StringIO import logging import datetime import os diff --git a/wof/core.py b/wof/core.py index 153ba67..02d3de9 100644 --- a/wof/core.py +++ b/wof/core.py @@ -9,12 +9,6 @@ from dateutil.parser import parse -from lxml import etree -from lxml.etree import XMLParser -from lxml.etree import XMLSyntaxError - -import pytz - from spyne.application import Application from spyne.const.http import HTTP_405 from spyne.error import RequestNotAllowed @@ -27,6 +21,18 @@ from spyne.server.http import HttpTransportContext from spyne.server.wsgi import WsgiApplication +import sys +print('sys.path!!!!') +print(sys.path) + +import pytz +from lxml import etree +from lxml.etree import XMLParser +from lxml.etree import XMLSyntaxError + + + + from wof.WofWsdls import WofWSDL_1_0, WofWSDL_1_1 from wof.apps.spyned_1_0 import TWOFService as wml10 from wof.apps.spyned_1_1 import TWOFService as wml11 diff --git a/wof/examples/flask/odm2/measurement/odm2_measurement_dao.py b/wof/examples/flask/odm2/measurement/odm2_measurement_dao.py index d11122f..fd60ea5 100644 --- a/wof/examples/flask/odm2/measurement/odm2_measurement_dao.py +++ b/wof/examples/flask/odm2/measurement/odm2_measurement_dao.py @@ -18,7 +18,7 @@ from sqlalchemy.sql import func from sqlalchemy.orm import with_polymorphic - +from sqlalchemy.dialects import postgresql import re class Odm2Dao(BaseDao): @@ -36,7 +36,7 @@ def __init__(self, db_connection_string): self.yml_dict = yaml.load(yml) def __del__(self): self.db_session.close() - + def db_check(self): try: self.db_session.query(odm2_models.SamplingFeatures).first() @@ -63,7 +63,7 @@ def get_all_sites(self): join(odm2_models.FeatureActions).\ join(odm2_models.MeasurementResults).\ filter(odm2_models.FeatureActions.SamplingFeatureID == odm2_models.Sites.SamplingFeatureID, - odm2_models.MeasurementResults.FeatureActionID == odm2_models.FeatureActions.FeatureActionID).distinct() + odm2_models.MeasurementResults.FeatureActionID == odm2_models.FeatureActions.FeatureActionID).distinct() # print('site count: ' + str(len(s_rArr))) for s_r in s_rArr: s = model.Site(s_r) @@ -73,14 +73,11 @@ def get_all_sites(self): def get_site_by_code(self, site_code): w_s = None - print('HEREEEEEEE') try: s = self.db_session.query(odm2_models.Sites).\ filter(odm2_models.Sites.SamplingFeatureCode == site_code).one() except: s = None - print(site_code) - print('NO SITE') if s is not None: w_s = model.Site(s) @@ -184,8 +181,6 @@ def get_variables_from_results(self,var_codes=None): t = result.TimeAggregationIntervalUnitsObj ti = result.TimeAggregationInterval ag = result.AggregationStatisticCV - print('ag') - print(ag) at = result.FeatureActionObj.ActionObj.ActionTypeCV w_v = model.Variable(v, s, u, t, ti, ag, at) # w_v = model.Variable(v,s,u,t,ti) @@ -206,16 +201,19 @@ def get_variable_by_code(self, var_code): w_v = v_arr.pop() return w_v - def get_variables_by_codes(self, var_codes_arr): + def _get_variables_by_codes(self, var_codes_arr): v_arr = self.get_variables_from_results(var_codes_arr) return v_arr - def get_series_by_sitecode(self, site_code): + + + + def get_series_by_sitecode(self, site_code): site = self.get_site_by_code(site_code) - if site is None: - print('here OMG') - return None + # f site is None: + # print('here OMG') + # return None # odm2_models.MeasurementResults.SampledMediumCV # odm2_models.RelatedFeatures.RelatedFeatureID == site.SiteID @@ -231,21 +229,40 @@ def get_series_by_sitecode(self, site_code): group_by(odm2_models.MeasurementResults.VariableID, odm2_models.MeasurementResults.ResultID, odm2_models.Results.ResultID) - print('get the results') - print(len(result.all())) + # print('get the results') + # print(len(result.all())) # print(str(result.statement.compile(dialect=postgresql.dialect()))) result = result.all() - print('number of results') + # print('number of results') r_arr = [] aff = None first_flag = True # for series in result: # for item in r: - print('about to it results') + ids = [i.ResultID for i in result] + edt_dict = _get_msrv_enddatetimes(self.db_session, ids) + sdt_dict = _get_msrv_startdatetimes(self.db_session, ids) + + # print('about to it results') for i in range(len(result)): - # print('it result ' + str(result.Resultid)) + # print('it result ' + str(result[i].ResultID)) + try: + # q = self.db_session.query(odm2_models.MeasurementResultValues.ResultID, + # func.max( + # odm2_models.MeasurementResultValues.ValueDateTime)).filter( # noqa + # odm2_models.MeasurementResultValues.ResultID == result[i].ResultID).group_by(odm2_models.MeasurementResultValues.ResultID) + result[i].msrv_EndDateTime = edt_dict[result[i].ResultID] # q.first()[1]# [result[i].ResultID]# .ValueDateTime + # print('end date time') + # print(result[i].msrv_EndDateTime) + # print(str(q.statement.compile(dialect=postgresql.dialect()))) + # edt_dict[result[i].ResultID] + result[i].msrv_BeginDateTime = sdt_dict[result[i].ResultID] + except Exception as e: + print(str(e)) + continue + # print('set dates') if i == 0: - + aff = self.db_session.query(odm2_models.Affiliations). \ join(odm2_models.ActionBy). \ filter(odm2_models.ActionBy.ActionID == result[i].FeatureActionObj.ActionID).first() # noqa @@ -256,10 +273,14 @@ def get_series_by_sitecode(self, site_code): w_r.Variable.DataType = self.get_match('datatype', w_r.Variable.DataType) w_r.Variable.SampleMedium = self.get_match('samplemedium', w_r.Variable.SampleMedium) w_r.SampleMedium = w_r.Variable.SampleMedium - print(w_r.Variable) + # try: + # print(w_r.EndDateTimeUTC) + # print(w_r.BeginDateTimeUTC) + # except: + # print('Error!') # print(w_r.Definition) r_arr.append(w_r) - print('done') + # print('done') return r_arr def get_series_by_sitecode_and_varcode(self, site_code, var_code): @@ -267,7 +288,7 @@ def get_series_by_sitecode_and_varcode(self, site_code, var_code): site = self.get_site_by_code(site_code) if site is None: return None - print('HELLO HELLO') + # print('HELLO HELLO') r = self.db_session.query(odm2_models.MeasurementResults.VariableID.label('vid'), odm2_models.MeasurementResults.UnitsID.label('unitid'), odm2_models.MeasurementResults.SampledMediumCV.label('samplemedium'), @@ -333,18 +354,18 @@ def get_datavalues(self, site_code, var_code, begin_date_time=None, try: # print('HERE HERE') q = self.get_specimen_data() - print(len(q.all())) - print('all spcimen values') - print(site_code) - print(var_code) + # print(len(q.all())) + # print('all spcimen values') + # print(site_code) + # print(var_code) q = q.filter( odm2_models.MeasurementResults.FeatureActionID == odm2_models.FeatureActions.FeatureActionID, # noqa odm2_models.MeasurementResults.VariableID == odm2_models.Variables.VariableID, odm2_models.SamplingFeatures.SamplingFeatureCode == site_code, odm2_models.Variables.VariableCode == var_code). \ order_by(odm2_models.MeasurementResultValues.ValueDateTime) - print(len(q.all())) - print('number of values') + # print(len(q.all())) + # print('number of values') # q.filter(odm2_models.Specimens.SamplingFeatureID == odm2_models.RelatedFeatures.SamplingFeatureID, # odm2_models.MeasurementResults.FeatureActionID == odm2_models.FeatureActions.FeatureActionID, # odm2_models.RelatedFeatures.RelatedFeatureID == site.SiteID, @@ -353,36 +374,36 @@ def get_datavalues(self, site_code, var_code, begin_date_time=None, q = q.filter(odm2_models.MeasurementResults.UnitsID == int(unitid)) if samplemedium is not None: q = q.filter(odm2_models.MeasurementResults.SampledMediumCV == samplemedium) - print(len(q.all())) - print('number of values more filtered') + # print(len(q.all())) + # print('number of values more filtered') valueResultArr = q.all() except: valueResultArr = [] else: - print('parse dates') + # print('parse dates') begin_date_time = parse(begin_date_time) end_date_time = parse(end_date_time) - print(begin_date_time) - print(end_date_time) + # print(begin_date_time) + # print(end_date_time) try: # print(len(q.all())) # print('all spcimen values') - print(site_code) - print(var_code) + # print(site_code) + # print(var_code) q = self.get_specimen_data() q = q.filter(odm2_models.MeasurementResults.FeatureActionID == odm2_models.FeatureActions.FeatureActionID, # noqa odm2_models.MeasurementResults.VariableID == odm2_models.Variables.VariableID, odm2_models.SamplingFeatures.SamplingFeatureCode == site_code,odm2_models.Variables.VariableCode == var_code, odm2_models.MeasurementResultValues.ValueDateTime >= begin_date_time, odm2_models.MeasurementResultValues.ValueDateTime <= end_date_time).order_by(odm2_models.MeasurementResultValues.ValueDateTime) - print(len(q.all())) - print('number of values') + # print(len(q.all())) + # print('number of values') if unitid is not None: q = q.filter(odm2_models.MeasurementResults.UnitsID == int(unitid)) if samplemedium is not None: q = q.filter(odm2_models.MeasurementResults.SampledMediumCV == samplemedium) - print(len(q.all())) - print('number of filtered values') + # print(len(q.all())) + # print('number of filtered values') valueResultArr = q.all() except: valueResultArr = [] @@ -466,3 +487,37 @@ def get_qualcontrollvls_by_ids(self, qual_control_lvl_id_arr): w_pl = model.QualityControlLevel(pl[i]) pl_arr.append(w_pl) return pl_arr +def _get_msrv_enddatetimes(db_session, resultids): + """Extracts Latest DateTime from Timeseries Result Values. + + :param db_session: SQLAlchemy Session Object + :param resultids: List of result id. Ex. [1, 2, 3] + :return: Dictionary of End Date Time + """ + edt_dict = dict(db_session.query(odm2_models.MeasurementResultValues.ResultID, + func.max( + odm2_models.MeasurementResultValues.ValueDateTime)).filter( # noqa + odm2_models.MeasurementResultValues.ResultID.in_(resultids)). \ + group_by(odm2_models.MeasurementResultValues.ResultID).all()) + #edt_dict['timezone'] =db_session.query(odm2_models.MeasurementResultValues.ResultID, + # func.max( + # odm2_models.MeasurementResultValues.ValueDateTime)).filter( # noqa + # odm2_models.MeasurementResultValues.ResultID.in_(resultids)). \ + # group_by(odm2_models.MeasurementResultValues.ResultID).all() + + return edt_dict + + +def _get_msrv_startdatetimes(db_session, resultids): + """Extracts Latest DateTime from Timeseries Result Values. + + :param db_session: SQLAlchemy Session Object + :param resultids: List of result id. Ex. [1, 2, 3] +: return: Dictionary of End Date Time + """ + sdt_dict = dict(db_session.query(odm2_models.MeasurementResultValues.ResultID, + func.min( + odm2_models.MeasurementResultValues.ValueDateTime)).filter( # noqas + odm2_models.MeasurementResultValues.ResultID.in_(resultids)). \ + group_by(odm2_models.MeasurementResultValues.ResultID).all()) + return sdt_dict diff --git a/wof/examples/flask/odm2/measurement/sqlalch_odm2_models.py b/wof/examples/flask/odm2/measurement/sqlalch_odm2_models.py index e91f767..09070e2 100644 --- a/wof/examples/flask/odm2/measurement/sqlalch_odm2_models.py +++ b/wof/examples/flask/odm2/measurement/sqlalch_odm2_models.py @@ -1,6 +1,7 @@ from __future__ import (absolute_import, division, print_function) import wof.models as wof_base +from datetime import datetime, timedelta class Variable(wof_base.BaseVariable): @@ -14,7 +15,14 @@ def __init__(self, v=None, self.VariableCode = variable_key self.VariableName = v.VariableNameCV self.VariableDescription = v.VariableDefinition - self.NoDataValue = v.NoDataValue + try: + nodvfloat = float(v.NoDataValue) + if nodvfloat.is_integer(): + self.NoDataValue = int(v.NoDataValue) + else: + self.NoDataValue = v.NoDataValue + except: + self.NoDataValue = v.NoDataValue self.SampleMedium = VarSampleMedium self.DataType = v.VariableTypeCV self.Speciation = v.SpeciationCV @@ -104,9 +112,15 @@ def __init__(self, r=None, aff=None,bdate=None,edate=None): #self.MethodID = m_obj.MethodID #self.MethodDescription = m_obj.MethodDescription self.Method = Method(m_obj) - self.BeginDateTimeUTC = bdate # .isoformat() #a_obj.BeginDateTime.isoformat() + # self.BeginDateTimeUTC = bdate # .isoformat() #a_obj.BeginDateTime.isoformat() #if a_obj.EndDateTime is not None: - self.EndDateTimeUTC = edate #.isoformat() #a_obj.EndDateTime.isoformat() + # self.EndDateTimeUTC = edate #.isoformat() #a_obj.EndDateTime.isoformat() + try: + self.EndDateTimeUTC = r.msrv_EndDateTime.isoformat() # + timedelta(hours=r.) + self.BeginDateTimeUTC = r.msrv_BeginDateTime.isoformat() + except Exception as inst: + # print('date error') + print(inst) self.ValueCount = r.ValueCount if aff is not None: if aff.OrganizationObj is not None: From c32d42b6ca7f53e06143f4d5057c03a192a8d584 Mon Sep 17 00:00:00 2001 From: Miguel Leon Date: Wed, 28 Nov 2018 13:13:59 -0500 Subject: [PATCH 5/8] remove extra debugging statement remove extra debugging statement --- wof/core.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/wof/core.py b/wof/core.py index 02d3de9..cfed47c 100644 --- a/wof/core.py +++ b/wof/core.py @@ -21,9 +21,6 @@ from spyne.server.http import HttpTransportContext from spyne.server.wsgi import WsgiApplication -import sys -print('sys.path!!!!') -print(sys.path) import pytz from lxml import etree From cf73e7070d092a541569ef105ab69802c2d86eec Mon Sep 17 00:00:00 2001 From: Miguel Leon Date: Thu, 3 Jan 2019 10:40:30 -0500 Subject: [PATCH 6/8] remove debugging print lines. --- wof/examples/flask/odm2/timeseries/odm2_timeseries_dao.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/wof/examples/flask/odm2/timeseries/odm2_timeseries_dao.py b/wof/examples/flask/odm2/timeseries/odm2_timeseries_dao.py index 50a7d62..7257585 100644 --- a/wof/examples/flask/odm2/timeseries/odm2_timeseries_dao.py +++ b/wof/examples/flask/odm2/timeseries/odm2_timeseries_dao.py @@ -277,8 +277,6 @@ def get_series_by_sitecode(self, site_code): w_r.Variable.SampleMedium = self.get_match('samplemedium', w_r.Variable.SampleMedium) w_r.SampleMedium = w_r.Variable.SampleMedium r_arr.append(w_r) - print('HERE') h - print(r_arr) return r_arr def get_series_by_sitecode_and_varcode(self, site_code, var_code): From 71df96e37a55991e0bdf20dec600e2eb9136cf9f Mon Sep 17 00:00:00 2001 From: Miguel Leon Date: Sat, 9 Mar 2019 13:45:30 -0500 Subject: [PATCH 7/8] python 2 to 3 changes; remove sourcelink python 2 to 3 changes; remove sourcelink. CUAHSI HIS could not ingest WOF data when it includes a source link so it is commented out here. --- wof/WaterML.py | 4 +- wof/WaterML_1_1.py | 34 ++--- wof/apps/spyned_1_0.py | 4 +- wof/apps/waterml2.py | 2 +- wof/core_1_0.py | 4 +- wof/core_1_1.py | 13 +- .../flask/odm2/measurement/core_1_0.py | 4 +- .../flask/odm2/measurement/core_1_1.py | 5 +- .../odm2/measurement/odm2_measurement_dao.py | 122 ++++++++++++------ .../odm2/measurement/sqlalch_odm2_models.py | 3 +- .../flask/odm_1_1/sqlalch_odm_models.py | 2 +- wof/examples/flask/swis/swis_dao.py | 2 +- wof/flask/templates/wsdl_1_1_template.wsdl | 1 - wof/flask/templates/wsdl_temp.wsdl | 1 - wof/models.py | 2 +- 15 files changed, 124 insertions(+), 79 deletions(-) diff --git a/wof/WaterML.py b/wof/WaterML.py index df7a9b5..1fbfad7 100644 --- a/wof/WaterML.py +++ b/wof/WaterML.py @@ -89,7 +89,7 @@ def showIndent(outfile, level): def quote_xml(inStr): if not inStr: return '' - s1 = (isinstance(inStr, basestring) and inStr or + s1 = (isinstance(inStr, str) and inStr or '%s' % inStr) s1 = s1.replace('&', '&') s1 = s1.replace('<', '<') @@ -97,7 +97,7 @@ def quote_xml(inStr): return s1 def quote_attrib(inStr): - s1 = (isinstance(inStr, basestring) and inStr or + s1 = (isinstance(inStr, str) and inStr or '%s' % inStr) s1 = s1.replace('&', '&') s1 = s1.replace('<', '<') diff --git a/wof/WaterML_1_1.py b/wof/WaterML_1_1.py index 442629e..d345a87 100644 --- a/wof/WaterML_1_1.py +++ b/wof/WaterML_1_1.py @@ -4189,10 +4189,10 @@ def __init__(self, sourceID=None, sourceCode=None, organization=None, sourceDesc self.contactInformation = [] else: self.contactInformation = contactInformation - if sourceLink is None: - self.sourceLink = [] - else: - self.sourceLink = sourceLink + # if sourceLink is None: + # self.sourceLink = [] + # else: + # self.sourceLink = sourceLink self.citation = citation def factory(*args_, **kwargs_): if SourceType.subclass: @@ -4213,11 +4213,11 @@ def set_contactInformation(self, contactInformation): self.contactInformation = def add_contactInformation(self, value): self.contactInformation.append(value) def insert_contactInformation_at(self, index, value): self.contactInformation.insert(index, value) def replace_contactInformation_at(self, index, value): self.contactInformation[index] = value - def get_sourceLink(self): return self.sourceLink - def set_sourceLink(self, sourceLink): self.sourceLink = sourceLink - def add_sourceLink(self, value): self.sourceLink.append(value) - def insert_sourceLink_at(self, index, value): self.sourceLink.insert(index, value) - def replace_sourceLink_at(self, index, value): self.sourceLink[index] = value + # def get_sourceLink(self): return self.sourceLink + # def set_sourceLink(self, sourceLink): self.sourceLink = sourceLink + # def add_sourceLink(self, value): self.sourceLink.append(value) + # def insert_sourceLink_at(self, index, value): self.sourceLink.insert(index, value) + # def replace_sourceLink_at(self, index, value): self.sourceLink[index] = value def get_citation(self): return self.citation def set_citation(self, citation): self.citation = citation def get_sourceID(self): return self.sourceID @@ -4234,7 +4234,7 @@ def hasContent_(self): self.sourceDescription is not None or self.metadata is not None or self.contactInformation or - self.sourceLink or + # self.sourceLink or self.citation is not None ): return True @@ -4282,9 +4282,9 @@ def exportChildren(self, outfile, level, namespace_='', name_='SourceType', from if contactInformation_ is not None: contactInformation_.export(outfile, level, namespace_, name_='contactInformation', pretty_print=pretty_print) #for sourceLink_ in self.sourceLink: - if self.sourceLink is not None: - showIndent(outfile, level, pretty_print) - outfile.write(u'<%ssourceLink>%s%s' % (namespace_, self.gds_format_string(quote_xml(self.sourceLink), input_name='sourceLink'), namespace_, eol_)) + #if self.sourceLink is not None: + # showIndent(outfile, level, pretty_print) + # outfile.write(u'<%ssourceLink>%s%s' % (namespace_, self.gds_format_string(quote_xml(self.sourceLink), input_name='sourceLink'), namespace_, eol_)) if self.citation is not None: showIndent(outfile, level, pretty_print) @@ -4329,10 +4329,10 @@ def buildChildren(self, child_, node, nodeName_, fromsubclass_=False): obj_.build(child_) self.contactInformation.append(obj_) obj_.original_tagname_ = 'contactInformation' - elif nodeName_ == 'sourceLink': - sourceLink_ = child_.text - sourceLink_ = self.gds_validate_string(sourceLink_, node, 'sourceLink') - self.sourceLink.append(sourceLink_) + #elif nodeName_ == 'sourceLink': + # sourceLink_ = child_.text + # sourceLink_ = self.gds_validate_string(sourceLink_, node, 'sourceLink') + # self.sourceLink.append(sourceLink_) elif nodeName_ == 'citation': citation_ = child_.text citation_ = self.gds_validate_string(citation_, node, 'citation') diff --git a/wof/apps/spyned_1_0.py b/wof/apps/spyned_1_0.py index bfde032..140cb21 100644 --- a/wof/apps/spyned_1_0.py +++ b/wof/apps/spyned_1_0.py @@ -1,6 +1,6 @@ from __future__ import (absolute_import, division, print_function) -import StringIO +from io import StringIO import logging from spyne.decorator import rpc @@ -38,7 +38,7 @@ def GetSites(ctx, site=None, authToken=None): logging.debug(site) logging.debug(siteArg) siteResponse = wof_inst.create_get_site_response(siteArg) - outStream = StringIO.StringIO() + outStream = StringIO() siteResponse.export(outStream, 0, name_="sitesResponse", namespacedef_=NSDEF) return outStream.getvalue() diff --git a/wof/apps/waterml2.py b/wof/apps/waterml2.py index 4f01b0a..ecd3667 100644 --- a/wof/apps/waterml2.py +++ b/wof/apps/waterml2.py @@ -89,7 +89,7 @@ def GetValues(self, location, variable, startDate=None, #full format: %Y-%m-%dT%H:%M:%S.%f%z def isoformat(value, format='%Y-%m-%dT%H:%M:%S%z'): - if isinstance(value, basestring): + if isinstance(value, str): try: value = parse(value) except: diff --git a/wof/core_1_0.py b/wof/core_1_0.py index ecff1e4..5c0c5c6 100644 --- a/wof/core_1_0.py +++ b/wof/core_1_0.py @@ -427,8 +427,8 @@ def create_source_element(self, sourceResult): source = WaterML.SourceType( sourceID=sourceResult.SourceID, Organization=sourceResult.Organization, - SourceDescription=sourceResult.SourceDescription, - SourceLink=sourceResult.SourceLink) + SourceDescription=sourceResult.SourceDescription) + #SourceLink=sourceResult.SourceLink contactInfo = self.create_contact_info_element(sourceResult) diff --git a/wof/core_1_1.py b/wof/core_1_1.py index 4a7ca7d..a0745dc 100644 --- a/wof/core_1_1.py +++ b/wof/core_1_1.py @@ -89,7 +89,8 @@ def config_from_file(self, file_name): self.link = config.link def get_site_code(self, siteArg): - + if siteArg is None: + return None if ':' in siteArg: networkname, siteCode = siteArg.split(':', 1) networkname = networkname.lower() @@ -222,7 +223,7 @@ def to_bool(self, value): if isinstance(value, bool): return value - if not isinstance(value, basestring): + if not isinstance(value, str): value = False return value @@ -607,8 +608,8 @@ def create_source_element(self, sourceResult): sourceID=sourceResult.SourceID, sourceCode=sourceResult.SourceCode, organization=sourceResult.Organization, - sourceDescription=sourceResult.SourceDescription, - sourceLink=sourceResult.SourceLink) + sourceDescription=sourceResult.SourceDescription) # , + # sourceLink=sourceResult.SourceLink) contactInfo = self.create_contact_info_element(sourceResult) @@ -806,11 +807,11 @@ def create_series_element(self, seriesResult): try: beginDateTime = dateutil.parser.parse(beginDateTime) except Exception as inst: - logging.warn('bad datetime conversion on beginDateTime:' + inst.message) # noqa + logging.warning('bad datetime conversion on beginDateTime:' + str(inst)) # noqa try: endDateTime = dateutil.parser.parse(endDateTime) except Exception as inst: - logging.warn('bad datetime conversion on endDateTime:' + inst.message) # noqa + logging.warning('bad datetime conversion on endDateTime:' + str(inst)) # noqa # TimeInterval. if beginDateTime is None: diff --git a/wof/examples/flask/odm2/measurement/core_1_0.py b/wof/examples/flask/odm2/measurement/core_1_0.py index e2a719d..2935aef 100644 --- a/wof/examples/flask/odm2/measurement/core_1_0.py +++ b/wof/examples/flask/odm2/measurement/core_1_0.py @@ -427,8 +427,8 @@ def create_source_element(self, sourceResult): source = WaterML.SourceType( sourceID=sourceResult.SourceID, Organization=sourceResult.Organization, - SourceDescription=sourceResult.SourceDescription, - SourceLink=sourceResult.SourceLink) + SourceDescription=sourceResult.SourceDescription) + #SourceLink=sourceResult.SourceLink) contactInfo = self.create_contact_info_element(sourceResult) diff --git a/wof/examples/flask/odm2/measurement/core_1_1.py b/wof/examples/flask/odm2/measurement/core_1_1.py index de075b4..4b733a2 100644 --- a/wof/examples/flask/odm2/measurement/core_1_1.py +++ b/wof/examples/flask/odm2/measurement/core_1_1.py @@ -232,7 +232,7 @@ def to_bool(self, value): if isinstance(value, bool): return value - if not isinstance(value, basestring): + if not isinstance(value, str): value = False return value @@ -618,7 +618,8 @@ def create_source_element(self, sourceResult): sourceCode=sourceResult.SourceCode, organization=sourceResult.Organization, sourceDescription=sourceResult.SourceDescription, - sourceLink=sourceResult.SourceLink) + #sourceLink=sourceResult.SourceLink) + citation=sourceResult.Citation) contactInfo = self.create_contact_info_element(sourceResult) diff --git a/wof/examples/flask/odm2/measurement/odm2_measurement_dao.py b/wof/examples/flask/odm2/measurement/odm2_measurement_dao.py index fd60ea5..55d37f3 100644 --- a/wof/examples/flask/odm2/measurement/odm2_measurement_dao.py +++ b/wof/examples/flask/odm2/measurement/odm2_measurement_dao.py @@ -50,40 +50,59 @@ def get_match(self, cvkey, term): if term in v: return k return term + def get_all_sites(self): - s_Arr = [] - # s_rArr = self.db_session.query(odm2_models.MeasurementResults,odm2_models.Sites).\ - # distinct(odm2_models.Sites.SamplingFeatureID).\ - # join(odm2_models.FeatureActions).\ - # join(odm2_models.Specimens).\ - # filter(odm2_models.Specimens.SamplingFeatureID == odm2_models.RelatedFeatures.SamplingFeatureID, - # odm2_models.MeasurementResults.FeatureActionID == odm2_models.FeatureActions.FeatureActionID, - # odm2_models.Sites.SamplingFeatureID == odm2_models.RelatedFeatures.RelatedFeatureID).all() - s_rArr = self.db_session.query(odm2_models.Sites).\ - join(odm2_models.FeatureActions).\ - join(odm2_models.MeasurementResults).\ + """Get all wof sites from odm2 database. + + :return: List of WOF Sites + """ + self.db_check() + s_rArr = self.db_session.query(odm2_models.Sites). \ + join(odm2_models.FeatureActions). \ + join(odm2_models.MeasurementResults). \ filter(odm2_models.FeatureActions.SamplingFeatureID == odm2_models.Sites.SamplingFeatureID, - odm2_models.MeasurementResults.FeatureActionID == odm2_models.FeatureActions.FeatureActionID).distinct() - # print('site count: ' + str(len(s_rArr))) + odm2_models.MeasurementResults.FeatureActionID == odm2_models.FeatureActions.FeatureActionID).distinct() # noqa + + s_Arr = [] for s_r in s_rArr: s = model.Site(s_r) s_Arr.append(s) - return s_Arr def get_site_by_code(self, site_code): + """Get wof site from odm2 database by site code. + + :param site_code: Site Code Ex. 'USU-LBR-Mendon' + :return: WOF Site + """ + self.db_check() w_s = None try: - s = self.db_session.query(odm2_models.Sites).\ + s = self.db_session.query(odm2_models.Sites). \ filter(odm2_models.Sites.SamplingFeatureCode == site_code).one() + + aff = self.db_session.query(odm2_models.Affiliations). \ + join(odm2_models.ActionBy). \ + join(odm2_models.Actions). \ + join(odm2_models.FeatureActions). \ + join(odm2_models.Sites). \ + filter(odm2_models.Sites.SamplingFeatureCode == s.SamplingFeatureCode).first() + # filter(odm2_models.ActionBy.IsActionLead == True) : may be too strict, + # removed on 8/14/17 except: s = None + aff = None if s is not None: - w_s = model.Site(s) - + w_s = model.Site(s, aff) return w_s def get_sites_by_codes(self, site_codes_arr): + """Get wof sites from odm2 database by a list of site codes. + + :param site_codes_arr: List of Site Codes Ex. ['USU-LBR-Mendon', 'USU-LBR-Mendon2'] + :return: List of WOF Sites + """ + self.db_check() s_arr = [] for site_code in site_codes_arr: w_s = self.get_site_by_code(site_code) @@ -91,32 +110,32 @@ def get_sites_by_codes(self, site_codes_arr): s_arr.append(w_s) return s_arr - def get_sites_by_box(self, west,south,east,north): - """ - north - ymax - latitude - south - ymin - latitude - west - xmin - longitude - east - xmax - longitude + def get_sites_by_box(self, west, south, east, north): + """Get wof sites from odm2 database by a bounding box. + + :param north: north - ymax - latitude + :param south: south - ymin - latitude + :param west: west - xmin - longitude + :param east: east - xmax - longitude + :return: List of WOF Sites """ - s_Arr = [] - s_rArr = self.db_session.query(odm2_models.MeasurementResults, - odm2_models.Sites).\ - distinct(odm2_models.Sites.SamplingFeatureID).\ - join(odm2_models.FeatureActions).\ - join(odm2_models.Specimens).\ - filter(odm2_models.Specimens.SamplingFeatureID == odm2_models.RelatedFeatures.SamplingFeatureID, - odm2_models.MeasurementResults.FeatureActionID == odm2_models.FeatureActions.FeatureActionID, - odm2_models.Sites.SamplingFeatureID == odm2_models.RelatedFeatures.RelatedFeatureID, + self.db_check() + s_rArr = self.db_session.query(odm2_models.Sites). \ + join(odm2_models.FeatureActions). \ + join(odm2_models.MeasurementResults). \ + filter(odm2_models.FeatureActions.SamplingFeatureID == odm2_models.Sites.SamplingFeatureID, + odm2_models.MeasurementResults.FeatureActionID == odm2_models.FeatureActions.FeatureActionID, # noqa odm2_models.Sites.Latitude >= south, odm2_models.Sites.Latitude <= north, odm2_models.Sites.Longitude >= west, - odm2_models.Sites.Longitude <= east).all() + odm2_models.Sites.Longitude <= east).distinct() + s_Arr = [] for s_r in s_rArr: - s = model.Site(s_r.Sites) + s = model.Site(s_r) s_Arr.append(s) - return s_Arr + def get_variable_params(self, var_code): """ For machataria timeseries data: Variable code::unit id-sample medium @@ -459,13 +478,25 @@ def get_methods_by_ids(self, method_id_arr): return m_arr def get_source_by_id(self, source_id): - aff = self.db_session.query(odm2_models.Affiliations).\ + """Get wof source from odm2 database by Affiliation ID. + + :param source_id: Affiliation ID. + :return: A WOF Source + """ + self.db_check() + aff = self.db_session.query(odm2_models.Affiliations). \ filter(odm2_models.Affiliations.AffiliationID == source_id).one() w_aff = model.Source(aff) return w_aff def get_sources_by_ids(self, source_id_arr): - aff = self.db_session.query(odm2_models.Affiliations).\ + """Get wof source from odm2 database by a list of Affiliation ID's. + + :param source_id_arr: List of Affiliation ID. + :return: List WOF Source + """ + self.db_check() + aff = self.db_session.query(odm2_models.Affiliations). \ filter(odm2_models.Affiliations.AffiliationID.in_(source_id_arr)).all() aff_arr = [] for i in range(len(aff)): @@ -474,19 +505,32 @@ def get_sources_by_ids(self, source_id_arr): return aff_arr def get_qualcontrollvl_by_id(self, qual_control_lvl_id): - pl = self.db_session.query(odm2_models.ProcessingLevels)\ + """Get wof Quality Control Level from odm2 database by Processing Level ID. + + :param qual_control_lvl_id: Processing Level ID. + :return: A WOF Quality Control Level + """ + self.db_check() + pl = self.db_session.query(odm2_models.ProcessingLevels) \ .filter(odm2_models.ProcessingLevels.ProcessingLevelID == qual_control_lvl_id).first() w_pl = model.QualityControlLevel(pl) return w_pl def get_qualcontrollvls_by_ids(self, qual_control_lvl_id_arr): - pl = self.db_session.query(odm2_models.ProcessingLevels)\ + """Get wof Quality Control Level from odm2 database by a list of Processing Level ID's. + + :param qual_control_lvl_id_arr: List Processing Level ID. + :return: List of WOF Quality Control Level + """ + self.db_check() + pl = self.db_session.query(odm2_models.ProcessingLevels) \ .filter(odm2_models.ProcessingLevels.ProcessingLevelID.in_(qual_control_lvl_id_arr)).all() pl_arr = [] for i in range(len(pl)): w_pl = model.QualityControlLevel(pl[i]) pl_arr.append(w_pl) return pl_arr + def _get_msrv_enddatetimes(db_session, resultids): """Extracts Latest DateTime from Timeseries Result Values. diff --git a/wof/examples/flask/odm2/measurement/sqlalch_odm2_models.py b/wof/examples/flask/odm2/measurement/sqlalch_odm2_models.py index 09070e2..314908d 100644 --- a/wof/examples/flask/odm2/measurement/sqlalch_odm2_models.py +++ b/wof/examples/flask/odm2/measurement/sqlalch_odm2_models.py @@ -172,7 +172,7 @@ def __init__(self,aff_obj): self.Organization = aff_obj.OrganizationObj.OrganizationName self.SourceCode = self.SourceID self.SourceDescription = aff_obj.OrganizationObj.OrganizationDescription - self.SourceLink = aff_obj.OrganizationObj.OrganizationLink + # self.SourceLink = aff_obj.OrganizationObj.OrganizationLink self.ContactName = '%s %s' % (aff_obj.PersonObj.PersonFirstName, aff_obj.PersonObj.PersonLastName) @@ -182,6 +182,7 @@ def __init__(self,aff_obj): self.Email = aff_obj.PrimaryEmail if aff_obj.PrimaryAddress is not None: self.Address = aff_obj.PrimaryAddress + self.Citation = self.ContactName + ', ' + self.Organization + ', ' + aff_obj.OrganizationObj.OrganizationLink #self.City = 'San Diego' #self.State = 'CA' #self.ZipCode = '92122' diff --git a/wof/examples/flask/odm_1_1/sqlalch_odm_models.py b/wof/examples/flask/odm_1_1/sqlalch_odm_models.py index 8155fc2..9f4757e 100644 --- a/wof/examples/flask/odm_1_1/sqlalch_odm_models.py +++ b/wof/examples/flask/odm_1_1/sqlalch_odm_models.py @@ -136,7 +136,7 @@ class Source(Base, wof_base.BaseSource): SourceID = Column(Integer, primary_key=True) Organization = Column(String) SourceDescription = Column(String) - SourceLink = Column(String) + # SourceLink = Column(String) ContactName = Column(String) Phone = Column(String) Email = Column(String) diff --git a/wof/examples/flask/swis/swis_dao.py b/wof/examples/flask/swis/swis_dao.py index aefbe53..c319a8d 100644 --- a/wof/examples/flask/swis/swis_dao.py +++ b/wof/examples/flask/swis/swis_dao.py @@ -180,7 +180,7 @@ def get_source_by_id(self, source_id=1): source.Phone = self.contact_info['phone'] source.Email = self.contact_info['email'] source.Organization = self.contact_info['organization'] - source.SourceLink = self.contact_info['link'] + # source.SourceLink = self.contact_info['link'] source.SourceDescription = self.contact_info['description'] source.Address = self.contact_info['address'] source.City = self.contact_info['city'] diff --git a/wof/flask/templates/wsdl_1_1_template.wsdl b/wof/flask/templates/wsdl_1_1_template.wsdl index a14fb69..95ade0b 100644 --- a/wof/flask/templates/wsdl_1_1_template.wsdl +++ b/wof/flask/templates/wsdl_1_1_template.wsdl @@ -610,7 +610,6 @@ - diff --git a/wof/flask/templates/wsdl_temp.wsdl b/wof/flask/templates/wsdl_temp.wsdl index aef7f20..573f18c 100644 --- a/wof/flask/templates/wsdl_temp.wsdl +++ b/wof/flask/templates/wsdl_temp.wsdl @@ -629,7 +629,6 @@ - diff --git a/wof/models.py b/wof/models.py index 648b0a9..51e0e12 100644 --- a/wof/models.py +++ b/wof/models.py @@ -166,7 +166,7 @@ class BaseSource(object): SourceCode = None Organization = None SourceDescription = None - SourceLink = None + # SourceLink = None ContactName = None Phone = None Email = None From 98669342beab57e0b4260b8bca0740cb8352c3d7 Mon Sep 17 00:00:00 2001 From: Miguel Leon Date: Mon, 16 Aug 2021 09:38:09 -0400 Subject: [PATCH 8/8] updates from prod for timeseries dao updates from prod for timeseries dao --- wof/core_1_1.py | 19 ++-- .../odm2/timeseries/odm2_timeseries_dao.py | 103 +++++------------- .../odm2/timeseries/sqlalch_odm2_models.py | 39 +++---- 3 files changed, 50 insertions(+), 111 deletions(-) diff --git a/wof/core_1_1.py b/wof/core_1_1.py index a0745dc..49b1c55 100644 --- a/wof/core_1_1.py +++ b/wof/core_1_1.py @@ -94,10 +94,10 @@ def get_site_code(self, siteArg): if ':' in siteArg: networkname, siteCode = siteArg.split(':', 1) networkname = networkname.lower() - if self.network == networkname: - return siteCode - else: - return None + # if self.network == networkname: + return siteCode + # else: + # return None return siteArg def get_variable_code(self, varArg): @@ -223,7 +223,7 @@ def to_bool(self, value): if isinstance(value, bool): return value - if not isinstance(value, str): + if not isinstance(value, basestring): value = False return value @@ -608,8 +608,8 @@ def create_source_element(self, sourceResult): sourceID=sourceResult.SourceID, sourceCode=sourceResult.SourceCode, organization=sourceResult.Organization, - sourceDescription=sourceResult.SourceDescription) # , - # sourceLink=sourceResult.SourceLink) + sourceDescription=sourceResult.SourceDescription, + sourceLink=sourceResult.SourceLink) contactInfo = self.create_contact_info_element(sourceResult) @@ -807,11 +807,12 @@ def create_series_element(self, seriesResult): try: beginDateTime = dateutil.parser.parse(beginDateTime) except Exception as inst: - logging.warning('bad datetime conversion on beginDateTime:' + str(inst)) # noqa + logging.warn('bad datetime conversion on beginDateTime:' + str(inst) + + ' beginDateTime ' + str(beginDateTime)) # noqa try: endDateTime = dateutil.parser.parse(endDateTime) except Exception as inst: - logging.warning('bad datetime conversion on endDateTime:' + str(inst)) # noqa + logging.warn('bad datetime conversion on endDateTime:' + str(inst)) # noqa # TimeInterval. if beginDateTime is None: diff --git a/wof/examples/flask/odm2/timeseries/odm2_timeseries_dao.py b/wof/examples/flask/odm2/timeseries/odm2_timeseries_dao.py index 7257585..2793a84 100644 --- a/wof/examples/flask/odm2/timeseries/odm2_timeseries_dao.py +++ b/wof/examples/flask/odm2/timeseries/odm2_timeseries_dao.py @@ -3,8 +3,6 @@ from __future__ import (absolute_import, division, print_function) from dateutil.parser import parse -import os -import yaml from sqlalchemy import (create_engine, func) from sqlalchemy.orm import (scoped_session, sessionmaker) @@ -39,39 +37,21 @@ def __init__(self, db_connection_string): self.db_session = Session() - # Read in WaterML -> ODM2 CV Mapping - with open(os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'cvmap_wml_1_1.yml'))) as yml: - self.yml_dict = yaml.load(yml) - def __del__(self): """Closes Database Session.""" self.db_session.close() - def db_check(self): - try: - self.db_session.query(odm2_models.SamplingFeatures).first() - except: - self.db_session.rollback() - finally: - pass - - def get_match(self, cvkey, term): - for k, v in self.yml_dict[cvkey].items(): - if term in v: - return k - return term - def get_all_sites(self): """Get all wof sites from odm2 database. :return: List of WOF Sites """ - self.db_check() s_rArr = self.db_session.query(odm2_models.Sites). \ join(odm2_models.FeatureActions). \ join(odm2_models.TimeSeriesResults). \ filter(odm2_models.FeatureActions.SamplingFeatureID == odm2_models.Sites.SamplingFeatureID, - odm2_models.TimeSeriesResults.FeatureActionID == odm2_models.FeatureActions.FeatureActionID).distinct() # noqa + odm2_models.TimeSeriesResults.FeatureActionID == odm2_models.FeatureActions.FeatureActionID, + odm2_models.Results.ProcessingLevelID == 2).distinct() # noqa s_Arr = [] for s_r in s_rArr: @@ -85,25 +65,14 @@ def get_site_by_code(self, site_code): :param site_code: Site Code Ex. 'USU-LBR-Mendon' :return: WOF Site """ - self.db_check() w_s = None try: s = self.db_session.query(odm2_models.Sites). \ filter(odm2_models.Sites.SamplingFeatureCode == site_code).one() - - aff = self.db_session.query(odm2_models.Affiliations). \ - join(odm2_models.ActionBy). \ - join(odm2_models.Actions). \ - join(odm2_models.FeatureActions). \ - join(odm2_models.Sites). \ - filter(odm2_models.Sites.SamplingFeatureCode == s.SamplingFeatureCode).first() - # filter(odm2_models.ActionBy.IsActionLead == True) : may be too strict, - # removed on 8/14/17 except: s = None - aff = None if s is not None: - w_s = model.Site(s, aff) + w_s = model.Site(s) return w_s def get_sites_by_codes(self, site_codes_arr): @@ -112,7 +81,6 @@ def get_sites_by_codes(self, site_codes_arr): :param site_codes_arr: List of Site Codes Ex. ['USU-LBR-Mendon', 'USU-LBR-Mendon2'] :return: List of WOF Sites """ - self.db_check() s_arr = [] for site_code in site_codes_arr: w_s = self.get_site_by_code(site_code) @@ -129,12 +97,12 @@ def get_sites_by_box(self, west, south, east, north): :param east: east - xmax - longitude :return: List of WOF Sites """ - self.db_check() s_rArr = self.db_session.query(odm2_models.Sites). \ join(odm2_models.FeatureActions). \ join(odm2_models.TimeSeriesResults). \ filter(odm2_models.FeatureActions.SamplingFeatureID == odm2_models.Sites.SamplingFeatureID, odm2_models.TimeSeriesResults.FeatureActionID == odm2_models.FeatureActions.FeatureActionID, # noqa + odm2_models.Results.ProcessingLevelID == 2, odm2_models.Sites.Latitude >= south, odm2_models.Sites.Latitude <= north, odm2_models.Sites.Longitude >= west, @@ -151,35 +119,36 @@ def get_variables_from_results(self, var_codes=None): :param var_codes: List of Variable Codes Ex. ['TEMP', 'SAL'] :return: List of WOF Variables """ - self.db_check() # TODO: Need to refine this function; currently seeking database. l_var_codes = None if var_codes is not None: if not isinstance(var_codes, list): - l_var_codes = [var_codes] + l_var_codes = [] + l_var_codes.append(var_codes) else: l_var_codes = var_codes r_t_Arr = [] if l_var_codes is None: - if self.engine.name in ('postgresql', 'mssql'): + if self.engine.name is 'postgresql': r_t = self.db_session.query(odm2_models.TimeSeriesResultValues). \ - join(odm2_models.TimeSeriesResults). \ + join(odm2_models.TimeSeriesResults).filter(odm2_models.Results.ProcessingLevelID == 2). \ distinct(odm2_models.TimeSeriesResults.VariableID).all() else: r_t = self.db_session.query(odm2_models.TimeSeriesResultValues). \ - join(odm2_models.TimeSeriesResults). \ + join(odm2_models.TimeSeriesResults).filter(odm2_models.Results.ProcessingLevelID == 2). \ group_by(odm2_models.TimeSeriesResults.VariableID).all() r_t_Arr.append(r_t) else: for item in l_var_codes: - if self.engine.name in ('postgresql', 'mssql'): + if self.engine.name is 'postgresql': r_t = self.db_session.query(odm2_models.TimeSeriesResultValues). \ join(odm2_models.TimeSeriesResults). \ join(odm2_models.Variables). \ filter( odm2_models.Variables.VariableID == odm2_models.TimeSeriesResults.VariableID, - odm2_models.Variables.VariableCode == item). \ + odm2_models.Variables.VariableCode == item, + odm2_models.Results.ProcessingLevelID == 2). \ distinct(odm2_models.Variables.VariableID).all() else: r_t = self.db_session.query(odm2_models.TimeSeriesResultValues). \ @@ -187,7 +156,8 @@ def get_variables_from_results(self, var_codes=None): join(odm2_models.Variables). \ filter( odm2_models.Variables.VariableID == odm2_models.TimeSeriesResults.VariableID, - odm2_models.Variables.VariableCode == item). \ + odm2_models.Variables.VariableCode == item, + odm2_models.Results.ProcessingLevelID == 2). \ group_by(odm2_models.Variables.VariableID).all() r_t_Arr.append(r_t) v_arr = [] @@ -199,11 +169,7 @@ def get_variables_from_results(self, var_codes=None): s = result.ResultObj.SampledMediumCV t = result.TimeAggregationIntervalUnitsObj ti = result.TimeAggregationInterval - ag = result.AggregationStatisticCV - at = result.ResultObj.FeatureActionObj.ActionObj.ActionTypeCV - w_v = model.Variable(v, s, u, t, ti, ag, at) - w_v.DataType = self.get_match('datatype', w_v.DataType) - w_v.SampleMedium = self.get_match('samplemedium', w_v.SampleMedium) + w_v = model.Variable(v, s, u, t, ti) v_arr.append(w_v) return v_arr @@ -213,7 +179,6 @@ def get_all_variables(self): :return: List of WOF Variables """ - self.db_check() v_arr = self.get_variables_from_results() return v_arr @@ -223,7 +188,6 @@ def get_variable_by_code(self, var_code): :param var_code: Variable Codes Ex. 'TEMP' :return: WOF Variable """ - self.db_check() w_v = None v_arr = self.get_variables_from_results(var_code) if len(v_arr) is not 0: @@ -236,7 +200,6 @@ def get_variables_by_codes(self, var_codes_arr): :param var_codes_arr: List of Variable Codes Ex. ['TEMP', 'SAL'] :return: List of WOF Variables """ - self.db_check() v_arr = self.get_variables_from_results(var_codes_arr) return v_arr @@ -246,13 +209,13 @@ def get_series_by_sitecode(self, site_code): :param site_code: Site Code Ex. 'USU-LBR-Mendon' :return: List of WOF Series """ - self.db_check() r = self.db_session.query(odm2_models.TimeSeriesResults). \ join(odm2_models.FeatureActions). \ join(odm2_models.SamplingFeatures). \ filter( odm2_models.TimeSeriesResults.FeatureActionID == odm2_models.FeatureActions.FeatureActionID, - odm2_models.SamplingFeatures.SamplingFeatureCode == site_code). \ + odm2_models.SamplingFeatures.SamplingFeatureCode == site_code, + odm2_models.Results.ProcessingLevelID == 2). \ group_by(odm2_models.TimeSeriesResults.VariableID, odm2_models.TimeSeriesResults.ResultID, odm2_models.Results.ResultID).all() @@ -270,12 +233,8 @@ def get_series_by_sitecode(self, site_code): filter(odm2_models.ActionBy.ActionID == r[i].FeatureActionObj.ActionID).first() # noqa r[i].tsrv_EndDateTime = edt_dict[r[i].ResultID] - tsrv = self.db_session.query(odm2_models.TimeSeriesResultValues). \ - filter(odm2_models.TimeSeriesResults.ResultID == r[i].ResultID).first() - w_r = model.Series(r[i], aff, tsrv) - w_r.Variable.DataType = self.get_match('datatype', w_r.Variable.DataType) - w_r.Variable.SampleMedium = self.get_match('samplemedium', w_r.Variable.SampleMedium) - w_r.SampleMedium = w_r.Variable.SampleMedium + + w_r = model.Series(r[i], aff) r_arr.append(w_r) return r_arr @@ -286,7 +245,6 @@ def get_series_by_sitecode_and_varcode(self, site_code, var_code): :param var_code: Variable Code Ex. 'TEMP' :return: List of WOF Series """ - self.db_check() r = self.db_session.query(odm2_models.TimeSeriesResults). \ join(odm2_models.FeatureActions). \ join(odm2_models.SamplingFeatures). \ @@ -295,7 +253,8 @@ def get_series_by_sitecode_and_varcode(self, site_code, var_code): odm2_models.TimeSeriesResults.FeatureActionID == odm2_models.FeatureActions.FeatureActionID, odm2_models.TimeSeriesResults.VariableID == odm2_models.Variables.VariableID, odm2_models.SamplingFeatures.SamplingFeatureCode == site_code, - odm2_models.Variables.VariableCode == var_code). \ + odm2_models.Variables.VariableCode == var_code, + odm2_models.Results.ProcessingLevelID == 2). \ group_by(odm2_models.TimeSeriesResults.VariableID, odm2_models.TimeSeriesResults.ResultID, odm2_models.Results.ResultID).all() @@ -313,12 +272,8 @@ def get_series_by_sitecode_and_varcode(self, site_code, var_code): filter(odm2_models.ActionBy.ActionID == r[i].FeatureActionObj.ActionID).first() r[i].tsrv_EndDateTime = edt_dict[r[i].ResultID] - tsrv = self.db_session.query(odm2_models.TimeSeriesResultValues). \ - filter(odm2_models.TimeSeriesResults.ResultID == r[i].ResultID).first() - w_r = model.Series(r[i], aff, tsrv) - w_r.Variable.DataType = self.get_match('datatype', w_r.Variable.DataType) - w_r.Variable.SampleMedium = self.get_match('samplemedium', w_r.Variable.SampleMedium) - w_r.SampleMedium = w_r.Variable.SampleMedium + + w_r = model.Series(r[i], aff) r_arr.append(w_r) return r_arr @@ -333,7 +288,6 @@ def get_datavalues(self, site_code, var_code, :param end_date_time: End Time Ex. '2008-03-27 19:30:00.000' :return: List of WOF DataValue """ - self.db_check() if not begin_date_time or not end_date_time: try: valueResultArr = self.db_session.query(odm2_models.TimeSeriesResultValues). \ @@ -345,7 +299,8 @@ def get_datavalues(self, site_code, var_code, odm2_models.TimeSeriesResults.FeatureActionID == odm2_models.FeatureActions.FeatureActionID, # noqa odm2_models.TimeSeriesResults.VariableID == odm2_models.Variables.VariableID, odm2_models.SamplingFeatures.SamplingFeatureCode == site_code, - odm2_models.Variables.VariableCode == var_code). \ + odm2_models.Variables.VariableCode == var_code, + odm2_models.Results.ProcessingLevelID == 2). \ order_by(odm2_models.TimeSeriesResultValues.ValueDateTime).all() except: valueResultArr = [] @@ -363,6 +318,7 @@ def get_datavalues(self, site_code, var_code, odm2_models.TimeSeriesResults.VariableID == odm2_models.Variables.VariableID, odm2_models.SamplingFeatures.SamplingFeatureCode == site_code, odm2_models.Variables.VariableCode == var_code, + odm2_models.Results.ProcessingLevelID == 2, odm2_models.TimeSeriesResultValues.ValueDateTime >= begin_date_time, odm2_models.TimeSeriesResultValues.ValueDateTime <= end_date_time). \ order_by(odm2_models.TimeSeriesResultValues.ValueDateTime).all() @@ -381,7 +337,6 @@ def get_datavalues(self, site_code, var_code, join(odm2_models.ActionBy). \ filter(odm2_models.ActionBy.ActionID == act_id).first() w_v = model.DataValue(valueResult, aff) - w_v.CensorCode = self.get_match('censorcode', w_v.CensorCode) v_arr.append(w_v) return v_arr @@ -391,7 +346,6 @@ def get_method_by_id(self, method_id): :param method_id: Method ID. Ex. 'METHOD1' :return: A WOF Method """ - self.db_check() m = self.db_session.query(odm2_models.Methods). \ filter(odm2_models.Methods.MethodID == method_id).first() w_m = model.Method(m) @@ -403,7 +357,6 @@ def get_methods_by_ids(self, method_id_arr): :param method_id_arr: List of Method ID. Ex. ['METHOD1', 'METHOD2'] :return: List of WOF Method """ - self.db_check() m = self.db_session.query(odm2_models.Methods). \ filter(odm2_models.Methods.MethodID.in_(method_id_arr)).all() m_arr = [] @@ -418,7 +371,6 @@ def get_source_by_id(self, source_id): :param source_id: Affiliation ID. :return: A WOF Source """ - self.db_check() aff = self.db_session.query(odm2_models.Affiliations). \ filter(odm2_models.Affiliations.AffiliationID == source_id).one() w_aff = model.Source(aff) @@ -430,7 +382,6 @@ def get_sources_by_ids(self, source_id_arr): :param source_id_arr: List of Affiliation ID. :return: List WOF Source """ - self.db_check() aff = self.db_session.query(odm2_models.Affiliations). \ filter(odm2_models.Affiliations.AffiliationID.in_(source_id_arr)).all() aff_arr = [] @@ -445,7 +396,6 @@ def get_qualcontrollvl_by_id(self, qual_control_lvl_id): :param qual_control_lvl_id: Processing Level ID. :return: A WOF Quality Control Level """ - self.db_check() pl = self.db_session.query(odm2_models.ProcessingLevels) \ .filter(odm2_models.ProcessingLevels.ProcessingLevelID == qual_control_lvl_id).first() w_pl = model.QualityControlLevel(pl) @@ -457,7 +407,6 @@ def get_qualcontrollvls_by_ids(self, qual_control_lvl_id_arr): :param qual_control_lvl_id_arr: List Processing Level ID. :return: List of WOF Quality Control Level """ - self.db_check() pl = self.db_session.query(odm2_models.ProcessingLevels) \ .filter(odm2_models.ProcessingLevels.ProcessingLevelID.in_(qual_control_lvl_id_arr)).all() pl_arr = [] diff --git a/wof/examples/flask/odm2/timeseries/sqlalch_odm2_models.py b/wof/examples/flask/odm2/timeseries/sqlalch_odm2_models.py index 781607e..935c408 100644 --- a/wof/examples/flask/odm2/timeseries/sqlalch_odm2_models.py +++ b/wof/examples/flask/odm2/timeseries/sqlalch_odm2_models.py @@ -9,9 +9,9 @@ class Variable(wof_base.BaseVariable): """WOF Variable Model.""" - def __init__(self, v=None, VarSampleMedium=None, - v_unit=None, v_tunit=None, v_timeinterval=None, - aggregationstatisticCV=None, actiontypeCV=None): + def __init__(self, v=None, VarSampleMedium=None, + v_unit=None, v_tunit=None, v_timeinterval=None): + """Initialize WOF Variable Object. :param v: ODM2 Variable Object @@ -19,24 +19,18 @@ def __init__(self, v=None, VarSampleMedium=None, :param v_unit: ODM2 Unit Object :param v_tunit: ODM2 TimeAggregationIntervalUnits Object :param v_timeinterval: ODM2 TimeSeriesResultValues TimeAggregationInterval - :param aggregationstatisticCV: ODM2 Result AggregationStatisticCV - :param actiontypeCV: ODM2 Action ActionTypeCV + """ self.VariableID = v.VariableID self.VariableCode = v.VariableCode self.VariableName = v.VariableNameCV self.VariableDescription = v.VariableDefinition self.NoDataValue = v.NoDataValue + self.SampleMedium = VarSampleMedium + self.DataType = v.VariableTypeCV self.Speciation = v.SpeciationCV self.VariableUnitsID = v_unit.UnitsID - self.GeneralCategory = v.VariableTypeCV - - self.SampleMedium = VarSampleMedium - self.DataType = aggregationstatisticCV - self.ValueType = actiontypeCV - - self.GeneralCategoryValidate = False - self.ValueTypeValidate = False + if v_unit is not None: self.VariableUnits = Unit(v_unit) @@ -62,7 +56,7 @@ def __init__(self, v=None, VarSampleMedium=None, class Site(wof_base.BaseSite): """WOF Site Model.""" - def __init__(self, s=None, aff=None): + def __init__(self, s=None): """Initialize WOF Site Object. :param s: ODM2 SamplingFeature Object @@ -81,15 +75,12 @@ def __init__(self, s=None, aff=None): sr.SRSName = s.SpatialReferenceObj.SRSName sr.Notes = s.SpatialReferenceObj.SRSDescription self.LatLongDatum = sr - if aff is not None: - if aff.OrganizationObj.OrganizationTypeCV in ['Government agency', 'Research agency']: - self.AgencyName = aff.OrganizationObj.OrganizationName - self.AgencyCode = aff.OrganizationObj.OrganizationCode + class Series(wof_base.BaseSeries): """WOF Series Model.""" - def __init__(self, r=None, aff=None, tsrv=None): + def __init__(self, r=None, aff=None): """Initialize WOF Series Object. :param r: ODM2 TimeSeriesResult Object @@ -108,10 +99,9 @@ def __init__(self, r=None, aff=None, tsrv=None): # self.SiteID = fa_obj.SamplingFeatureID # self.SiteCode = sf_obj.SamplingFeatureCode # self.SiteName = sf_obj.SamplingFeatureName - self.Variable = Variable(v=v_obj, VarSampleMedium=r.SampledMediumCV, - v_tunit= tsrv.TimeAggregationIntervalUnitsObj, - v_timeinterval=tsrv.TimeAggregationInterval, v_unit=u_obj, - aggregationstatisticCV=r.AggregationStatisticCV, actiontypeCV=a_obj.ActionTypeCV) + self.Variable = Variable(v_obj, + r.SampledMediumCV, + u_obj) # self.VariableID = r.VariableID # self.VariableCode = v_obj.VariableCode # self.VariableName = v_obj.VariableNameCV @@ -202,7 +192,6 @@ def __init__(self, u_obj): self.UnitsType = u_obj.UnitsTypeCV self.UnitsAbbreviation = u_obj.UnitsAbbreviation - self.UnitsTypeValidate = False class Source(wof_base.BaseSource): @@ -216,7 +205,7 @@ def __init__(self, aff_obj): if aff_obj.OrganizationObj is not None: self.Organization = aff_obj.OrganizationObj.OrganizationName - self.SourceCode = self.SourceID + self.OrganizationCode = aff_obj.OrganizationObj.OrganizationCode self.SourceDescription = aff_obj.OrganizationObj.OrganizationDescription self.SourceLink = aff_obj.OrganizationObj.OrganizationLink