diff --git a/.buildinfo b/.buildinfo new file mode 100644 index 0000000..44a0dee --- /dev/null +++ b/.buildinfo @@ -0,0 +1,4 @@ +# Sphinx build info version 1 +# This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done. +config: ab81b1b815cedb7ca1fdb7811bd4966f +tags: 645f666f9bcd5a90fca523b33c5a78b7 diff --git a/.doctrees/environment.pickle b/.doctrees/environment.pickle new file mode 100644 index 0000000..21f9f4a Binary files /dev/null and b/.doctrees/environment.pickle differ diff --git a/.doctrees/index.doctree b/.doctrees/index.doctree new file mode 100644 index 0000000..73e2fa7 Binary files /dev/null and b/.doctrees/index.doctree differ diff --git a/.doctrees/introduction.doctree b/.doctrees/introduction.doctree new file mode 100644 index 0000000..d591160 Binary files /dev/null and b/.doctrees/introduction.doctree differ diff --git a/.doctrees/writer.doctree b/.doctrees/writer.doctree new file mode 100644 index 0000000..441f1ae Binary files /dev/null and b/.doctrees/writer.doctree differ diff --git a/.doctrees/wsgi.doctree b/.doctrees/wsgi.doctree new file mode 100644 index 0000000..785cce3 Binary files /dev/null and b/.doctrees/wsgi.doctree differ diff --git a/.doctrees/xml-rpc-server.doctree b/.doctrees/xml-rpc-server.doctree new file mode 100644 index 0000000..125ba02 Binary files /dev/null and b/.doctrees/xml-rpc-server.doctree differ diff --git a/.nojekyll b/.nojekyll new file mode 100644 index 0000000..e69de29 diff --git a/_modules/examples/datastore_config_server.html b/_modules/examples/datastore_config_server.html new file mode 100644 index 0000000..d70e2a3 --- /dev/null +++ b/_modules/examples/datastore_config_server.html @@ -0,0 +1,154 @@ + + + + + + + examples.datastore_config_server — HiSPARC-datastore 1.0.0 documentation + + + + + + + + + + + + +
+
+
+
+ +

Source code for examples.datastore_config_server

+#!/usr/local/bin/python
+"""Simple XML-RPC Server to run on the datastore server.
+
+This daemon should be run on HiSPARC's datastore server.  It will
+handle the cluster layouts and station passwords.  When an update is
+necessary, it will reload the HTTP daemon.
+
+The basis for this code was ripped from the python SimpleXMLRPCServer
+library documentation and extended.
+
+"""
+
+import hashlib
+import os
+import urllib.error
+import urllib.parse
+import urllib.request
+
+from xmlrpc.server import SimpleXMLRPCRequestHandler, SimpleXMLRPCServer
+
+HASH = '/tmp/hash_datastore'
+DATASTORE_CFG = '/databases/frome/station_list.csv'
+CFG_URL = 'https://data.hisparc.nl/config/datastore'
+DATASTORE_XMLRPC_SERVER = ('192.168.99.13', 8001)
+RELOAD_PATH = '/tmp/uwsgi-reload.me'
+
+
+

+[docs]
+def reload_datastore():
+    """Load datastore config and reload datastore, if necessary"""
+
+    datastore_cfg = urllib.request.urlopen(CFG_URL).read()
+    new_hash = hashlib.sha1(datastore_cfg).hexdigest()
+
+    try:
+        with open(HASH) as file:
+            old_hash = file.readline()
+    except OSError:
+        old_hash = None
+
+    if new_hash == old_hash:
+        return True
+    else:
+        with open(DATASTORE_CFG, 'w') as file:
+            file.write(datastore_cfg)
+
+        # reload uWSGI
+        with open(RELOAD_PATH, 'a'):
+            os.utime(RELOAD_PATH, None)
+
+        with open(HASH, 'w') as file:
+            file.write(new_hash)
+
+    return True

+
+
+
+if __name__ == '__main__':
+    # Restrict to a particular path.
+    class RequestHandler(SimpleXMLRPCRequestHandler):
+        rpc_paths = ('/RPC2',)
+
+    # Create server
+    server = SimpleXMLRPCServer(DATASTORE_XMLRPC_SERVER, requestHandler=RequestHandler)
+    server.register_introspection_functions()
+
+    server.register_function(reload_datastore)
+
+    # Run the server's main loop
+    server.serve_forever()
+
+ +
+
+
+
+ +
+
+ + + + \ No newline at end of file diff --git a/_modules/index.html b/_modules/index.html new file mode 100644 index 0000000..0d86565 --- /dev/null +++ b/_modules/index.html @@ -0,0 +1,86 @@ + + + + + + + Overview: module code — HiSPARC-datastore 1.0.0 documentation + + + + + + + + + + + + +
+
+
+
+ +

All modules for which code is available

+ + +
+
+
+
+ +
+
+ + + + \ No newline at end of file diff --git a/_modules/writer/storage.html b/_modules/writer/storage.html new file mode 100644 index 0000000..e726ee3 --- /dev/null +++ b/_modules/writer/storage.html @@ -0,0 +1,471 @@ + + + + + + + writer.storage — HiSPARC-datastore 1.0.0 documentation + + + + + + + + + + + + +
+
+
+
+ +

Source code for writer.storage

+"""Storage docstrings"""
+
+import tables
+
+
+class HisparcEvent(tables.IsDescription):
+    # DISCUSS: use of signed (dflt -1) vs unsigned (labview code)
+    event_id = tables.UInt32Col(pos=0)
+    timestamp = tables.Time32Col(pos=1)
+    nanoseconds = tables.UInt32Col(pos=2)
+    ext_timestamp = tables.UInt64Col(pos=3)
+    data_reduction = tables.BoolCol(pos=4)
+    trigger_pattern = tables.UInt32Col(pos=5)
+    baseline = tables.Int16Col(shape=4, dflt=-1, pos=6)
+    std_dev = tables.Int16Col(shape=4, dflt=-1, pos=7)
+    n_peaks = tables.Int16Col(shape=4, dflt=-1, pos=8)
+    pulseheights = tables.Int16Col(shape=4, dflt=-1, pos=9)
+    integrals = tables.Int32Col(shape=4, dflt=-1, pos=10)
+    traces = tables.Int32Col(shape=4, dflt=-1, pos=11)
+    event_rate = tables.Float32Col(pos=12)
+
+
+class HisparcError(tables.IsDescription):
+    """HiSPARC Error messages tables"""
+
+    event_id = tables.UInt32Col(pos=0)
+    timestamp = tables.Time32Col(pos=1)
+    messages = tables.Int32Col(dflt=-1, pos=2)
+
+
+class HisparcConfiguration(tables.IsDescription):
+    event_id = tables.UInt32Col(pos=0)
+    timestamp = tables.Time32Col(pos=1)
+    gps_latitude = tables.Float64Col(pos=2)
+    gps_longitude = tables.Float64Col(pos=3)
+    gps_altitude = tables.Float64Col(pos=4)
+    mas_version = tables.Int32Col(dflt=-1, pos=5)
+    slv_version = tables.Int32Col(dflt=-1, pos=6)
+    trig_low_signals = tables.UInt32Col(pos=7)
+    trig_high_signals = tables.UInt32Col(pos=8)
+    trig_external = tables.UInt32Col(pos=9)
+    trig_and_or = tables.BoolCol(pos=10)
+    precoinctime = tables.Float64Col(pos=11)
+    coinctime = tables.Float64Col(pos=12)
+    postcoinctime = tables.Float64Col(pos=13)
+    detnum = tables.UInt16Col(pos=14)
+    password = tables.Int32Col(dflt=-1, pos=15)
+    spare_bytes = tables.UInt8Col(pos=16)
+    use_filter = tables.BoolCol(pos=17)
+    use_filter_threshold = tables.BoolCol(pos=18)
+    reduce_data = tables.BoolCol(pos=19)
+    buffer = tables.Int32Col(dflt=-1, pos=20)
+    startmode = tables.BoolCol(pos=21)
+    delay_screen = tables.Float64Col(pos=22)
+    delay_check = tables.Float64Col(pos=23)
+    delay_error = tables.Float64Col(pos=24)
+    mas_ch1_thres_low = tables.Float64Col(pos=25)
+    mas_ch1_thres_high = tables.Float64Col(pos=26)
+    mas_ch2_thres_low = tables.Float64Col(pos=27)
+    mas_ch2_thres_high = tables.Float64Col(pos=28)
+    mas_ch1_inttime = tables.Float64Col(pos=29)
+    mas_ch2_inttime = tables.Float64Col(pos=30)
+    mas_ch1_voltage = tables.Float64Col(pos=31)
+    mas_ch2_voltage = tables.Float64Col(pos=32)
+    mas_ch1_current = tables.Float64Col(pos=33)
+    mas_ch2_current = tables.Float64Col(pos=34)
+    mas_comp_thres_low = tables.Float64Col(pos=35)
+    mas_comp_thres_high = tables.Float64Col(pos=36)
+    mas_max_voltage = tables.Float64Col(pos=37)
+    mas_reset = tables.BoolCol(pos=38)
+    mas_ch1_gain_pos = tables.UInt8Col(pos=39)
+    mas_ch1_gain_neg = tables.UInt8Col(pos=40)
+    mas_ch2_gain_pos = tables.UInt8Col(pos=41)
+    mas_ch2_gain_neg = tables.UInt8Col(pos=42)
+    mas_ch1_offset_pos = tables.UInt8Col(pos=43)
+    mas_ch1_offset_neg = tables.UInt8Col(pos=44)
+    mas_ch2_offset_pos = tables.UInt8Col(pos=45)
+    mas_ch2_offset_neg = tables.UInt8Col(pos=46)
+    mas_common_offset = tables.UInt8Col(pos=47)
+    mas_internal_voltage = tables.UInt8Col(pos=48)
+    mas_ch1_adc_gain = tables.Float64Col(pos=49)
+    mas_ch1_adc_offset = tables.Float64Col(pos=50)
+    mas_ch2_adc_gain = tables.Float64Col(pos=51)
+    mas_ch2_adc_offset = tables.Float64Col(pos=52)
+    mas_ch1_comp_gain = tables.Float64Col(pos=53)
+    mas_ch1_comp_offset = tables.Float64Col(pos=54)
+    mas_ch2_comp_gain = tables.Float64Col(pos=55)
+    mas_ch2_comp_offset = tables.Float64Col(pos=56)
+    slv_ch1_thres_low = tables.Float64Col(pos=57)
+    slv_ch1_thres_high = tables.Float64Col(pos=58)
+    slv_ch2_thres_low = tables.Float64Col(pos=59)
+    slv_ch2_thres_high = tables.Float64Col(pos=60)
+    slv_ch1_inttime = tables.Float64Col(pos=61)
+    slv_ch2_inttime = tables.Float64Col(pos=62)
+    slv_ch1_voltage = tables.Float64Col(pos=63)
+    slv_ch2_voltage = tables.Float64Col(pos=64)
+    slv_ch1_current = tables.Float64Col(pos=65)
+    slv_ch2_current = tables.Float64Col(pos=66)
+    slv_comp_thres_low = tables.Float64Col(pos=67)
+    slv_comp_thres_high = tables.Float64Col(pos=68)
+    slv_max_voltage = tables.Float64Col(pos=69)
+    slv_reset = tables.BoolCol(pos=70)
+    slv_ch1_gain_pos = tables.UInt8Col(pos=71)
+    slv_ch1_gain_neg = tables.UInt8Col(pos=72)
+    slv_ch2_gain_pos = tables.UInt8Col(pos=73)
+    slv_ch2_gain_neg = tables.UInt8Col(pos=74)
+    slv_ch1_offset_pos = tables.UInt8Col(pos=75)
+    slv_ch1_offset_neg = tables.UInt8Col(pos=76)
+    slv_ch2_offset_pos = tables.UInt8Col(pos=77)
+    slv_ch2_offset_neg = tables.UInt8Col(pos=78)
+    slv_common_offset = tables.UInt8Col(pos=79)
+    slv_internal_voltage = tables.UInt8Col(pos=80)
+    slv_ch1_adc_gain = tables.Float64Col(pos=81)
+    slv_ch1_adc_offset = tables.Float64Col(pos=82)
+    slv_ch2_adc_gain = tables.Float64Col(pos=83)
+    slv_ch2_adc_offset = tables.Float64Col(pos=84)
+    slv_ch1_comp_gain = tables.Float64Col(pos=85)
+    slv_ch1_comp_offset = tables.Float64Col(pos=86)
+    slv_ch2_comp_gain = tables.Float64Col(pos=87)
+    slv_ch2_comp_offset = tables.Float64Col(pos=88)
+
+
+class HisparcComparator(tables.IsDescription):
+    event_id = tables.UInt32Col(pos=0)
+    timestamp = tables.Time32Col(pos=1)
+    nanoseconds = tables.UInt32Col(pos=2)
+    ext_timestamp = tables.UInt64Col(pos=3)
+    device = tables.UInt8Col(pos=4)
+    comparator = tables.UInt8Col(pos=5)
+    count = tables.UInt16Col(pos=6)
+
+
+class HisparcSingle(tables.IsDescription):
+    event_id = tables.UInt32Col(pos=0)
+    timestamp = tables.Time32Col(pos=1)
+    mas_ch1_low = tables.Int32Col(dflt=-1, pos=2)
+    mas_ch1_high = tables.Int32Col(dflt=-1, pos=3)
+    mas_ch2_low = tables.Int32Col(dflt=-1, pos=4)
+    mas_ch2_high = tables.Int32Col(dflt=-1, pos=5)
+    slv_ch1_low = tables.Int32Col(dflt=-1, pos=6)
+    slv_ch1_high = tables.Int32Col(dflt=-1, pos=7)
+    slv_ch2_low = tables.Int32Col(dflt=-1, pos=8)
+    slv_ch2_high = tables.Int32Col(dflt=-1, pos=9)
+
+
+class HisparcSatellite(tables.IsDescription):
+    event_id = tables.UInt32Col(pos=0)
+    timestamp = tables.Time32Col(pos=1)
+    min_n = tables.UInt16Col(pos=2)
+    mean_n = tables.Float32Col(pos=3)
+    max_n = tables.UInt16Col(pos=4)
+    min_signal = tables.UInt16Col(pos=5)
+    mean_signal = tables.Float32Col(pos=6)
+    max_signal = tables.UInt16Col(pos=7)
+
+
+class WeatherEvent(tables.IsDescription):
+    event_id = tables.UInt32Col(pos=0)
+    timestamp = tables.Time32Col(pos=1)
+    temp_inside = tables.Float32Col(pos=2)
+    temp_outside = tables.Float32Col(pos=3)
+    humidity_inside = tables.Int16Col(pos=4)
+    humidity_outside = tables.Int16Col(pos=5)
+    barometer = tables.Float32Col(pos=6)
+    wind_dir = tables.Int16Col(pos=7)
+    wind_speed = tables.Int16Col(pos=8)
+    solar_rad = tables.Int16Col(pos=9)
+    uv = tables.Int16Col(pos=10)
+    evapotranspiration = tables.Float32Col(pos=11)
+    rain_rate = tables.Float32Col(pos=12)
+    heat_index = tables.Int16Col(pos=13)
+    dew_point = tables.Float32Col(pos=14)
+    wind_chill = tables.Float32Col(pos=15)
+
+
+class WeatherError(tables.IsDescription):
+    event_id = tables.UInt32Col(pos=0)
+    timestamp = tables.Time32Col(pos=1)
+    messages = tables.Int32Col(dflt=-1, pos=2)
+
+
+class WeatherConfig(tables.IsDescription):
+    event_id = tables.UInt32Col(pos=0)
+    timestamp = tables.Time32Col(pos=1)
+    com_port = tables.UInt8Col(pos=2)
+    baud_rate = tables.Int16Col(pos=3)
+    station_id = tables.UInt32Col(pos=4)
+    database_name = tables.Int32Col(dflt=-1, pos=5)
+    help_url = tables.Int32Col(dflt=-1, pos=6)
+    daq_mode = tables.BoolCol(pos=7)
+    latitude = tables.Float64Col(pos=8)
+    longitude = tables.Float64Col(pos=9)
+    altitude = tables.Float64Col(pos=10)
+    temperature_inside = tables.BoolCol(pos=11)
+    temperature_outside = tables.BoolCol(pos=12)
+    humidity_inside = tables.BoolCol(pos=13)
+    humidity_outside = tables.BoolCol(pos=14)
+    barometer = tables.BoolCol(pos=15)
+    wind_direction = tables.BoolCol(pos=16)
+    wind_speed = tables.BoolCol(pos=17)
+    solar_radiation = tables.BoolCol(pos=18)
+    uv_index = tables.BoolCol(pos=19)
+    evapotranspiration = tables.BoolCol(pos=20)
+    rain_rate = tables.BoolCol(pos=21)
+    heat_index = tables.BoolCol(pos=22)
+    dew_point = tables.BoolCol(pos=23)
+    wind_chill = tables.BoolCol(pos=24)
+    offset_inside_temperature = tables.Float32Col(pos=25)
+    offset_outside_temperature = tables.Float32Col(pos=26)
+    offset_inside_humidity = tables.Int16Col(pos=27)
+    offset_outside_humidity = tables.Int16Col(pos=28)
+    offset_wind_direction = tables.Int16Col(pos=29)
+    offset_station_altitude = tables.Float32Col(pos=30)
+    offset_bar_sea_level = tables.Float32Col(pos=31)
+
+
+class LightningEvent(tables.IsDescription):
+    event_id = tables.UInt32Col(pos=0)
+    timestamp = tables.Time32Col(pos=1)
+    corr_distance = tables.Int16Col(pos=2)
+    uncorr_distance = tables.Int16Col(pos=3)
+    uncorr_angle = tables.Float32Col(pos=4)
+    corr_angle = tables.Float32Col(pos=5)
+
+
+class LightningError(tables.IsDescription):
+    event_id = tables.UInt32Col(pos=0)
+    timestamp = tables.Time32Col(pos=1)
+    messages = tables.Int32Col(dflt=-1, pos=2)
+
+
+class LightningConfig(tables.IsDescription):
+    event_id = tables.UInt32Col(pos=0)
+    timestamp = tables.Time32Col(pos=1)
+    com_port = tables.UInt8Col(pos=2)
+    baud_rate = tables.Int16Col(pos=3)
+    station_id = tables.UInt32Col(pos=4)
+    database_name = tables.Int32Col(dflt=-1, pos=5)
+    help_url = tables.Int32Col(dflt=-1, pos=6)
+    daq_mode = tables.BoolCol(pos=7)
+    latitude = tables.Float64Col(pos=8)
+    longitude = tables.Float64Col(pos=9)
+    altitude = tables.Float64Col(pos=10)
+    squelch_seting = tables.Int32Col(pos=11)
+    close_alarm_distance = tables.Int32Col(pos=12)
+    severe_alarm_distance = tables.Int32Col(pos=13)
+    noise_beep = tables.BoolCol(pos=14)
+    minimum_gps_speed = tables.Int32Col(pos=15)
+    angle_correction = tables.Float32Col(pos=16)
+
+
+class LightningStatus(tables.IsDescription):
+    event_id = tables.UInt32Col(pos=0)
+    timestamp = tables.Time32Col(pos=1)
+    close_rate = tables.Int16Col(pos=2)
+    total_rate = tables.Int16Col(pos=3)
+    close_alarm = tables.BoolCol(pos=4)
+    sever_alarm = tables.BoolCol(pos=5)
+    current_heading = tables.Float32Col(pos=6)
+
+
+class LightningNoise(tables.IsDescription):
+    event_id = tables.UInt32Col(pos=0)
+    timestamp = tables.Time32Col(pos=1)
+
+
+

+[docs]
+def open_or_create_file(data_dir, date):
+    """Open an existing file or create a new one
+
+    This function opens an existing PyTables file according to the event
+    date.  If the file does not yet exist, a new one is created.
+
+    :param data_dir: the directory containing all data files
+    :param date: the event date
+
+    """
+    directory = data_dir / f'{date.year}/{date.month}'
+    file = directory / f'{date.year}_{date.month}_{date.day}.h5'
+
+    # Ensure dir and parent directories exist with mode rwxr-xr-x
+    directory.mkdir(mode=0o755, parents=True, exist_ok=True)
+
+    return tables.open_file(file, 'a')

+
+
+
+

+[docs]
+def get_or_create_station_group(file, cluster, station_id):
+    """Get an existing station group or create a new one
+
+    :param file: the PyTables data file
+    :param cluster: the name of the cluster
+    :param station_id: the station number
+
+    """
+    cluster = get_or_create_cluster_group(file, cluster)
+    node_name = f'station_{station_id}'
+    try:
+        station = file.get_node(cluster, node_name)
+    except tables.NoSuchNodeError:
+        station = file.create_group(cluster, node_name, f'HiSPARC station {station_id} data')
+        file.flush()
+
+    return station

+
+
+
+

+[docs]
+def get_or_create_cluster_group(file, cluster):
+    """Get an existing cluster group or create a new one
+
+    :param file: the PyTables data file
+    :param cluster: the name of the cluster
+
+    """
+    try:
+        hisparc = file.get_node('/', 'hisparc')
+    except tables.NoSuchNodeError:
+        hisparc = file.create_group('/', 'hisparc', 'HiSPARC data')
+        file.flush()
+
+    node_name = 'cluster_' + cluster.lower()
+    try:
+        cluster = file.get_node(hisparc, node_name)
+    except tables.NoSuchNodeError:
+        cluster = file.create_group(hisparc, node_name, f'HiSPARC cluster {cluster} data')
+        file.flush()
+
+    return cluster

+
+
+
+

+[docs]
+def get_or_create_node(file, cluster, node):
+    """Get an existing node or create a new one
+
+    :param file: the PyTables data file
+    :param cluster: the parent (cluster) node
+    :param node: the node (e.g. events, blobs)
+
+    """
+    try:
+        node = file.get_node(cluster, node)
+    except tables.NoSuchNodeError:
+        if node == 'events':
+            node = file.create_table(cluster, 'events', HisparcEvent, 'HiSPARC event data')
+        elif node == 'errors':
+            node = file.create_table(cluster, 'errors', HisparcError, 'HiSPARC error messages')
+        elif node == 'config':
+            node = file.create_table(cluster, 'config', HisparcConfiguration, 'HiSPARC configuration messages')
+        elif node == 'comparator':
+            node = file.create_table(cluster, 'comparator', HisparcComparator, 'HiSPARC comparator messages')
+        elif node == 'singles':
+            node = file.create_table(cluster, 'singles', HisparcSingle, 'HiSPARC single messages')
+        elif node == 'satellites':
+            node = file.create_table(cluster, 'satellites', HisparcSatellite, 'HiSPARC satellite messages')
+        elif node == 'blobs':
+            node = file.create_vlarray(cluster, 'blobs', tables.VLStringAtom(), 'HiSPARC binary data')
+        elif node == 'weather':
+            node = file.create_table(cluster, 'weather', WeatherEvent, 'HiSPARC weather data')
+        elif node == 'weather_errors':
+            node = file.create_table(cluster, 'weather_errors', WeatherError, 'HiSPARC weather error messages')
+        elif node == 'weather_config':
+            node = file.create_table(cluster, 'weather_config', WeatherConfig, 'HiSPARC weather configuration messages')
+        elif node == 'lightning':
+            node = file.create_table(cluster, 'lightning', LightningEvent, 'HiSPARC lightning data')
+        elif node == 'lightning_errors':
+            node = file.create_table(cluster, 'lightning_errors', LightningError, 'HiSPARC lightning error messages')
+        elif node == 'lightning_config':
+            node = file.create_table(
+                cluster,
+                'lightning_config',
+                LightningConfig,
+                'HiSPARC lightning configuration messages',
+            )
+        elif node == 'lightning_status':
+            node = file.create_table(cluster, 'lightning_status', LightningStatus, 'HiSPARC lightning status messages')
+        elif node == 'lightning_noise':
+            node = file.create_table(cluster, 'lightning_noise', LightningNoise, 'HiSPARC lightning noise messages')
+        file.flush()
+
+    return node

+
+
+ +
+
+
+
+ +
+
+ + + + \ No newline at end of file diff --git a/_modules/writer/store_events.html b/_modules/writer/store_events.html new file mode 100644 index 0000000..93b5895 --- /dev/null +++ b/_modules/writer/store_events.html @@ -0,0 +1,230 @@ + + + + + + + writer.store_events — HiSPARC-datastore 1.0.0 documentation + + + + + + + + + + + + +
+
+
+
+ +

Source code for writer.store_events

+import base64
+import calendar
+import logging
+
+from writer import storage
+from writer.upload_codes import eventtype_upload_codes
+
+logger = logging.getLogger('writer.store_events')
+
+MINIMUM_YEAR = 2020
+
+
+

+[docs]
+def store_event(datafile, cluster, station_id, event):
+    """Stores an event in the h5 filesystem
+
+    :param datafile: the h5 data file
+    :param cluster: the name of the cluster to which the station belongs
+    :param station_id: the id of the station this event belongs to
+    :param event: the event to store
+
+    """
+    eventheader = event['header']
+    eventdatalist = event['datalist']
+    eventtype = eventheader['eventtype_uploadcode']
+
+    try:
+        upload_codes = eventtype_upload_codes[eventtype]
+    except KeyError:
+        logger.error(f'Unknown event type: {eventtype}, discarding event (station: {station_id})')
+        return
+
+    parentnode = storage.get_or_create_station_group(datafile, cluster, station_id)
+    table = storage.get_or_create_node(datafile, parentnode, upload_codes['_tablename'])
+    blobs = storage.get_or_create_node(datafile, parentnode, 'blobs')
+
+    row = table.row
+    row['event_id'] = table.nrows + 1
+    # make a unix-like timestamp
+    timestamp = calendar.timegm(eventheader['datetime'].utctimetuple())
+    nanoseconds = eventheader['nanoseconds']
+    # make an extended timestamp, which is the number of nanoseconds since
+    # epoch
+    ext_timestamp = timestamp * 1_000_000_000 + nanoseconds
+    row['timestamp'] = timestamp
+
+    if upload_codes['_has_ext_time']:
+        # This is e.g. a HiSPARC coincidence or comparator message,
+        # extended timing information is available
+        row['nanoseconds'] = nanoseconds
+        row['ext_timestamp'] = ext_timestamp
+
+    # get default values for the data
+    data = {}
+    for key, value in upload_codes.items():
+        if key[0] != '_':
+            # private meta information starts with a _ (e.g. _tablename)
+            data[key] = row[value]
+
+    # process event data
+    for item in eventdatalist:
+        # uploadcode: EVENTRATE, PH1, IN3, etc.
+        uploadcode = item['data_uploadcode']
+        # value: actual data value
+        value = item['data']
+
+        if data_is_blob(uploadcode, upload_codes['_blobs']):
+            # data should be stored inside the blob array, ...
+            if uploadcode[:-1] == 'TR':
+                # traces are base64 encoded
+                value = base64.decodebytes(value.encode('iso-8859-1'))
+            else:
+                # blobs are bytestrings
+                value = value.encode('iso-8859-1')
+            blobs.append(value)
+            # ... with a pointer stored in the event table
+            value = len(blobs) - 1
+
+        if uploadcode[-1].isdigit():
+            # uploadcode: PH1, IN3, etc.
+            key, index = uploadcode[:-1], int(uploadcode[-1]) - 1
+            if key in data:
+                data[key][index] = value
+            else:
+                logger.warning(f'Datatype not known on server side: {key} ({eventtype})')
+        elif uploadcode in data:
+            # uploadcode: EVENTRATE, RED, etc.
+            data[uploadcode] = value
+        else:
+            logger.warning(f'Datatype not known on server side: {uploadcode} ({eventtype})')
+
+    # write data values to row
+    for key, value in upload_codes.items():
+        if key[0] != '_':
+            # private meta information starts with a _ (e.g. _tablename)
+            row[value] = data[key]
+
+    row.append()
+    table.flush()
+    blobs.flush()

+
+
+
+

+[docs]
+def data_is_blob(uploadcode, blob_types):
+    """Determine if data is a variable length binary value (blob)"""
+
+    if uploadcode[-1].isdigit():
+        if uploadcode[:-1] in blob_types:
+            return True
+    elif uploadcode in blob_types:
+        return True
+    return False

+
+
+
+

+[docs]
+def store_event_list(data_dir, station_id, cluster, event_list):
+    """Store a list of events"""
+
+    prev_date = None
+    datafile = None
+    for event in event_list:
+        try:
+            timestamp = event['header']['datetime']
+            if timestamp:
+                date = timestamp.date()
+                if date.year < MINIMUM_YEAR:
+                    logger.error(f'Old event ({date}), discarding event (station: {station_id})')
+                    continue
+                if date != prev_date:
+                    if datafile:
+                        datafile.close()
+                    datafile = storage.open_or_create_file(data_dir, date)
+                    prev_date = date
+                store_event(datafile, cluster, station_id, event)
+            else:
+                logger.error(f'Strange event (no timestamp!), discarding event (station: {station_id})')
+        except Exception:
+            logger.exception(f'Cannot process event, discarding event (station: {station_id})')
+
+    if datafile:
+        datafile.close()

+
+
+ +
+
+
+
+ +
+
+ + + + \ No newline at end of file diff --git a/_modules/writer/writer_app.html b/_modules/writer/writer_app.html new file mode 100644 index 0000000..480e58e --- /dev/null +++ b/_modules/writer/writer_app.html @@ -0,0 +1,196 @@ + + + + + + + writer.writer_app — HiSPARC-datastore 1.0.0 documentation + + + + + + + + + + + + +
+
+
+
+ +

Source code for writer.writer_app

+"""datastore writer application
+
+This module empties the station data `incoming` queue and writes the
+data into HDF5 files using PyTables.
+
+"""
+
+import configparser
+import logging
+import logging.handlers
+import pickle
+import time
+
+from pathlib import Path
+
+from writer.store_events import store_event_list
+
+LEVELS = {
+    'debug': logging.DEBUG,
+    'info': logging.INFO,
+    'warning': logging.WARNING,
+    'error': logging.ERROR,
+    'critical': logging.CRITICAL,
+}
+
+logger = logging.getLogger('writer')
+formatter = logging.Formatter('%(asctime)s %(name)s[%(process)d].%(funcName)s.%(levelname)s: %(message)s')
+
+
+

+[docs]
+def writer(configfile):
+    """hisparc datastore writer application
+
+    This script polls ``/datatore/frome/incoming`` for incoming data written
+    by the WSGI app. It then store the data into the raw datastore.
+
+    Configuration is read from the datastore configuation file (usually
+    `config.ini`):
+
+    .. include:: ../examples/config.ini
+       :literal:
+
+    """
+    # set up config
+    global config
+    config = configparser.ConfigParser()
+    config.read(configfile)
+
+    # set up logger
+    file = config.get('General', 'log') + '-writer'
+    handler = logging.handlers.TimedRotatingFileHandler(file, when='midnight', backupCount=14)
+    handler.setFormatter(formatter)
+    logger.addHandler(handler)
+    level = LEVELS.get(config.get('General', 'loglevel'), logging.NOTSET)
+    logger.setLevel(level=level)
+
+    data_dir = Path(config.get('General', 'data_dir'))
+    queue = data_dir / 'incoming'
+    partial_queue = data_dir / 'partial'
+
+    sleep_duration = config.getint('Writer', 'sleep')
+
+    # writer process
+    try:
+        while True:
+            entries = queue.iterdir()
+
+            if not entries:
+                time.sleep(sleep_duration)
+
+            for entry in entries:
+                partial_path = partial_queue / entry.name
+                entry.rename(partial_path)
+
+                process_data(partial_path, data_dir)
+                partial_path.unlink()
+
+    except Exception:
+        logger.exception('Exception occured, quitting.')

+
+
+
+

+[docs]
+def process_data(file, data_dir):
+    """Read data from a pickled object and store store in raw datastore"""
+    with file.open('rb') as handle:
+        try:
+            data = pickle.load(handle)
+        except UnicodeDecodeError:
+            logger.debug('Data seems to be pickled using python 2. Decoding.')
+            data = decode_object(pickle.load(handle, encoding='bytes'))
+
+    logger.debug(f"Processing data for station {data['station_id']}")
+    store_event_list(data_dir, data['station_id'], data['cluster'], data['event_list'])

+
+
+
+

+[docs]
+def decode_object(o):
+    """recursively decode all bytestrings in object"""
+
+    if isinstance(o, bytes):
+        return o.decode()
+    elif isinstance(o, dict):
+        return {decode_object(k): decode_object(v) for k, v in o.items()}
+    elif isinstance(o, list):
+        return [decode_object(obj) for obj in o]
+    else:
+        return o

+
+
+ +
+
+
+
+ +
+
+ + + + \ No newline at end of file diff --git a/_modules/wsgi/wsgi_app.html b/_modules/wsgi/wsgi_app.html new file mode 100644 index 0000000..3306c14 --- /dev/null +++ b/_modules/wsgi/wsgi_app.html @@ -0,0 +1,299 @@ + + + + + + + wsgi.wsgi_app — HiSPARC-datastore 1.0.0 documentation + + + + + + + + + + + + +
+
+
+
+ +

Source code for wsgi.wsgi_app

+import configparser
+import csv
+import hashlib
+import logging
+import logging.handlers
+import os
+import pickle
+import shutil
+import tempfile
+import urllib.parse
+
+from pathlib import Path
+
+from . import rcodes
+
+LEVELS = {
+    'debug': logging.DEBUG,
+    'info': logging.INFO,
+    'warning': logging.WARNING,
+    'error': logging.ERROR,
+    'critical': logging.CRITICAL,
+}
+
+logger = logging.getLogger('wsgi_app')
+formatter = logging.Formatter('%(asctime)s %(name)s[%(process)d].%(funcName)s.%(levelname)s: %(message)s')
+
+MINIMUM_YEAR = 2020
+
+
+

+[docs]
+def application(environ, start_response, configfile):
+    """The hisparc upload application
+
+    This handler is called by uWSGI whenever someone requests our URL.
+
+    First, we generate a dictionary of POSTed variables and try to read out the
+    station_id, password, checksum and data. When we do a readline(), we
+    already read out the entire datastream. I don't know if we can first check
+    on station_id/password combinations before reading out the datastream
+    without setting up a bidirectional communication channel.
+
+    When the checksum matches, we unpickle the event_list and pass everything
+    on to store_event_list.
+
+    """
+    do_init(configfile)
+
+    # start http response
+    status = '200 OK'
+    response_headers = [('Content-type', 'text/plain')]
+    start_response(status, response_headers)
+
+    # read data from the POST variables
+    post_input = environ['wsgi.input'].readline().decode()
+    post_data = urllib.parse.parse_qs(post_input)
+
+    # process POST data
+    try:
+        data = post_data['data'][0]
+        checksum = post_data['checksum'][0]
+        station_id = int(post_data['station_id'][0])
+        password = post_data['password'][0]
+    except (KeyError, EOFError):
+        logger.debug('POST (vars) error')
+        return [rcodes.RC_ISE_INV_POSTDATA]
+
+    try:
+        cluster, station_password = station_list[station_id]
+    except KeyError:
+        logger.debug(f'Station {station_id} is unknown')
+        return [rcodes.RC_PE_INV_STATIONID]
+
+    if station_password != password:
+        logger.debug(f'Station {station_id}: password mismatch: {password}')
+        return [rcodes.RC_PE_INV_AUTHCODE]
+    else:
+        our_checksum = hashlib.md5(data.encode('iso-8859-1')).hexdigest()
+        if our_checksum != checksum:
+            logger.debug(f'Station {station_id}: checksum mismatch')
+            return [rcodes.RC_PE_INV_INPUT]
+        else:
+            try:
+                try:
+                    event_list = pickle.loads(data.encode('iso-8859-1'))
+                except UnicodeDecodeError:
+                    # string was probably pickled on python 2.
+                    # decode as bytes and decode all bytestrings to string.
+                    logger.debug('UnicodeDecodeError on python 2 pickle. Decoding bytestrings.')
+                    event_list = decode_object(pickle.loads(data.encode('iso-8859-1'), encoding='bytes'))
+            except (pickle.UnpicklingError, AttributeError):
+                logger.debug(f'Station {station_id}: pickling error')
+                return [rcodes.RC_PE_PICKLING_ERROR]
+
+            store_data(station_id, cluster, event_list)
+            logger.debug(f'Station {station_id}: succesfully completed')
+            return [rcodes.RC_OK]

+
+
+
+

+[docs]
+def do_init(configfile):
+    """Load configuration and passwords and set up a logger handler
+
+    This function will do one-time initialization.  By using global
+    variables, we eliminate the need to reread configuration and passwords
+    on every request.
+
+    Configuration is read from the datastore configuation file (usually
+    `config.ini`):
+
+    .. include:: ../examples/config.ini
+       :literal:
+
+    Station information is read from the `station_list` config variable.
+    (`station_list.csv` on frome)
+
+    """
+    # set up config
+    global config
+    try:
+        config
+    except NameError:
+        config = configparser.ConfigParser()
+        config.read(configfile)
+
+    # set up logger
+    if not logger.handlers:
+        file = config.get('General', 'log') + f'-wsgi.{os.getpid()}'
+        handler = logging.handlers.TimedRotatingFileHandler(file, when='midnight', backupCount=14)
+        handler.setFormatter(formatter)
+        logger.addHandler(handler)
+        level = LEVELS.get(config.get('General', 'loglevel'), logging.NOTSET)
+        logger.setLevel(level=level)
+
+    # read station list
+    global station_list
+    try:
+        station_list
+    except NameError:
+        station_list = {}
+        station_list_path = Path(config.get('General', 'station_list'))
+        with station_list_path.open() as file:
+            reader = csv.reader(file)
+            for station in reader:
+                if station:
+                    num, cluster, password = station
+                    num = int(num)
+                    station_list[num] = (cluster, password)

+
+
+
+

+[docs]
+def store_data(station_id, cluster, event_list):
+    """Store verified event data to temporary storage"""
+
+    logger.debug(f'Storing data for station {station_id}')
+
+    data_dir = Path(config.get('General', 'data_dir'))
+
+    directory = data_dir / 'incoming'
+    tmp_dir = data_dir / 'tmp'
+
+    if is_data_suspicious(event_list):
+        logger.debug('Event list marked as suspicious.')
+        directory = data_dir / 'suspicious'
+
+    file = tempfile.NamedTemporaryFile(dir=tmp_dir, delete=False)
+    logger.debug(f'Filename: {file.name}')
+    data = {'station_id': station_id, 'cluster': cluster, 'event_list': event_list}
+    pickle.dump(data, file)
+    file.close()
+
+    shutil.move(file.name, directory)

+
+
+
+

+[docs]
+def is_data_suspicious(event_list):
+    """Check data for suspiciousness
+
+    Suspiciousness, a previously unknown quantum number that may signify
+    the actual birth of the universe and the reweaving of past fates into
+    current events has come to hunt us and our beloved data.
+
+    Note: Apr 7, 2019 0:00 is the default time after a cold start without GPS signal.
+    The DAQ will happily send events even when no GPS signal has been
+    acquired (yet). Events with timestamp Apr 7, 2019 are most probably caused
+    by no or bad GPS signal. Such events must be eigenstates of suspiciousness.
+
+    """
+    for event in event_list:
+        if event['header']['datetime'].year < MINIMUM_YEAR:
+            logger.debug('Date < 2020: Timestamp has high suspiciousness.')
+            return True
+    return False

+
+
+
+

+[docs]
+def decode_object(o):
+    """recursively decode all bytestrings in object"""
+
+    if isinstance(o, bytes):
+        return o.decode()
+    elif isinstance(o, dict):
+        return {decode_object(key): decode_object(value) for key, value in o.items()}
+    elif isinstance(o, list):
+        return [decode_object(obj) for obj in o]
+    else:
+        return o

+
+
+ +
+
+
+
+ +
+
+ + + + \ No newline at end of file diff --git a/_sources/index.rst.txt b/_sources/index.rst.txt new file mode 100644 index 0000000..29a4fa7 --- /dev/null +++ b/_sources/index.rst.txt @@ -0,0 +1,25 @@ +.. HiSPARC-datastore documentation master file, created by + sphinx-quickstart on Wed May 10 10:11:52 2017. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to HiSPARC-datastore's documentation! +============================================= + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + introduction + wsgi + writer + xml-rpc-server + + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/_sources/introduction.rst.txt b/_sources/introduction.rst.txt new file mode 100644 index 0000000..6b441ca --- /dev/null +++ b/_sources/introduction.rst.txt @@ -0,0 +1,11 @@ +Introduction +============ + +Todo: + + - document datastore setup on frome + - document `station-list.csv` <-- pique + - document raw datastore tables + - document upload codes?? + - document logfiles + diff --git a/_sources/writer.rst.txt b/_sources/writer.rst.txt new file mode 100644 index 0000000..772ae2c --- /dev/null +++ b/_sources/writer.rst.txt @@ -0,0 +1,37 @@ +datastore writer package +======================== + +.. automodule:: writer + :members: + :undoc-members: + + +Writer application +------------------ + +.. automodule:: writer.writer_app + :members: + :undoc-members: + +Storage classes - raw datastore tables +-------------------------------------- + +.. automodule:: writer.storage + :members: + :undoc-members: + :imported-members: + +Store events module +------------------- + +.. automodule:: writer.store_events + :members: + :undoc-members: + +Upload codes +------------ + + +.. automodule:: writer.upload_codes + :members: + :undoc-members: diff --git a/_sources/wsgi.rst.txt b/_sources/wsgi.rst.txt new file mode 100644 index 0000000..b59801a --- /dev/null +++ b/_sources/wsgi.rst.txt @@ -0,0 +1,22 @@ +datastore WSGI app package +========================== + +.. automodule:: wsgi + :members: + :undoc-members: + + +wsgi\.rcodes module +------------------- + +.. automodule:: wsgi.rcodes + :members: + :undoc-members: + + +wsgi\.wsgi\_app module +---------------------- + +.. automodule:: wsgi.wsgi_app + :members: + :undoc-members: diff --git a/_sources/xml-rpc-server.rst.txt b/_sources/xml-rpc-server.rst.txt new file mode 100644 index 0000000..1373ca7 --- /dev/null +++ b/_sources/xml-rpc-server.rst.txt @@ -0,0 +1,6 @@ +XML-RPC-Server +============== + +.. automodule:: examples.datastore_config_server + :members: + :undoc-members: diff --git a/_static/basic.css b/_static/basic.css new file mode 100644 index 0000000..f316efc --- /dev/null +++ b/_static/basic.css @@ -0,0 +1,925 @@ +/* + * basic.css + * ~~~~~~~~~ + * + * Sphinx stylesheet -- basic theme. + * + * :copyright: Copyright 2007-2024 by the Sphinx team, see AUTHORS. + * :license: BSD, see LICENSE for details. + * + */ + +/* -- main layout ----------------------------------------------------------- */ + +div.clearer { + clear: both; +} + +div.section::after { + display: block; + content: ''; + clear: left; +} + +/* -- relbar ---------------------------------------------------------------- */ + +div.related { + width: 100%; + font-size: 90%; +} + +div.related h3 { + display: none; +} + +div.related ul { + margin: 0; + padding: 0 0 0 10px; + list-style: none; +} + +div.related li { + display: inline; +} + +div.related li.right { + float: right; + margin-right: 5px; +} + +/* -- sidebar --------------------------------------------------------------- */ + +div.sphinxsidebarwrapper { + padding: 10px 5px 0 10px; +} + +div.sphinxsidebar { + float: left; + width: 230px; + margin-left: -100%; + font-size: 90%; + word-wrap: break-word; + overflow-wrap : break-word; +} + +div.sphinxsidebar ul { + list-style: none; +} + +div.sphinxsidebar ul ul, +div.sphinxsidebar ul.want-points { + margin-left: 20px; + list-style: square; +} + +div.sphinxsidebar ul ul { + margin-top: 0; + margin-bottom: 0; +} + +div.sphinxsidebar form { + margin-top: 10px; +} + +div.sphinxsidebar input { + border: 1px solid #98dbcc; + font-family: sans-serif; + font-size: 1em; +} + +div.sphinxsidebar #searchbox form.search { + overflow: hidden; +} + +div.sphinxsidebar #searchbox input[type="text"] { + float: left; + width: 80%; + padding: 0.25em; + box-sizing: border-box; +} + +div.sphinxsidebar #searchbox input[type="submit"] { + float: left; + width: 20%; + border-left: none; + padding: 0.25em; + box-sizing: border-box; +} + + +img { + border: 0; + max-width: 100%; +} + +/* -- search page ----------------------------------------------------------- */ + +ul.search { + margin: 10px 0 0 20px; + padding: 0; +} + +ul.search li { + padding: 5px 0 5px 20px; + background-image: url(file.png); + background-repeat: no-repeat; + background-position: 0 7px; +} + +ul.search li a { + font-weight: bold; +} + +ul.search li p.context { + color: #888; + margin: 2px 0 0 30px; + text-align: left; +} + +ul.keywordmatches li.goodmatch a { + font-weight: bold; +} + +/* -- index page ------------------------------------------------------------ */ + +table.contentstable { + width: 90%; + margin-left: auto; + margin-right: auto; +} + +table.contentstable p.biglink { + line-height: 150%; +} + +a.biglink { + font-size: 1.3em; +} + +span.linkdescr { + font-style: italic; + padding-top: 5px; + font-size: 90%; +} + +/* -- general index --------------------------------------------------------- */ + +table.indextable { + width: 100%; +} + +table.indextable td { + text-align: left; + vertical-align: top; +} + +table.indextable ul { + margin-top: 0; + margin-bottom: 0; + list-style-type: none; +} + +table.indextable > tbody > tr > td > ul { + padding-left: 0em; +} + +table.indextable tr.pcap { + height: 10px; +} + +table.indextable tr.cap { + margin-top: 10px; + background-color: #f2f2f2; +} + +img.toggler { + margin-right: 3px; + margin-top: 3px; + cursor: pointer; +} + +div.modindex-jumpbox { + border-top: 1px solid #ddd; + border-bottom: 1px solid #ddd; + margin: 1em 0 1em 0; + padding: 0.4em; +} + +div.genindex-jumpbox { + border-top: 1px solid #ddd; + border-bottom: 1px solid #ddd; + margin: 1em 0 1em 0; + padding: 0.4em; +} + +/* -- domain module index --------------------------------------------------- */ + +table.modindextable td { + padding: 2px; + border-collapse: collapse; +} + +/* -- general body styles --------------------------------------------------- */ + +div.body { + min-width: 360px; + max-width: 800px; +} + +div.body p, div.body dd, div.body li, div.body blockquote { + -moz-hyphens: auto; + -ms-hyphens: auto; + -webkit-hyphens: auto; + hyphens: auto; +} + +a.headerlink { + visibility: hidden; +} + +a:visited { + color: #551A8B; +} + +h1:hover > a.headerlink, +h2:hover > a.headerlink, +h3:hover > a.headerlink, +h4:hover > a.headerlink, +h5:hover > a.headerlink, +h6:hover > a.headerlink, +dt:hover > a.headerlink, +caption:hover > a.headerlink, +p.caption:hover > a.headerlink, +div.code-block-caption:hover > a.headerlink { + visibility: visible; +} + +div.body p.caption { + text-align: inherit; +} + +div.body td { + text-align: left; +} + +.first { + margin-top: 0 !important; +} + +p.rubric { + margin-top: 30px; + font-weight: bold; +} + +img.align-left, figure.align-left, .figure.align-left, object.align-left { + clear: left; + float: left; + margin-right: 1em; +} + +img.align-right, figure.align-right, .figure.align-right, object.align-right { + clear: right; + float: right; + margin-left: 1em; +} + +img.align-center, figure.align-center, .figure.align-center, object.align-center { + display: block; + margin-left: auto; + margin-right: auto; +} + +img.align-default, figure.align-default, .figure.align-default { + display: block; + margin-left: auto; + margin-right: auto; +} + +.align-left { + text-align: left; +} + +.align-center { + text-align: center; +} + +.align-default { + text-align: center; +} + +.align-right { + text-align: right; +} + +/* -- sidebars -------------------------------------------------------------- */ + +div.sidebar, +aside.sidebar { + margin: 0 0 0.5em 1em; + border: 1px solid #ddb; + padding: 7px; + background-color: #ffe; + width: 40%; + float: right; + clear: right; + overflow-x: auto; +} + +p.sidebar-title { + font-weight: bold; +} + +nav.contents, +aside.topic, +div.admonition, div.topic, blockquote { + clear: left; +} + +/* -- topics ---------------------------------------------------------------- */ + +nav.contents, +aside.topic, +div.topic { + border: 1px solid #ccc; + padding: 7px; + margin: 10px 0 10px 0; +} + +p.topic-title { + font-size: 1.1em; + font-weight: bold; + margin-top: 10px; +} + +/* -- admonitions ----------------------------------------------------------- */ + +div.admonition { + margin-top: 10px; + margin-bottom: 10px; + padding: 7px; +} + +div.admonition dt { + font-weight: bold; +} + +p.admonition-title { + margin: 0px 10px 5px 0px; + font-weight: bold; +} + +div.body p.centered { + text-align: center; + margin-top: 25px; +} + +/* -- content of sidebars/topics/admonitions -------------------------------- */ + +div.sidebar > :last-child, +aside.sidebar > :last-child, +nav.contents > :last-child, +aside.topic > :last-child, +div.topic > :last-child, +div.admonition > :last-child { + margin-bottom: 0; +} + +div.sidebar::after, +aside.sidebar::after, +nav.contents::after, +aside.topic::after, +div.topic::after, +div.admonition::after, +blockquote::after { + display: block; + content: ''; + clear: both; +} + +/* -- tables ---------------------------------------------------------------- */ + +table.docutils { + margin-top: 10px; + margin-bottom: 10px; + border: 0; + border-collapse: collapse; +} + +table.align-center { + margin-left: auto; + margin-right: auto; +} + +table.align-default { + margin-left: auto; + margin-right: auto; +} + +table caption span.caption-number { + font-style: italic; +} + +table caption span.caption-text { +} + +table.docutils td, table.docutils th { + padding: 1px 8px 1px 5px; + border-top: 0; + border-left: 0; + border-right: 0; + border-bottom: 1px solid #aaa; +} + +th { + text-align: left; + padding-right: 5px; +} + +table.citation { + border-left: solid 1px gray; + margin-left: 1px; +} + +table.citation td { + border-bottom: none; +} + +th > :first-child, +td > :first-child { + margin-top: 0px; +} + +th > :last-child, +td > :last-child { + margin-bottom: 0px; +} + +/* -- figures --------------------------------------------------------------- */ + +div.figure, figure { + margin: 0.5em; + padding: 0.5em; +} + +div.figure p.caption, figcaption { + padding: 0.3em; +} + +div.figure p.caption span.caption-number, +figcaption span.caption-number { + font-style: italic; +} + +div.figure p.caption span.caption-text, +figcaption span.caption-text { +} + +/* -- field list styles ----------------------------------------------------- */ + +table.field-list td, table.field-list th { + border: 0 !important; +} + +.field-list ul { + margin: 0; + padding-left: 1em; +} + +.field-list p { + margin: 0; +} + +.field-name { + -moz-hyphens: manual; + -ms-hyphens: manual; + -webkit-hyphens: manual; + hyphens: manual; +} + +/* -- hlist styles ---------------------------------------------------------- */ + +table.hlist { + margin: 1em 0; +} + +table.hlist td { + vertical-align: top; +} + +/* -- object description styles --------------------------------------------- */ + +.sig { + font-family: 'Consolas', 'Menlo', 'DejaVu Sans Mono', 'Bitstream Vera Sans Mono', monospace; +} + +.sig-name, code.descname { + background-color: transparent; + font-weight: bold; +} + +.sig-name { + font-size: 1.1em; +} + +code.descname { + font-size: 1.2em; +} + +.sig-prename, code.descclassname { + background-color: transparent; +} + +.optional { + font-size: 1.3em; +} + +.sig-paren { + font-size: larger; +} + +.sig-param.n { + font-style: italic; +} + +/* C++ specific styling */ + +.sig-inline.c-texpr, +.sig-inline.cpp-texpr { + font-family: unset; +} + +.sig.c .k, .sig.c .kt, +.sig.cpp .k, .sig.cpp .kt { + color: #0033B3; +} + +.sig.c .m, +.sig.cpp .m { + color: #1750EB; +} + +.sig.c .s, .sig.c .sc, +.sig.cpp .s, .sig.cpp .sc { + color: #067D17; +} + + +/* -- other body styles ----------------------------------------------------- */ + +ol.arabic { + list-style: decimal; +} + +ol.loweralpha { + list-style: lower-alpha; +} + +ol.upperalpha { + list-style: upper-alpha; +} + +ol.lowerroman { + list-style: lower-roman; +} + +ol.upperroman { + list-style: upper-roman; +} + +:not(li) > ol > li:first-child > :first-child, +:not(li) > ul > li:first-child > :first-child { + margin-top: 0px; +} + +:not(li) > ol > li:last-child > :last-child, +:not(li) > ul > li:last-child > :last-child { + margin-bottom: 0px; +} + +ol.simple ol p, +ol.simple ul p, +ul.simple ol p, +ul.simple ul p { + margin-top: 0; +} + +ol.simple > li:not(:first-child) > p, +ul.simple > li:not(:first-child) > p { + margin-top: 0; +} + +ol.simple p, +ul.simple p { + margin-bottom: 0; +} + +aside.footnote > span, +div.citation > span { + float: left; +} +aside.footnote > span:last-of-type, +div.citation > span:last-of-type { + padding-right: 0.5em; +} +aside.footnote > p { + margin-left: 2em; +} +div.citation > p { + margin-left: 4em; +} +aside.footnote > p:last-of-type, +div.citation > p:last-of-type { + margin-bottom: 0em; +} +aside.footnote > p:last-of-type:after, +div.citation > p:last-of-type:after { + content: ""; + clear: both; +} + +dl.field-list { + display: grid; + grid-template-columns: fit-content(30%) auto; +} + +dl.field-list > dt { + font-weight: bold; + word-break: break-word; + padding-left: 0.5em; + padding-right: 5px; +} + +dl.field-list > dd { + padding-left: 0.5em; + margin-top: 0em; + margin-left: 0em; + margin-bottom: 0em; +} + +dl { + margin-bottom: 15px; +} + +dd > :first-child { + margin-top: 0px; +} + +dd ul, dd table { + margin-bottom: 10px; +} + +dd { + margin-top: 3px; + margin-bottom: 10px; + margin-left: 30px; +} + +.sig dd { + margin-top: 0px; + margin-bottom: 0px; +} + +.sig dl { + margin-top: 0px; + margin-bottom: 0px; +} + +dl > dd:last-child, +dl > dd:last-child > :last-child { + margin-bottom: 0; +} + +dt:target, span.highlighted { + background-color: #fbe54e; +} + +rect.highlighted { + fill: #fbe54e; +} + +dl.glossary dt { + font-weight: bold; + font-size: 1.1em; +} + +.versionmodified { + font-style: italic; +} + +.system-message { + background-color: #fda; + padding: 5px; + border: 3px solid red; +} + +.footnote:target { + background-color: #ffa; +} + +.line-block { + display: block; + margin-top: 1em; + margin-bottom: 1em; +} + +.line-block .line-block { + margin-top: 0; + margin-bottom: 0; + margin-left: 1.5em; +} + +.guilabel, .menuselection { + font-family: sans-serif; +} + +.accelerator { + text-decoration: underline; +} + +.classifier { + font-style: oblique; +} + +.classifier:before { + font-style: normal; + margin: 0 0.5em; + content: ":"; + display: inline-block; +} + +abbr, acronym { + border-bottom: dotted 1px; + cursor: help; +} + +.translated { + background-color: rgba(207, 255, 207, 0.2) +} + +.untranslated { + background-color: rgba(255, 207, 207, 0.2) +} + +/* -- code displays --------------------------------------------------------- */ + +pre { + overflow: auto; + overflow-y: hidden; /* fixes display issues on Chrome browsers */ +} + +pre, div[class*="highlight-"] { + clear: both; +} + +span.pre { + -moz-hyphens: none; + -ms-hyphens: none; + -webkit-hyphens: none; + hyphens: none; + white-space: nowrap; +} + +div[class*="highlight-"] { + margin: 1em 0; +} + +td.linenos pre { + border: 0; + background-color: transparent; + color: #aaa; +} + +table.highlighttable { + display: block; +} + +table.highlighttable tbody { + display: block; +} + +table.highlighttable tr { + display: flex; +} + +table.highlighttable td { + margin: 0; + padding: 0; +} + +table.highlighttable td.linenos { + padding-right: 0.5em; +} + +table.highlighttable td.code { + flex: 1; + overflow: hidden; +} + +.highlight .hll { + display: block; +} + +div.highlight pre, +table.highlighttable pre { + margin: 0; +} + +div.code-block-caption + div { + margin-top: 0; +} + +div.code-block-caption { + margin-top: 1em; + padding: 2px 5px; + font-size: small; +} + +div.code-block-caption code { + background-color: transparent; +} + +table.highlighttable td.linenos, +span.linenos, +div.highlight span.gp { /* gp: Generic.Prompt */ + user-select: none; + -webkit-user-select: text; /* Safari fallback only */ + -webkit-user-select: none; /* Chrome/Safari */ + -moz-user-select: none; /* Firefox */ + -ms-user-select: none; /* IE10+ */ +} + +div.code-block-caption span.caption-number { + padding: 0.1em 0.3em; + font-style: italic; +} + +div.code-block-caption span.caption-text { +} + +div.literal-block-wrapper { + margin: 1em 0; +} + +code.xref, a code { + background-color: transparent; + font-weight: bold; +} + +h1 code, h2 code, h3 code, h4 code, h5 code, h6 code { + background-color: transparent; +} + +.viewcode-link { + float: right; +} + +.viewcode-back { + float: right; + font-family: sans-serif; +} + +div.viewcode-block:target { + margin: -1px -10px; + padding: 0 10px; +} + +/* -- math display ---------------------------------------------------------- */ + +img.math { + vertical-align: middle; +} + +div.body div.math p { + text-align: center; +} + +span.eqno { + float: right; +} + +span.eqno a.headerlink { + position: absolute; + z-index: 1; +} + +div.math:hover a.headerlink { + visibility: visible; +} + +/* -- printout stylesheet --------------------------------------------------- */ + +@media print { + div.document, + div.documentwrapper, + div.bodywrapper { + margin: 0 !important; + width: 100%; + } + + div.sphinxsidebar, + div.related, + div.footer, + #top-link { + display: none; + } +} \ No newline at end of file diff --git a/_static/doctools.js b/_static/doctools.js new file mode 100644 index 0000000..4d67807 --- /dev/null +++ b/_static/doctools.js @@ -0,0 +1,156 @@ +/* + * doctools.js + * ~~~~~~~~~~~ + * + * Base JavaScript utilities for all Sphinx HTML documentation. + * + * :copyright: Copyright 2007-2024 by the Sphinx team, see AUTHORS. + * :license: BSD, see LICENSE for details. + * + */ +"use strict"; + +const BLACKLISTED_KEY_CONTROL_ELEMENTS = new Set([ + "TEXTAREA", + "INPUT", + "SELECT", + "BUTTON", +]); + +const _ready = (callback) => { + if (document.readyState !== "loading") { + callback(); + } else { + document.addEventListener("DOMContentLoaded", callback); + } +}; + +/** + * Small JavaScript module for the documentation. + */ +const Documentation = { + init: () => { + Documentation.initDomainIndexTable(); + Documentation.initOnKeyListeners(); + }, + + /** + * i18n support + */ + TRANSLATIONS: {}, + PLURAL_EXPR: (n) => (n === 1 ? 0 : 1), + LOCALE: "unknown", + + // gettext and ngettext don't access this so that the functions + // can safely bound to a different name (_ = Documentation.gettext) + gettext: (string) => { + const translated = Documentation.TRANSLATIONS[string]; + switch (typeof translated) { + case "undefined": + return string; // no translation + case "string": + return translated; // translation exists + default: + return translated[0]; // (singular, plural) translation tuple exists + } + }, + + ngettext: (singular, plural, n) => { + const translated = Documentation.TRANSLATIONS[singular]; + if (typeof translated !== "undefined") + return translated[Documentation.PLURAL_EXPR(n)]; + return n === 1 ? singular : plural; + }, + + addTranslations: (catalog) => { + Object.assign(Documentation.TRANSLATIONS, catalog.messages); + Documentation.PLURAL_EXPR = new Function( + "n", + `return (${catalog.plural_expr})` + ); + Documentation.LOCALE = catalog.locale; + }, + + /** + * helper function to focus on search bar + */ + focusSearchBar: () => { + document.querySelectorAll("input[name=q]")[0]?.focus(); + }, + + /** + * Initialise the domain index toggle buttons + */ + initDomainIndexTable: () => { + const toggler = (el) => { + const idNumber = el.id.substr(7); + const toggledRows = document.querySelectorAll(`tr.cg-${idNumber}`); + if (el.src.substr(-9) === "minus.png") { + el.src = `${el.src.substr(0, el.src.length - 9)}plus.png`; + toggledRows.forEach((el) => (el.style.display = "none")); + } else { + el.src = `${el.src.substr(0, el.src.length - 8)}minus.png`; + toggledRows.forEach((el) => (el.style.display = "")); + } + }; + + const togglerElements = document.querySelectorAll("img.toggler"); + togglerElements.forEach((el) => + el.addEventListener("click", (event) => toggler(event.currentTarget)) + ); + togglerElements.forEach((el) => (el.style.display = "")); + if (DOCUMENTATION_OPTIONS.COLLAPSE_INDEX) togglerElements.forEach(toggler); + }, + + initOnKeyListeners: () => { + // only install a listener if it is really needed + if ( + !DOCUMENTATION_OPTIONS.NAVIGATION_WITH_KEYS && + !DOCUMENTATION_OPTIONS.ENABLE_SEARCH_SHORTCUTS + ) + return; + + document.addEventListener("keydown", (event) => { + // bail for input elements + if (BLACKLISTED_KEY_CONTROL_ELEMENTS.has(document.activeElement.tagName)) return; + // bail with special keys + if (event.altKey || event.ctrlKey || event.metaKey) return; + + if (!event.shiftKey) { + switch (event.key) { + case "ArrowLeft": + if (!DOCUMENTATION_OPTIONS.NAVIGATION_WITH_KEYS) break; + + const prevLink = document.querySelector('link[rel="prev"]'); + if (prevLink && prevLink.href) { + window.location.href = prevLink.href; + event.preventDefault(); + } + break; + case "ArrowRight": + if (!DOCUMENTATION_OPTIONS.NAVIGATION_WITH_KEYS) break; + + const nextLink = document.querySelector('link[rel="next"]'); + if (nextLink && nextLink.href) { + window.location.href = nextLink.href; + event.preventDefault(); + } + break; + } + } + + // some keyboard layouts may need Shift to get / + switch (event.key) { + case "/": + if (!DOCUMENTATION_OPTIONS.ENABLE_SEARCH_SHORTCUTS) break; + Documentation.focusSearchBar(); + event.preventDefault(); + } + }); + }, +}; + +// quick alias for translations +const _ = Documentation.gettext; + +_ready(Documentation.init); diff --git a/_static/documentation_options.js b/_static/documentation_options.js new file mode 100644 index 0000000..89435bb --- /dev/null +++ b/_static/documentation_options.js @@ -0,0 +1,13 @@ +const DOCUMENTATION_OPTIONS = { + VERSION: '1.0.0', + LANGUAGE: 'en', + COLLAPSE_INDEX: false, + BUILDER: 'html', + FILE_SUFFIX: '.html', + LINK_SUFFIX: '.html', + HAS_SOURCE: true, + SOURCELINK_SUFFIX: '.txt', + NAVIGATION_WITH_KEYS: false, + SHOW_SEARCH_SUMMARY: true, + ENABLE_SEARCH_SHORTCUTS: true, +}; \ No newline at end of file diff --git a/_static/favicon.ico b/_static/favicon.ico new file mode 100644 index 0000000..2632b90 Binary files /dev/null and b/_static/favicon.ico differ diff --git a/_static/file.png b/_static/file.png new file mode 100644 index 0000000..a858a41 Binary files /dev/null and b/_static/file.png differ diff --git a/_static/header.png b/_static/header.png new file mode 100644 index 0000000..383625f Binary files /dev/null and b/_static/header.png differ diff --git a/_static/hisparc_style.css b/_static/hisparc_style.css new file mode 100644 index 0000000..21b105f --- /dev/null +++ b/_static/hisparc_style.css @@ -0,0 +1,70 @@ +body { + background: #fff; + max-width: 900px; + margin: 0 auto;} + +div.document { + background-color: #fafafa;} + +div.sphinxsidebar ul { + margin: 10px 10px;} + +div.sphinxsidebarwrapper { + padding: 0px 0px 10px 0px;} + +div.sphinxsidebar h3, +div.sphinxsidebar h4 { + color: #111; + background: none; + padding: 8px 10px 5px 10px; + border-bottom: 1px solid #ddd;} + +div.sphinxsidebar h3 a { + color: #111;} + +div.sphinxsidebar a { + color: #333;} + +div.sphinxsidebarwrapper ul { + margin-right: 5px;} + +p.searchtip { + display: none;} + +div.body { + padding: 0 25px 25px 30px; + color: #333;} + +div.body h1, +div.body h2, +div.body h3, +div.body h4, +div.body h5, +div.body h6 { + background: none; + padding-top: 0; + padding-left: 0;} + +div.body h1 { + border-top: none; + padding-top: 20px; + font-size: 260%;} + +div.body h2 { + font-size: 210%; + padding-top: 15px;} + +div.body h3 { + font-size: 170%; + border-bottom: 1px solid #BBB;} + +div.body h4 { + font-size: 130%; + margin-top: 20px; + padding-left: 3px;} + +div.related { + background-color: #0071E2;} + +div.related a { + color: #fff;} diff --git a/_static/language_data.js b/_static/language_data.js new file mode 100644 index 0000000..367b8ed --- /dev/null +++ b/_static/language_data.js @@ -0,0 +1,199 @@ +/* + * language_data.js + * ~~~~~~~~~~~~~~~~ + * + * This script contains the language-specific data used by searchtools.js, + * namely the list of stopwords, stemmer, scorer and splitter. + * + * :copyright: Copyright 2007-2024 by the Sphinx team, see AUTHORS. + * :license: BSD, see LICENSE for details. + * + */ + +var stopwords = ["a", "and", "are", "as", "at", "be", "but", "by", "for", "if", "in", "into", "is", "it", "near", "no", "not", "of", "on", "or", "such", "that", "the", "their", "then", "there", "these", "they", "this", "to", "was", "will", "with"]; + + +/* Non-minified version is copied as a separate JS file, if available */ + +/** + * Porter Stemmer + */ +var Stemmer = function() { + + var step2list = { + ational: 'ate', + tional: 'tion', + enci: 'ence', + anci: 'ance', + izer: 'ize', + bli: 'ble', + alli: 'al', + entli: 'ent', + eli: 'e', + ousli: 'ous', + ization: 'ize', + ation: 'ate', + ator: 'ate', + alism: 'al', + iveness: 'ive', + fulness: 'ful', + ousness: 'ous', + aliti: 'al', + iviti: 'ive', + biliti: 'ble', + logi: 'log' + }; + + var step3list = { + icate: 'ic', + ative: '', + alize: 'al', + iciti: 'ic', + ical: 'ic', + ful: '', + ness: '' + }; + + var c = "[^aeiou]"; // consonant + var v = "[aeiouy]"; // vowel + var C = c + "[^aeiouy]*"; // consonant sequence + var V = v + "[aeiou]*"; // vowel sequence + + var mgr0 = "^(" + C + ")?" + V + C; // [C]VC... is m>0 + var meq1 = "^(" + C + ")?" + V + C + "(" + V + ")?$"; // [C]VC[V] is m=1 + var mgr1 = "^(" + C + ")?" + V + C + V + C; // [C]VCVC... is m>1 + var s_v = "^(" + C + ")?" + v; // vowel in stem + + this.stemWord = function (w) { + var stem; + var suffix; + var firstch; + var origword = w; + + if (w.length < 3) + return w; + + var re; + var re2; + var re3; + var re4; + + firstch = w.substr(0,1); + if (firstch == "y") + w = firstch.toUpperCase() + w.substr(1); + + // Step 1a + re = /^(.+?)(ss|i)es$/; + re2 = /^(.+?)([^s])s$/; + + if (re.test(w)) + w = w.replace(re,"$1$2"); + else if (re2.test(w)) + w = w.replace(re2,"$1$2"); + + // Step 1b + re = /^(.+?)eed$/; + re2 = /^(.+?)(ed|ing)$/; + if (re.test(w)) { + var fp = re.exec(w); + re = new RegExp(mgr0); + if (re.test(fp[1])) { + re = /.$/; + w = w.replace(re,""); + } + } + else if (re2.test(w)) { + var fp = re2.exec(w); + stem = fp[1]; + re2 = new RegExp(s_v); + if (re2.test(stem)) { + w = stem; + re2 = /(at|bl|iz)$/; + re3 = new RegExp("([^aeiouylsz])\\1$"); + re4 = new RegExp("^" + C + v + "[^aeiouwxy]$"); + if (re2.test(w)) + w = w + "e"; + else if (re3.test(w)) { + re = /.$/; + w = w.replace(re,""); + } + else if (re4.test(w)) + w = w + "e"; + } + } + + // Step 1c + re = /^(.+?)y$/; + if (re.test(w)) { + var fp = re.exec(w); + stem = fp[1]; + re = new RegExp(s_v); + if (re.test(stem)) + w = stem + "i"; + } + + // Step 2 + re = /^(.+?)(ational|tional|enci|anci|izer|bli|alli|entli|eli|ousli|ization|ation|ator|alism|iveness|fulness|ousness|aliti|iviti|biliti|logi)$/; + if (re.test(w)) { + var fp = re.exec(w); + stem = fp[1]; + suffix = fp[2]; + re = new RegExp(mgr0); + if (re.test(stem)) + w = stem + step2list[suffix]; + } + + // Step 3 + re = /^(.+?)(icate|ative|alize|iciti|ical|ful|ness)$/; + if (re.test(w)) { + var fp = re.exec(w); + stem = fp[1]; + suffix = fp[2]; + re = new RegExp(mgr0); + if (re.test(stem)) + w = stem + step3list[suffix]; + } + + // Step 4 + re = /^(.+?)(al|ance|ence|er|ic|able|ible|ant|ement|ment|ent|ou|ism|ate|iti|ous|ive|ize)$/; + re2 = /^(.+?)(s|t)(ion)$/; + if (re.test(w)) { + var fp = re.exec(w); + stem = fp[1]; + re = new RegExp(mgr1); + if (re.test(stem)) + w = stem; + } + else if (re2.test(w)) { + var fp = re2.exec(w); + stem = fp[1] + fp[2]; + re2 = new RegExp(mgr1); + if (re2.test(stem)) + w = stem; + } + + // Step 5 + re = /^(.+?)e$/; + if (re.test(w)) { + var fp = re.exec(w); + stem = fp[1]; + re = new RegExp(mgr1); + re2 = new RegExp(meq1); + re3 = new RegExp("^" + C + v + "[^aeiouwxy]$"); + if (re.test(stem) || (re2.test(stem) && !(re3.test(stem)))) + w = stem; + } + re = /ll$/; + re2 = new RegExp(mgr1); + if (re.test(w) && re2.test(w)) { + re = /.$/; + w = w.replace(re,""); + } + + // and turn initial Y back to y + if (firstch == "y") + w = firstch.toLowerCase() + w.substr(1); + return w; + } +} + diff --git a/_static/minus.png b/_static/minus.png new file mode 100644 index 0000000..d96755f Binary files /dev/null and b/_static/minus.png differ diff --git a/_static/nature.css b/_static/nature.css new file mode 100644 index 0000000..ba033b0 --- /dev/null +++ b/_static/nature.css @@ -0,0 +1,252 @@ +/* + * nature.css_t + * ~~~~~~~~~~~~ + * + * Sphinx stylesheet -- nature theme. + * + * :copyright: Copyright 2007-2024 by the Sphinx team, see AUTHORS. + * :license: BSD, see LICENSE for details. + * + */ + +@import url("basic.css"); + +/* -- page layout ----------------------------------------------------------- */ + +body { + font-family: Arial, sans-serif; + font-size: 100%; + background-color: #fff; + color: #555; + margin: 0; + padding: 0; +} + +div.documentwrapper { + float: left; + width: 100%; +} + +div.bodywrapper { + margin: 0 0 0 230px; +} + +hr { + border: 1px solid #B1B4B6; +} + +div.document { + background-color: #eee; +} + +div.body { + background-color: #ffffff; + color: #3E4349; + padding: 0 30px 30px 30px; + font-size: 0.9em; +} + +div.footer { + color: #555; + width: 100%; + padding: 13px 0; + text-align: center; + font-size: 75%; +} + +div.footer a { + color: #444; + text-decoration: underline; +} + +div.related { + background-color: #6BA81E; + line-height: 32px; + color: #fff; + text-shadow: 0px 1px 0 #444; + font-size: 0.9em; +} + +div.related a { + color: #E2F3CC; +} + +div.sphinxsidebar { + font-size: 0.75em; + line-height: 1.5em; +} + +div.sphinxsidebarwrapper{ + padding: 20px 0; +} + +div.sphinxsidebar h3, +div.sphinxsidebar h4 { + font-family: Arial, sans-serif; + color: #222; + font-size: 1.2em; + font-weight: normal; + margin: 0; + padding: 5px 10px; + background-color: #ddd; + text-shadow: 1px 1px 0 white +} + +div.sphinxsidebar h4{ + font-size: 1.1em; +} + +div.sphinxsidebar h3 a { + color: #444; +} + + +div.sphinxsidebar p { + color: #888; + padding: 5px 20px; +} + +div.sphinxsidebar p.topless { +} + +div.sphinxsidebar ul { + margin: 10px 20px; + padding: 0; + color: #000; +} + +div.sphinxsidebar a { + color: #444; +} + +div.sphinxsidebar input { + border: 1px solid #ccc; + font-family: sans-serif; + font-size: 1em; +} + +div.sphinxsidebar .searchformwrapper { + margin-left: 20px; + margin-right: 20px; +} + +/* -- body styles ----------------------------------------------------------- */ + +a { + color: #005B81; + text-decoration: none; +} + +a:hover { + color: #E32E00; + text-decoration: underline; +} + +a:visited { + color: #551A8B; +} + +div.body h1, +div.body h2, +div.body h3, +div.body h4, +div.body h5, +div.body h6 { + font-family: Arial, sans-serif; + background-color: #BED4EB; + font-weight: normal; + color: #212224; + margin: 30px 0px 10px 0px; + padding: 5px 0 5px 10px; + text-shadow: 0px 1px 0 white +} + +div.body h1 { border-top: 20px solid white; margin-top: 0; font-size: 200%; } +div.body h2 { font-size: 150%; background-color: #C8D5E3; } +div.body h3 { font-size: 120%; background-color: #D8DEE3; } +div.body h4 { font-size: 110%; background-color: #D8DEE3; } +div.body h5 { font-size: 100%; background-color: #D8DEE3; } +div.body h6 { font-size: 100%; background-color: #D8DEE3; } + +a.headerlink { + color: #c60f0f; + font-size: 0.8em; + padding: 0 4px 0 4px; + text-decoration: none; +} + +a.headerlink:hover { + background-color: #c60f0f; + color: white; +} + +div.body p, div.body dd, div.body li { + line-height: 1.5em; +} + +div.admonition p.admonition-title + p { + display: inline; +} + +div.note { + background-color: #eee; + border: 1px solid #ccc; +} + +div.seealso { + background-color: #ffc; + border: 1px solid #ff6; +} + +nav.contents, +aside.topic, +div.topic { + background-color: #eee; +} + +div.warning { + background-color: #ffe4e4; + border: 1px solid #f66; +} + +p.admonition-title { + display: inline; +} + +p.admonition-title:after { + content: ":"; +} + +pre { + padding: 10px; + line-height: 1.2em; + border: 1px solid #C6C9CB; + font-size: 1.1em; + margin: 1.5em 0 1.5em 0; + -webkit-box-shadow: 1px 1px 1px #d8d8d8; + -moz-box-shadow: 1px 1px 1px #d8d8d8; +} + +code { + background-color: #ecf0f3; + color: #222; + /* padding: 1px 2px; */ + font-size: 1.1em; + font-family: monospace; +} + +.viewcode-back { + font-family: Arial, sans-serif; +} + +div.viewcode-block:target { + background-color: #f4debf; + border-top: 1px solid #ac9; + border-bottom: 1px solid #ac9; +} + +div.code-block-caption { + background-color: #ddd; + color: #222; + border: 1px solid #C6C9CB; +} \ No newline at end of file diff --git a/_static/plus.png b/_static/plus.png new file mode 100644 index 0000000..7107cec Binary files /dev/null and b/_static/plus.png differ diff --git a/_static/pygments.css b/_static/pygments.css new file mode 100644 index 0000000..0d49244 --- /dev/null +++ b/_static/pygments.css @@ -0,0 +1,75 @@ +pre { line-height: 125%; } +td.linenos .normal { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; } +span.linenos { color: inherit; background-color: transparent; padding-left: 5px; padding-right: 5px; } +td.linenos .special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; } +span.linenos.special { color: #000000; background-color: #ffffc0; padding-left: 5px; padding-right: 5px; } +.highlight .hll { background-color: #ffffcc } +.highlight { background: #eeffcc; } +.highlight .c { color: #408090; font-style: italic } /* Comment */ +.highlight .err { border: 1px solid #FF0000 } /* Error */ +.highlight .k { color: #007020; font-weight: bold } /* Keyword */ +.highlight .o { color: #666666 } /* Operator */ +.highlight .ch { color: #408090; font-style: italic } /* Comment.Hashbang */ +.highlight .cm { color: #408090; font-style: italic } /* Comment.Multiline */ +.highlight .cp { color: #007020 } /* Comment.Preproc */ +.highlight .cpf { color: #408090; font-style: italic } /* Comment.PreprocFile */ +.highlight .c1 { color: #408090; font-style: italic } /* Comment.Single */ +.highlight .cs { color: #408090; background-color: #fff0f0 } /* Comment.Special */ +.highlight .gd { color: #A00000 } /* Generic.Deleted */ +.highlight .ge { font-style: italic } /* Generic.Emph */ +.highlight .ges { font-weight: bold; font-style: italic } /* Generic.EmphStrong */ +.highlight .gr { color: #FF0000 } /* Generic.Error */ +.highlight .gh { color: #000080; font-weight: bold } /* Generic.Heading */ +.highlight .gi { color: #00A000 } /* Generic.Inserted */ +.highlight .go { color: #333333 } /* Generic.Output */ +.highlight .gp { color: #c65d09; font-weight: bold } /* Generic.Prompt */ +.highlight .gs { font-weight: bold } /* Generic.Strong */ +.highlight .gu { color: #800080; font-weight: bold } /* Generic.Subheading */ +.highlight .gt { color: #0044DD } /* Generic.Traceback */ +.highlight .kc { color: #007020; font-weight: bold } /* Keyword.Constant */ +.highlight .kd { color: #007020; font-weight: bold } /* Keyword.Declaration */ +.highlight .kn { color: #007020; font-weight: bold } /* Keyword.Namespace */ +.highlight .kp { color: #007020 } /* Keyword.Pseudo */ +.highlight .kr { color: #007020; font-weight: bold } /* Keyword.Reserved */ +.highlight .kt { color: #902000 } /* Keyword.Type */ +.highlight .m { color: #208050 } /* Literal.Number */ +.highlight .s { color: #4070a0 } /* Literal.String */ +.highlight .na { color: #4070a0 } /* Name.Attribute */ +.highlight .nb { color: #007020 } /* Name.Builtin */ +.highlight .nc { color: #0e84b5; font-weight: bold } /* Name.Class */ +.highlight .no { color: #60add5 } /* Name.Constant */ +.highlight .nd { color: #555555; font-weight: bold } /* Name.Decorator */ +.highlight .ni { color: #d55537; font-weight: bold } /* Name.Entity */ +.highlight .ne { color: #007020 } /* Name.Exception */ +.highlight .nf { color: #06287e } /* Name.Function */ +.highlight .nl { color: #002070; font-weight: bold } /* Name.Label */ +.highlight .nn { color: #0e84b5; font-weight: bold } /* Name.Namespace */ +.highlight .nt { color: #062873; font-weight: bold } /* Name.Tag */ +.highlight .nv { color: #bb60d5 } /* Name.Variable */ +.highlight .ow { color: #007020; font-weight: bold } /* Operator.Word */ +.highlight .w { color: #bbbbbb } /* Text.Whitespace */ +.highlight .mb { color: #208050 } /* Literal.Number.Bin */ +.highlight .mf { color: #208050 } /* Literal.Number.Float */ +.highlight .mh { color: #208050 } /* Literal.Number.Hex */ +.highlight .mi { color: #208050 } /* Literal.Number.Integer */ +.highlight .mo { color: #208050 } /* Literal.Number.Oct */ +.highlight .sa { color: #4070a0 } /* Literal.String.Affix */ +.highlight .sb { color: #4070a0 } /* Literal.String.Backtick */ +.highlight .sc { color: #4070a0 } /* Literal.String.Char */ +.highlight .dl { color: #4070a0 } /* Literal.String.Delimiter */ +.highlight .sd { color: #4070a0; font-style: italic } /* Literal.String.Doc */ +.highlight .s2 { color: #4070a0 } /* Literal.String.Double */ +.highlight .se { color: #4070a0; font-weight: bold } /* Literal.String.Escape */ +.highlight .sh { color: #4070a0 } /* Literal.String.Heredoc */ +.highlight .si { color: #70a0d0; font-style: italic } /* Literal.String.Interpol */ +.highlight .sx { color: #c65d09 } /* Literal.String.Other */ +.highlight .sr { color: #235388 } /* Literal.String.Regex */ +.highlight .s1 { color: #4070a0 } /* Literal.String.Single */ +.highlight .ss { color: #517918 } /* Literal.String.Symbol */ +.highlight .bp { color: #007020 } /* Name.Builtin.Pseudo */ +.highlight .fm { color: #06287e } /* Name.Function.Magic */ +.highlight .vc { color: #bb60d5 } /* Name.Variable.Class */ +.highlight .vg { color: #bb60d5 } /* Name.Variable.Global */ +.highlight .vi { color: #bb60d5 } /* Name.Variable.Instance */ +.highlight .vm { color: #bb60d5 } /* Name.Variable.Magic */ +.highlight .il { color: #208050 } /* Literal.Number.Integer.Long */ \ No newline at end of file diff --git a/_static/searchtools.js b/_static/searchtools.js new file mode 100644 index 0000000..b08d58c --- /dev/null +++ b/_static/searchtools.js @@ -0,0 +1,620 @@ +/* + * searchtools.js + * ~~~~~~~~~~~~~~~~ + * + * Sphinx JavaScript utilities for the full-text search. + * + * :copyright: Copyright 2007-2024 by the Sphinx team, see AUTHORS. + * :license: BSD, see LICENSE for details. + * + */ +"use strict"; + +/** + * Simple result scoring code. + */ +if (typeof Scorer === "undefined") { + var Scorer = { + // Implement the following function to further tweak the score for each result + // The function takes a result array [docname, title, anchor, descr, score, filename] + // and returns the new score. + /* + score: result => { + const [docname, title, anchor, descr, score, filename] = result + return score + }, + */ + + // query matches the full name of an object + objNameMatch: 11, + // or matches in the last dotted part of the object name + objPartialMatch: 6, + // Additive scores depending on the priority of the object + objPrio: { + 0: 15, // used to be importantResults + 1: 5, // used to be objectResults + 2: -5, // used to be unimportantResults + }, + // Used when the priority is not in the mapping. + objPrioDefault: 0, + + // query found in title + title: 15, + partialTitle: 7, + // query found in terms + term: 5, + partialTerm: 2, + }; +} + +const _removeChildren = (element) => { + while (element && element.lastChild) element.removeChild(element.lastChild); +}; + +/** + * See https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions#escaping + */ +const _escapeRegExp = (string) => + string.replace(/[.*+\-?^${}()|[\]\\]/g, "\\$&"); // $& means the whole matched string + +const _displayItem = (item, searchTerms, highlightTerms) => { + const docBuilder = DOCUMENTATION_OPTIONS.BUILDER; + const docFileSuffix = DOCUMENTATION_OPTIONS.FILE_SUFFIX; + const docLinkSuffix = DOCUMENTATION_OPTIONS.LINK_SUFFIX; + const showSearchSummary = DOCUMENTATION_OPTIONS.SHOW_SEARCH_SUMMARY; + const contentRoot = document.documentElement.dataset.content_root; + + const [docName, title, anchor, descr, score, _filename] = item; + + let listItem = document.createElement("li"); + let requestUrl; + let linkUrl; + if (docBuilder === "dirhtml") { + // dirhtml builder + let dirname = docName + "/"; + if (dirname.match(/\/index\/$/)) + dirname = dirname.substring(0, dirname.length - 6); + else if (dirname === "index/") dirname = ""; + requestUrl = contentRoot + dirname; + linkUrl = requestUrl; + } else { + // normal html builders + requestUrl = contentRoot + docName + docFileSuffix; + linkUrl = docName + docLinkSuffix; + } + let linkEl = listItem.appendChild(document.createElement("a")); + linkEl.href = linkUrl + anchor; + linkEl.dataset.score = score; + linkEl.innerHTML = title; + if (descr) { + listItem.appendChild(document.createElement("span")).innerHTML = + " (" + descr + ")"; + // highlight search terms in the description + if (SPHINX_HIGHLIGHT_ENABLED) // set in sphinx_highlight.js + highlightTerms.forEach((term) => _highlightText(listItem, term, "highlighted")); + } + else if (showSearchSummary) + fetch(requestUrl) + .then((responseData) => responseData.text()) + .then((data) => { + if (data) + listItem.appendChild( + Search.makeSearchSummary(data, searchTerms, anchor) + ); + // highlight search terms in the summary + if (SPHINX_HIGHLIGHT_ENABLED) // set in sphinx_highlight.js + highlightTerms.forEach((term) => _highlightText(listItem, term, "highlighted")); + }); + Search.output.appendChild(listItem); +}; +const _finishSearch = (resultCount) => { + Search.stopPulse(); + Search.title.innerText = _("Search Results"); + if (!resultCount) + Search.status.innerText = Documentation.gettext( + "Your search did not match any documents. Please make sure that all words are spelled correctly and that you've selected enough categories." + ); + else + Search.status.innerText = _( + "Search finished, found ${resultCount} page(s) matching the search query." + ).replace('${resultCount}', resultCount); +}; +const _displayNextItem = ( + results, + resultCount, + searchTerms, + highlightTerms, +) => { + // results left, load the summary and display it + // this is intended to be dynamic (don't sub resultsCount) + if (results.length) { + _displayItem(results.pop(), searchTerms, highlightTerms); + setTimeout( + () => _displayNextItem(results, resultCount, searchTerms, highlightTerms), + 5 + ); + } + // search finished, update title and status message + else _finishSearch(resultCount); +}; +// Helper function used by query() to order search results. +// Each input is an array of [docname, title, anchor, descr, score, filename]. +// Order the results by score (in opposite order of appearance, since the +// `_displayNextItem` function uses pop() to retrieve items) and then alphabetically. +const _orderResultsByScoreThenName = (a, b) => { + const leftScore = a[4]; + const rightScore = b[4]; + if (leftScore === rightScore) { + // same score: sort alphabetically + const leftTitle = a[1].toLowerCase(); + const rightTitle = b[1].toLowerCase(); + if (leftTitle === rightTitle) return 0; + return leftTitle > rightTitle ? -1 : 1; // inverted is intentional + } + return leftScore > rightScore ? 1 : -1; +}; + +/** + * Default splitQuery function. Can be overridden in ``sphinx.search`` with a + * custom function per language. + * + * The regular expression works by splitting the string on consecutive characters + * that are not Unicode letters, numbers, underscores, or emoji characters. + * This is the same as ``\W+`` in Python, preserving the surrogate pair area. + */ +if (typeof splitQuery === "undefined") { + var splitQuery = (query) => query + .split(/[^\p{Letter}\p{Number}_\p{Emoji_Presentation}]+/gu) + .filter(term => term) // remove remaining empty strings +} + +/** + * Search Module + */ +const Search = { + _index: null, + _queued_query: null, + _pulse_status: -1, + + htmlToText: (htmlString, anchor) => { + const htmlElement = new DOMParser().parseFromString(htmlString, 'text/html'); + for (const removalQuery of [".headerlink", "script", "style"]) { + htmlElement.querySelectorAll(removalQuery).forEach((el) => { el.remove() }); + } + if (anchor) { + const anchorContent = htmlElement.querySelector(`[role="main"] ${anchor}`); + if (anchorContent) return anchorContent.textContent; + + console.warn( + `Anchored content block not found. Sphinx search tries to obtain it via DOM query '[role=main] ${anchor}'. Check your theme or template.` + ); + } + + // if anchor not specified or not found, fall back to main content + const docContent = htmlElement.querySelector('[role="main"]'); + if (docContent) return docContent.textContent; + + console.warn( + "Content block not found. Sphinx search tries to obtain it via DOM query '[role=main]'. Check your theme or template." + ); + return ""; + }, + + init: () => { + const query = new URLSearchParams(window.location.search).get("q"); + document + .querySelectorAll('input[name="q"]') + .forEach((el) => (el.value = query)); + if (query) Search.performSearch(query); + }, + + loadIndex: (url) => + (document.body.appendChild(document.createElement("script")).src = url), + + setIndex: (index) => { + Search._index = index; + if (Search._queued_query !== null) { + const query = Search._queued_query; + Search._queued_query = null; + Search.query(query); + } + }, + + hasIndex: () => Search._index !== null, + + deferQuery: (query) => (Search._queued_query = query), + + stopPulse: () => (Search._pulse_status = -1), + + startPulse: () => { + if (Search._pulse_status >= 0) return; + + const pulse = () => { + Search._pulse_status = (Search._pulse_status + 1) % 4; + Search.dots.innerText = ".".repeat(Search._pulse_status); + if (Search._pulse_status >= 0) window.setTimeout(pulse, 500); + }; + pulse(); + }, + + /** + * perform a search for something (or wait until index is loaded) + */ + performSearch: (query) => { + // create the required interface elements + const searchText = document.createElement("h2"); + searchText.textContent = _("Searching"); + const searchSummary = document.createElement("p"); + searchSummary.classList.add("search-summary"); + searchSummary.innerText = ""; + const searchList = document.createElement("ul"); + searchList.classList.add("search"); + + const out = document.getElementById("search-results"); + Search.title = out.appendChild(searchText); + Search.dots = Search.title.appendChild(document.createElement("span")); + Search.status = out.appendChild(searchSummary); + Search.output = out.appendChild(searchList); + + const searchProgress = document.getElementById("search-progress"); + // Some themes don't use the search progress node + if (searchProgress) { + searchProgress.innerText = _("Preparing search..."); + } + Search.startPulse(); + + // index already loaded, the browser was quick! + if (Search.hasIndex()) Search.query(query); + else Search.deferQuery(query); + }, + + _parseQuery: (query) => { + // stem the search terms and add them to the correct list + const stemmer = new Stemmer(); + const searchTerms = new Set(); + const excludedTerms = new Set(); + const highlightTerms = new Set(); + const objectTerms = new Set(splitQuery(query.toLowerCase().trim())); + splitQuery(query.trim()).forEach((queryTerm) => { + const queryTermLower = queryTerm.toLowerCase(); + + // maybe skip this "word" + // stopwords array is from language_data.js + if ( + stopwords.indexOf(queryTermLower) !== -1 || + queryTerm.match(/^\d+$/) + ) + return; + + // stem the word + let word = stemmer.stemWord(queryTermLower); + // select the correct list + if (word[0] === "-") excludedTerms.add(word.substr(1)); + else { + searchTerms.add(word); + highlightTerms.add(queryTermLower); + } + }); + + if (SPHINX_HIGHLIGHT_ENABLED) { // set in sphinx_highlight.js + localStorage.setItem("sphinx_highlight_terms", [...highlightTerms].join(" ")) + } + + // console.debug("SEARCH: searching for:"); + // console.info("required: ", [...searchTerms]); + // console.info("excluded: ", [...excludedTerms]); + + return [query, searchTerms, excludedTerms, highlightTerms, objectTerms]; + }, + + /** + * execute search (requires search index to be loaded) + */ + _performSearch: (query, searchTerms, excludedTerms, highlightTerms, objectTerms) => { + const filenames = Search._index.filenames; + const docNames = Search._index.docnames; + const titles = Search._index.titles; + const allTitles = Search._index.alltitles; + const indexEntries = Search._index.indexentries; + + // Collect multiple result groups to be sorted separately and then ordered. + // Each is an array of [docname, title, anchor, descr, score, filename]. + const normalResults = []; + const nonMainIndexResults = []; + + _removeChildren(document.getElementById("search-progress")); + + const queryLower = query.toLowerCase().trim(); + for (const [title, foundTitles] of Object.entries(allTitles)) { + if (title.toLowerCase().trim().includes(queryLower) && (queryLower.length >= title.length/2)) { + for (const [file, id] of foundTitles) { + const score = Math.round(Scorer.title * queryLower.length / title.length); + const boost = titles[file] === title ? 1 : 0; // add a boost for document titles + normalResults.push([ + docNames[file], + titles[file] !== title ? `${titles[file]} > ${title}` : title, + id !== null ? "#" + id : "", + null, + score + boost, + filenames[file], + ]); + } + } + } + + // search for explicit entries in index directives + for (const [entry, foundEntries] of Object.entries(indexEntries)) { + if (entry.includes(queryLower) && (queryLower.length >= entry.length/2)) { + for (const [file, id, isMain] of foundEntries) { + const score = Math.round(100 * queryLower.length / entry.length); + const result = [ + docNames[file], + titles[file], + id ? "#" + id : "", + null, + score, + filenames[file], + ]; + if (isMain) { + normalResults.push(result); + } else { + nonMainIndexResults.push(result); + } + } + } + } + + // lookup as object + objectTerms.forEach((term) => + normalResults.push(...Search.performObjectSearch(term, objectTerms)) + ); + + // lookup as search terms in fulltext + normalResults.push(...Search.performTermsSearch(searchTerms, excludedTerms)); + + // let the scorer override scores with a custom scoring function + if (Scorer.score) { + normalResults.forEach((item) => (item[4] = Scorer.score(item))); + nonMainIndexResults.forEach((item) => (item[4] = Scorer.score(item))); + } + + // Sort each group of results by score and then alphabetically by name. + normalResults.sort(_orderResultsByScoreThenName); + nonMainIndexResults.sort(_orderResultsByScoreThenName); + + // Combine the result groups in (reverse) order. + // Non-main index entries are typically arbitrary cross-references, + // so display them after other results. + let results = [...nonMainIndexResults, ...normalResults]; + + // remove duplicate search results + // note the reversing of results, so that in the case of duplicates, the highest-scoring entry is kept + let seen = new Set(); + results = results.reverse().reduce((acc, result) => { + let resultStr = result.slice(0, 4).concat([result[5]]).map(v => String(v)).join(','); + if (!seen.has(resultStr)) { + acc.push(result); + seen.add(resultStr); + } + return acc; + }, []); + + return results.reverse(); + }, + + query: (query) => { + const [searchQuery, searchTerms, excludedTerms, highlightTerms, objectTerms] = Search._parseQuery(query); + const results = Search._performSearch(searchQuery, searchTerms, excludedTerms, highlightTerms, objectTerms); + + // for debugging + //Search.lastresults = results.slice(); // a copy + // console.info("search results:", Search.lastresults); + + // print the results + _displayNextItem(results, results.length, searchTerms, highlightTerms); + }, + + /** + * search for object names + */ + performObjectSearch: (object, objectTerms) => { + const filenames = Search._index.filenames; + const docNames = Search._index.docnames; + const objects = Search._index.objects; + const objNames = Search._index.objnames; + const titles = Search._index.titles; + + const results = []; + + const objectSearchCallback = (prefix, match) => { + const name = match[4] + const fullname = (prefix ? prefix + "." : "") + name; + const fullnameLower = fullname.toLowerCase(); + if (fullnameLower.indexOf(object) < 0) return; + + let score = 0; + const parts = fullnameLower.split("."); + + // check for different match types: exact matches of full name or + // "last name" (i.e. last dotted part) + if (fullnameLower === object || parts.slice(-1)[0] === object) + score += Scorer.objNameMatch; + else if (parts.slice(-1)[0].indexOf(object) > -1) + score += Scorer.objPartialMatch; // matches in last name + + const objName = objNames[match[1]][2]; + const title = titles[match[0]]; + + // If more than one term searched for, we require other words to be + // found in the name/title/description + const otherTerms = new Set(objectTerms); + otherTerms.delete(object); + if (otherTerms.size > 0) { + const haystack = `${prefix} ${name} ${objName} ${title}`.toLowerCase(); + if ( + [...otherTerms].some((otherTerm) => haystack.indexOf(otherTerm) < 0) + ) + return; + } + + let anchor = match[3]; + if (anchor === "") anchor = fullname; + else if (anchor === "-") anchor = objNames[match[1]][1] + "-" + fullname; + + const descr = objName + _(", in ") + title; + + // add custom score for some objects according to scorer + if (Scorer.objPrio.hasOwnProperty(match[2])) + score += Scorer.objPrio[match[2]]; + else score += Scorer.objPrioDefault; + + results.push([ + docNames[match[0]], + fullname, + "#" + anchor, + descr, + score, + filenames[match[0]], + ]); + }; + Object.keys(objects).forEach((prefix) => + objects[prefix].forEach((array) => + objectSearchCallback(prefix, array) + ) + ); + return results; + }, + + /** + * search for full-text terms in the index + */ + performTermsSearch: (searchTerms, excludedTerms) => { + // prepare search + const terms = Search._index.terms; + const titleTerms = Search._index.titleterms; + const filenames = Search._index.filenames; + const docNames = Search._index.docnames; + const titles = Search._index.titles; + + const scoreMap = new Map(); + const fileMap = new Map(); + + // perform the search on the required terms + searchTerms.forEach((word) => { + const files = []; + const arr = [ + { files: terms[word], score: Scorer.term }, + { files: titleTerms[word], score: Scorer.title }, + ]; + // add support for partial matches + if (word.length > 2) { + const escapedWord = _escapeRegExp(word); + if (!terms.hasOwnProperty(word)) { + Object.keys(terms).forEach((term) => { + if (term.match(escapedWord)) + arr.push({ files: terms[term], score: Scorer.partialTerm }); + }); + } + if (!titleTerms.hasOwnProperty(word)) { + Object.keys(titleTerms).forEach((term) => { + if (term.match(escapedWord)) + arr.push({ files: titleTerms[term], score: Scorer.partialTitle }); + }); + } + } + + // no match but word was a required one + if (arr.every((record) => record.files === undefined)) return; + + // found search word in contents + arr.forEach((record) => { + if (record.files === undefined) return; + + let recordFiles = record.files; + if (recordFiles.length === undefined) recordFiles = [recordFiles]; + files.push(...recordFiles); + + // set score for the word in each file + recordFiles.forEach((file) => { + if (!scoreMap.has(file)) scoreMap.set(file, {}); + scoreMap.get(file)[word] = record.score; + }); + }); + + // create the mapping + files.forEach((file) => { + if (!fileMap.has(file)) fileMap.set(file, [word]); + else if (fileMap.get(file).indexOf(word) === -1) fileMap.get(file).push(word); + }); + }); + + // now check if the files don't contain excluded terms + const results = []; + for (const [file, wordList] of fileMap) { + // check if all requirements are matched + + // as search terms with length < 3 are discarded + const filteredTermCount = [...searchTerms].filter( + (term) => term.length > 2 + ).length; + if ( + wordList.length !== searchTerms.size && + wordList.length !== filteredTermCount + ) + continue; + + // ensure that none of the excluded terms is in the search result + if ( + [...excludedTerms].some( + (term) => + terms[term] === file || + titleTerms[term] === file || + (terms[term] || []).includes(file) || + (titleTerms[term] || []).includes(file) + ) + ) + break; + + // select one (max) score for the file. + const score = Math.max(...wordList.map((w) => scoreMap.get(file)[w])); + // add result to the result list + results.push([ + docNames[file], + titles[file], + "", + null, + score, + filenames[file], + ]); + } + return results; + }, + + /** + * helper function to return a node containing the + * search summary for a given text. keywords is a list + * of stemmed words. + */ + makeSearchSummary: (htmlText, keywords, anchor) => { + const text = Search.htmlToText(htmlText, anchor); + if (text === "") return null; + + const textLower = text.toLowerCase(); + const actualStartPosition = [...keywords] + .map((k) => textLower.indexOf(k.toLowerCase())) + .filter((i) => i > -1) + .slice(-1)[0]; + const startWithContext = Math.max(actualStartPosition - 120, 0); + + const top = startWithContext === 0 ? "" : "..."; + const tail = startWithContext + 240 < text.length ? "..." : ""; + + let summary = document.createElement("p"); + summary.classList.add("context"); + summary.textContent = top + text.substr(startWithContext, 240).trim() + tail; + + return summary; + }, +}; + +_ready(Search.init); diff --git a/_static/sphinx_highlight.js b/_static/sphinx_highlight.js new file mode 100644 index 0000000..8a96c69 --- /dev/null +++ b/_static/sphinx_highlight.js @@ -0,0 +1,154 @@ +/* Highlighting utilities for Sphinx HTML documentation. */ +"use strict"; + +const SPHINX_HIGHLIGHT_ENABLED = true + +/** + * highlight a given string on a node by wrapping it in + * span elements with the given class name. + */ +const _highlight = (node, addItems, text, className) => { + if (node.nodeType === Node.TEXT_NODE) { + const val = node.nodeValue; + const parent = node.parentNode; + const pos = val.toLowerCase().indexOf(text); + if ( + pos >= 0 && + !parent.classList.contains(className) && + !parent.classList.contains("nohighlight") + ) { + let span; + + const closestNode = parent.closest("body, svg, foreignObject"); + const isInSVG = closestNode && closestNode.matches("svg"); + if (isInSVG) { + span = document.createElementNS("http://www.w3.org/2000/svg", "tspan"); + } else { + span = document.createElement("span"); + span.classList.add(className); + } + + span.appendChild(document.createTextNode(val.substr(pos, text.length))); + const rest = document.createTextNode(val.substr(pos + text.length)); + parent.insertBefore( + span, + parent.insertBefore( + rest, + node.nextSibling + ) + ); + node.nodeValue = val.substr(0, pos); + /* There may be more occurrences of search term in this node. So call this + * function recursively on the remaining fragment. + */ + _highlight(rest, addItems, text, className); + + if (isInSVG) { + const rect = document.createElementNS( + "http://www.w3.org/2000/svg", + "rect" + ); + const bbox = parent.getBBox(); + rect.x.baseVal.value = bbox.x; + rect.y.baseVal.value = bbox.y; + rect.width.baseVal.value = bbox.width; + rect.height.baseVal.value = bbox.height; + rect.setAttribute("class", className); + addItems.push({ parent: parent, target: rect }); + } + } + } else if (node.matches && !node.matches("button, select, textarea")) { + node.childNodes.forEach((el) => _highlight(el, addItems, text, className)); + } +}; +const _highlightText = (thisNode, text, className) => { + let addItems = []; + _highlight(thisNode, addItems, text, className); + addItems.forEach((obj) => + obj.parent.insertAdjacentElement("beforebegin", obj.target) + ); +}; + +/** + * Small JavaScript module for the documentation. + */ +const SphinxHighlight = { + + /** + * highlight the search words provided in localstorage in the text + */ + highlightSearchWords: () => { + if (!SPHINX_HIGHLIGHT_ENABLED) return; // bail if no highlight + + // get and clear terms from localstorage + const url = new URL(window.location); + const highlight = + localStorage.getItem("sphinx_highlight_terms") + || url.searchParams.get("highlight") + || ""; + localStorage.removeItem("sphinx_highlight_terms") + url.searchParams.delete("highlight"); + window.history.replaceState({}, "", url); + + // get individual terms from highlight string + const terms = highlight.toLowerCase().split(/\s+/).filter(x => x); + if (terms.length === 0) return; // nothing to do + + // There should never be more than one element matching "div.body" + const divBody = document.querySelectorAll("div.body"); + const body = divBody.length ? divBody[0] : document.querySelector("body"); + window.setTimeout(() => { + terms.forEach((term) => _highlightText(body, term, "highlighted")); + }, 10); + + const searchBox = document.getElementById("searchbox"); + if (searchBox === null) return; + searchBox.appendChild( + document + .createRange() + .createContextualFragment( + '

' + + '' + + _("Hide Search Matches") + + "

" + ) + ); + }, + + /** + * helper function to hide the search marks again + */ + hideSearchWords: () => { + document + .querySelectorAll("#searchbox .highlight-link") + .forEach((el) => el.remove()); + document + .querySelectorAll("span.highlighted") + .forEach((el) => el.classList.remove("highlighted")); + localStorage.removeItem("sphinx_highlight_terms") + }, + + initEscapeListener: () => { + // only install a listener if it is really needed + if (!DOCUMENTATION_OPTIONS.ENABLE_SEARCH_SHORTCUTS) return; + + document.addEventListener("keydown", (event) => { + // bail for input elements + if (BLACKLISTED_KEY_CONTROL_ELEMENTS.has(document.activeElement.tagName)) return; + // bail with special keys + if (event.shiftKey || event.altKey || event.ctrlKey || event.metaKey) return; + if (DOCUMENTATION_OPTIONS.ENABLE_SEARCH_SHORTCUTS && (event.key === "Escape")) { + SphinxHighlight.hideSearchWords(); + event.preventDefault(); + } + }); + }, +}; + +_ready(() => { + /* Do not call highlightSearchWords() when we are on the search page. + * It will highlight words from the *previous* search query. + */ + if (typeof Search === "undefined") SphinxHighlight.highlightSearchWords(); + SphinxHighlight.initEscapeListener(); +}); diff --git a/genindex.html b/genindex.html new file mode 100644 index 0000000..1c4addc --- /dev/null +++ b/genindex.html @@ -0,0 +1,374 @@ + + + + + + + Index — HiSPARC-datastore 1.0.0 documentation + + + + + + + + + + + + +
+
+
+
+ + +

Index

+ +
+ A + | C + | D + | E + | G + | H + | I + | L + | M + | O + | P + | R + | S + | W + +
+

A

+ + +
+ +

C

+ + +
+ +

D

+ + + +
+ +

E

+ + +
    +
  • + examples.datastore_config_server + +
    • +
+ +

G

+ + + +
+ +

H

+ + + +
+ +

I

+ + +
+ +

L

+ + + +
+ +

M

+ + +
+ +

O

+ + +
+ +

P

+ + +
+ +

R

+ + +
+ +

S

+ + + +
+ +

W

+ + + +
    +
  • + writer.upload_codes + +
    • +
  • + writer.writer_app + +
    • +
  • + wsgi + +
    • +
  • + wsgi.rcodes + +
    • +
  • + wsgi.wsgi_app + +
    • +
+ + + +
+
+
+
+ +
+
+ + + + \ No newline at end of file diff --git a/index.html b/index.html new file mode 100644 index 0000000..5e5428f --- /dev/null +++ b/index.html @@ -0,0 +1,141 @@ + + + + + + + + Welcome to HiSPARC-datastore’s documentation! — HiSPARC-datastore 1.0.0 documentation + + + + + + + + + + + + + +
+
+
+
+ +
+

Welcome to HiSPARC-datastore’s documentation!

+
+

Contents:

+ +
+
+
+

Indices and tables

+ +
+ + +
+
+
+
+ +
+
+ + + + \ No newline at end of file diff --git a/introduction.html b/introduction.html new file mode 100644 index 0000000..eb20a9c --- /dev/null +++ b/introduction.html @@ -0,0 +1,125 @@ + + + + + + + + Introduction — HiSPARC-datastore 1.0.0 documentation + + + + + + + + + + + + + + +
+
+
+
+ +
+

Introduction

+

Todo:

+
+
    +

  • document datastore setup on frome

    • +

  • document station-list.csv <– pique

    • +

  • document raw datastore tables

    • +

  • document upload codes??

    • +

  • document logfiles

    • +
+
+
+ + +
+
+
+
+ +
+
+ + + + \ No newline at end of file diff --git a/objects.inv b/objects.inv new file mode 100644 index 0000000..810f3a2 Binary files /dev/null and b/objects.inv differ diff --git a/py-modindex.html b/py-modindex.html new file mode 100644 index 0000000..082cd30 --- /dev/null +++ b/py-modindex.html @@ -0,0 +1,152 @@ + + + + + + + Python Module Index — HiSPARC-datastore 1.0.0 documentation + + + + + + + + + + + + + + + +
+
+
+
+ + +

Python Module Index

+ +
+ e | + w +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
 
+ e
+ examples +
    + examples.datastore_config_server +
 
+ w
+ writer +
    + writer.storage +
    + writer.store_events +
    + writer.upload_codes +
    + writer.writer_app +
+ wsgi +
    + wsgi.rcodes +
    + wsgi.wsgi_app +
+ + +
+
+
+
+ +
+
+ + + + \ No newline at end of file diff --git a/search.html b/search.html new file mode 100644 index 0000000..9b85abf --- /dev/null +++ b/search.html @@ -0,0 +1,103 @@ + + + + + + + Search — HiSPARC-datastore 1.0.0 documentation + + + + + + + + + + + + + + + + + + + +
+
+
+
+ +

Search

+ + + + +

+ Searching for multiple words only shows matches that contain + all words. +

+ + +
+ + + +
+ + +
+ + +
+
+
+
+ +
+
+ + + + \ No newline at end of file diff --git a/searchindex.js b/searchindex.js new file mode 100644 index 0000000..c5fc12a --- /dev/null +++ b/searchindex.js @@ -0,0 +1 @@ +Search.setIndex({"alltitles": {"Contents:": [[0, null]], "Indices and tables": [[0, "indices-and-tables"]], "Introduction": [[1, null]], "Storage classes - raw datastore tables": [[2, "module-writer.storage"]], "Store events module": [[2, "module-writer.store_events"]], "Upload codes": [[2, "module-writer.upload_codes"]], "Welcome to HiSPARC-datastore\u2019s documentation!": [[0, null]], "Writer application": [[2, "module-writer.writer_app"]], "XML-RPC-Server": [[4, null]], "datastore WSGI app package": [[3, null]], "datastore writer package": [[2, null]], "wsgi.rcodes module": [[3, "module-wsgi.rcodes"]], "wsgi.wsgi_app module": [[3, "module-wsgi.wsgi_app"]]}, "docnames": ["index", "introduction", "writer", "wsgi", "xml-rpc-server"], "envversion": {"sphinx": 63, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx.ext.viewcode": 1}, "filenames": ["index.rst", "introduction.rst", "writer.rst", "wsgi.rst", "xml-rpc-server.rst"], "indexentries": {"application() (in module wsgi.wsgi_app)": [[3, "wsgi.wsgi_app.application", false]], "columns (writer.storage.hisparccomparator attribute)": [[2, "writer.storage.HisparcComparator.columns", false]], "columns (writer.storage.hisparcconfiguration attribute)": [[2, "writer.storage.HisparcConfiguration.columns", false]], "columns (writer.storage.hisparcerror attribute)": [[2, "writer.storage.HisparcError.columns", false]], "columns (writer.storage.hisparcevent attribute)": [[2, "writer.storage.HisparcEvent.columns", false]], "columns (writer.storage.hisparcsatellite attribute)": [[2, "writer.storage.HisparcSatellite.columns", false]], "columns (writer.storage.hisparcsingle attribute)": [[2, "writer.storage.HisparcSingle.columns", false]], "columns (writer.storage.lightningconfig attribute)": [[2, "writer.storage.LightningConfig.columns", false]], "columns (writer.storage.lightningerror attribute)": [[2, "writer.storage.LightningError.columns", false]], "columns (writer.storage.lightningevent attribute)": [[2, "writer.storage.LightningEvent.columns", false]], "columns (writer.storage.lightningnoise attribute)": [[2, "writer.storage.LightningNoise.columns", false]], "columns (writer.storage.lightningstatus attribute)": [[2, "writer.storage.LightningStatus.columns", false]], "columns (writer.storage.weatherconfig attribute)": [[2, "writer.storage.WeatherConfig.columns", false]], "columns (writer.storage.weathererror attribute)": [[2, "writer.storage.WeatherError.columns", false]], "columns (writer.storage.weatherevent attribute)": [[2, "writer.storage.WeatherEvent.columns", false]], "data_is_blob() (in module writer.store_events)": [[2, "writer.store_events.data_is_blob", false]], "decode_object() (in module writer.writer_app)": [[2, "writer.writer_app.decode_object", false]], "decode_object() (in module wsgi.wsgi_app)": [[3, "wsgi.wsgi_app.decode_object", false]], "do_init() (in module wsgi.wsgi_app)": [[3, "wsgi.wsgi_app.do_init", false]], "examples.datastore_config_server": [[4, "module-examples.datastore_config_server", false]], "get_or_create_cluster_group() (in module writer.storage)": [[2, "writer.storage.get_or_create_cluster_group", false]], "get_or_create_node() (in module writer.storage)": [[2, "writer.storage.get_or_create_node", false]], "get_or_create_station_group() (in module writer.storage)": [[2, "writer.storage.get_or_create_station_group", false]], "hisparccomparator (class in writer.storage)": [[2, "writer.storage.HisparcComparator", false]], "hisparcconfiguration (class in writer.storage)": [[2, "writer.storage.HisparcConfiguration", false]], "hisparcerror (class in writer.storage)": [[2, "writer.storage.HisparcError", false]], "hisparcevent (class in writer.storage)": [[2, "writer.storage.HisparcEvent", false]], "hisparcsatellite (class in writer.storage)": [[2, "writer.storage.HisparcSatellite", false]], "hisparcsingle (class in writer.storage)": [[2, "writer.storage.HisparcSingle", false]], "is_data_suspicious() (in module wsgi.wsgi_app)": [[3, "wsgi.wsgi_app.is_data_suspicious", false]], "lightningconfig (class in writer.storage)": [[2, "writer.storage.LightningConfig", false]], "lightningerror (class in writer.storage)": [[2, "writer.storage.LightningError", false]], "lightningevent (class in writer.storage)": [[2, "writer.storage.LightningEvent", false]], "lightningnoise (class in writer.storage)": [[2, "writer.storage.LightningNoise", false]], "lightningstatus (class in writer.storage)": [[2, "writer.storage.LightningStatus", false]], "module": [[2, "module-writer", false], [2, "module-writer.storage", false], [2, "module-writer.store_events", false], [2, "module-writer.upload_codes", false], [2, "module-writer.writer_app", false], [3, "module-wsgi", false], [3, "module-wsgi.rcodes", false], [3, "module-wsgi.wsgi_app", false], [4, "module-examples.datastore_config_server", false]], "open_or_create_file() (in module writer.storage)": [[2, "writer.storage.open_or_create_file", false]], "process_data() (in module writer.writer_app)": [[2, "writer.writer_app.process_data", false]], "reload_datastore() (in module examples.datastore_config_server)": [[4, "examples.datastore_config_server.reload_datastore", false]], "store_data() (in module wsgi.wsgi_app)": [[3, "wsgi.wsgi_app.store_data", false]], "store_event() (in module writer.store_events)": [[2, "writer.store_events.store_event", false]], "store_event_list() (in module writer.store_events)": [[2, "writer.store_events.store_event_list", false]], "weatherconfig (class in writer.storage)": [[2, "writer.storage.WeatherConfig", false]], "weathererror (class in writer.storage)": [[2, "writer.storage.WeatherError", false]], "weatherevent (class in writer.storage)": [[2, "writer.storage.WeatherEvent", false]], "writer": [[2, "module-writer", false]], "writer() (in module writer.writer_app)": [[2, "writer.writer_app.writer", false]], "writer.storage": [[2, "module-writer.storage", false]], "writer.store_events": [[2, "module-writer.store_events", false]], "writer.upload_codes": [[2, "module-writer.upload_codes", false]], "writer.writer_app": [[2, "module-writer.writer_app", false]], "wsgi": [[3, "module-wsgi", false]], "wsgi.rcodes": [[3, "module-wsgi.rcodes", false]], "wsgi.wsgi_app": [[3, "module-wsgi.wsgi_app", false]]}, "objects": {"": [[2, 0, 0, "-", "writer"], [3, 0, 0, "-", "wsgi"]], "examples": [[4, 0, 0, "-", "datastore_config_server"]], "examples.datastore_config_server": [[4, 1, 1, "", "reload_datastore"]], "writer": [[2, 0, 0, "-", "storage"], [2, 0, 0, "-", "store_events"], [2, 0, 0, "-", "upload_codes"], [2, 0, 0, "-", "writer_app"]], "writer.storage": [[2, 2, 1, "", "HisparcComparator"], [2, 2, 1, "", "HisparcConfiguration"], [2, 2, 1, "", "HisparcError"], [2, 2, 1, "", "HisparcEvent"], [2, 2, 1, "", "HisparcSatellite"], [2, 2, 1, "", "HisparcSingle"], [2, 2, 1, "", "LightningConfig"], [2, 2, 1, "", "LightningError"], [2, 2, 1, "", "LightningEvent"], [2, 2, 1, "", "LightningNoise"], [2, 2, 1, "", "LightningStatus"], [2, 2, 1, "", "WeatherConfig"], [2, 2, 1, "", "WeatherError"], [2, 2, 1, "", "WeatherEvent"], [2, 1, 1, "", "get_or_create_cluster_group"], [2, 1, 1, "", "get_or_create_node"], [2, 1, 1, "", "get_or_create_station_group"], [2, 1, 1, "", "open_or_create_file"]], "writer.storage.HisparcComparator": [[2, 3, 1, "", "columns"]], "writer.storage.HisparcConfiguration": [[2, 3, 1, "", "columns"]], "writer.storage.HisparcError": [[2, 3, 1, "", "columns"]], "writer.storage.HisparcEvent": [[2, 3, 1, "", "columns"]], "writer.storage.HisparcSatellite": [[2, 3, 1, "", "columns"]], "writer.storage.HisparcSingle": [[2, 3, 1, "", "columns"]], "writer.storage.LightningConfig": [[2, 3, 1, "", "columns"]], "writer.storage.LightningError": [[2, 3, 1, "", "columns"]], "writer.storage.LightningEvent": [[2, 3, 1, "", "columns"]], "writer.storage.LightningNoise": [[2, 3, 1, "", "columns"]], "writer.storage.LightningStatus": [[2, 3, 1, "", "columns"]], "writer.storage.WeatherConfig": [[2, 3, 1, "", "columns"]], "writer.storage.WeatherError": [[2, 3, 1, "", "columns"]], "writer.storage.WeatherEvent": [[2, 3, 1, "", "columns"]], "writer.store_events": [[2, 1, 1, "", "data_is_blob"], [2, 1, 1, "", "store_event"], [2, 1, 1, "", "store_event_list"]], "writer.writer_app": [[2, 1, 1, "", "decode_object"], [2, 1, 1, "", "process_data"], [2, 1, 1, "", "writer"]], "wsgi": [[3, 0, 0, "-", "rcodes"], [3, 0, 0, "-", "wsgi_app"]], "wsgi.wsgi_app": [[3, 1, 1, "", "application"], [3, 1, 1, "", "decode_object"], [3, 1, 1, "", "do_init"], [3, 1, 1, "", "is_data_suspicious"], [3, 1, 1, "", "store_data"]]}, "objnames": {"0": ["py", "module", "Python module"], "1": ["py", "function", "Python function"], "2": ["py", "class", "Python class"], "3": ["py", "attribute", "Python attribute"]}, "objtypes": {"0": "py:module", "1": "py:function", "2": "py:class", "3": "py:attribute"}, "terms": {"": 4, "0": [2, 3], "00": 3, "1": 2, "10": 2, "100": 3, "11": 2, "12": 2, "13": 2, "14": 2, "15": 2, "16": 2, "17": 2, "18": 2, "19": 2, "2": 2, "20": [2, 3], "201": 3, "2019": 3, "203": 3, "206": 3, "208": 3, "21": 2, "22": 2, "23": 2, "24": 2, "25": 2, "26": 2, "27": 2, "28": 2, "29": 2, "3": 2, "30": [2, 3], "31": 2, "32": 2, "33": 2, "34": 2, "35": 2, "36": 2, "37": 2, "38": 2, "39": 2, "4": 2, "40": [2, 3], "400": 3, "41": 2, "42": 2, "43": 2, "44": 2, "45": 2, "46": 2, "47": 2, "48": 2, "49": 2, "5": [2, 3], "50": 2, "51": 2, "52": 2, "53": 2, "54": 2, "55": 2, "56": 2, "57": 2, "58": 2, "59": 2, "6": 2, "60": 2, "61": 2, "62": 2, "63": 2, "64": 2, "65": 2, "66": 2, "67": 2, "68": 2, "69": 2, "7": [2, 3], "70": 2, "71": 2, "72": 2, "73": 2, "74": 2, "75": 2, "76": 2, "77": 2, "78": 2, "79": 2, "8": 2, "80": 2, "81": 2, "82": 2, "83": 2, "84": 2, "85": 2, "86": 2, "87": 2, "88": 2, "9": 2, "By": 3, "If": 2, "It": [2, 4], "Such": 3, "The": [3, 4], "There": 3, "accord": 2, "acquir": 3, "actual": 3, "after": 3, "all": [2, 3], "alreadi": 3, "altitud": 2, "an": [2, 4], "angle_correct": 2, "app": [0, 2], "append": [2, 3], "applic": [0, 3], "apr": 3, "ar": 3, "bad": 3, "baromet": 2, "baselin": 2, "basi": 4, "baud_rat": 2, "been": 3, "befor": 3, "belong": 2, "belov": 3, "between": [2, 3], "bidirect": 3, "bin": [2, 3], "binari": 2, "birth": 3, "blob": 2, "blob_typ": 2, "boolcol": 2, "buffer": 2, "bytestr": [2, 3], "call": 3, "can": 3, "caus": 3, "channel": 3, "check": 3, "checksum": 3, "class": 0, "close_alarm": 2, "close_alarm_dist": 2, "close_r": 2, "cluster": [2, 3, 4], "code": [0, 1, 3, 4], "coinctim": 2, "cold": 3, "column": 2, "com_port": 2, "combin": 3, "come": 3, "commun": 3, "compar": 2, "config": [2, 3, 4], "configfil": [2, 3], "configu": [2, 3], "configur": [2, 3], "contain": 2, "corr_angl": 2, "corr_dist": 2, "count": 2, "creat": 2, "csv": [1, 2, 3], "current": 3, "current_head": 2, "daemon": 4, "daq": 3, "daq_mod": 2, "data": [2, 3], "data_dir": [2, 3], "data_is_blob": 2, "data_reduct": 2, "database_nam": 2, "datafil": 2, "datastor": [1, 4], "datastore_config_serv": 4, "datastream": 3, "datator": 2, "date": 2, "debug": [2, 3], "decod": [2, 3], "decode_object": [2, 3], "default": 3, "delay_check": 2, "delay_error": 2, "delay_screen": 2, "determin": 2, "detnum": 2, "devic": 2, "dew_point": 2, "dflt": 2, "dictionari": 3, "directori": 2, "do": 3, "do_init": 3, "docstr": 2, "document": [1, 4], "doe": [2, 3], "don": 3, "e": 2, "eigenst": 3, "elimin": 3, "empti": 2, "entir": 3, "environ": 3, "error": [2, 3], "evapotranspir": 2, "even": 3, "event": [0, 3], "event_id": 2, "event_list": [2, 3], "event_r": 2, "everi": 3, "everyth": 3, "exampl": [2, 3, 4], "exist": 2, "ext_timestamp": 2, "extend": 4, "fals": 2, "fate": 3, "file": [2, 3], "filesystem": 2, "first": 3, "float32col": 2, "float64col": 2, "four": 3, "from": [2, 3, 4], "frome": [1, 2, 3], "function": [2, 3], "functool": 3, "g": 2, "gener": [2, 3], "get": 2, "get_or_create_cluster_group": 2, "get_or_create_nod": 2, "get_or_create_station_group": 2, "getev": 3, "global": 3, "gp": 3, "gps_altitud": 2, "gps_latitud": 2, "gps_longitud": 2, "group": 2, "h5": 2, "ha": 3, "handl": 4, "handler": 3, "happili": 3, "hdf5": 2, "heat_index": 2, "help_url": 2, "hisparc": [2, 3, 4], "hisparccompar": 2, "hisparcconfigur": 2, "hisparcerror": 2, "hisparcev": 2, "hisparcsatellit": 2, "hisparcsingl": 2, "http": [3, 4], "humidity_insid": 2, "humidity_outsid": 2, "hunt": 3, "i": [2, 3, 4], "id": 2, "import": [2, 3], "incom": 2, "index": 0, "inform": 3, "ini": [2, 3], "initi": 3, "input": 3, "int16col": 2, "int32col": 2, "integr": 2, "intern": 3, "introduct": 0, "invalid": 3, "is_data_suspici": 3, "know": 3, "latitud": 2, "layout": 4, "length": 2, "librari": 4, "lightningconfig": 2, "lightningerror": 2, "lightningev": 2, "lightningnois": 2, "lightningstatu": 2, "list": [1, 2, 3], "live": 3, "load": [3, 4], "log": [2, 3], "logfil": 1, "logger": 3, "loglevel": [2, 3], "longitud": 2, "mai": 3, "mas_ch1_adc_gain": 2, "mas_ch1_adc_offset": 2, "mas_ch1_comp_gain": 2, "mas_ch1_comp_offset": 2, "mas_ch1_curr": 2, "mas_ch1_gain_neg": 2, "mas_ch1_gain_po": 2, "mas_ch1_high": 2, "mas_ch1_inttim": 2, "mas_ch1_low": 2, "mas_ch1_offset_neg": 2, "mas_ch1_offset_po": 2, "mas_ch1_thres_high": 2, "mas_ch1_thres_low": 2, "mas_ch1_voltag": 2, "mas_ch2_adc_gain": 2, "mas_ch2_adc_offset": 2, "mas_ch2_comp_gain": 2, "mas_ch2_comp_offset": 2, "mas_ch2_curr": 2, "mas_ch2_gain_neg": 2, "mas_ch2_gain_po": 2, "mas_ch2_high": 2, "mas_ch2_inttim": 2, "mas_ch2_low": 2, "mas_ch2_offset_neg": 2, "mas_ch2_offset_po": 2, "mas_ch2_thres_high": 2, "mas_ch2_thres_low": 2, "mas_ch2_voltag": 2, "mas_common_offset": 2, "mas_comp_thres_high": 2, "mas_comp_thres_low": 2, "mas_internal_voltag": 2, "mas_max_voltag": 2, "mas_reset": 2, "mas_vers": 2, "match": 3, "max_n": 2, "max_sign": 2, "mean_n": 2, "mean_sign": 2, "messag": 2, "min_n": 2, "min_sign": 2, "minimum_gps_spe": 2, "modul": 0, "most": 3, "must": 3, "n_peak": 2, "name": 2, "nanosecond": 2, "necessari": 4, "need": 3, "new": 2, "nikef": 3, "nl": 3, "node": 2, "noise_beep": 2, "note": 3, "number": [2, 3], "o": [2, 3], "object": [2, 3], "offset_bar_sea_level": 2, "offset_inside_humid": 2, "offset_inside_temperatur": 2, "offset_outside_humid": 2, "offset_outside_temperatur": 2, "offset_station_altitud": 2, "offset_wind_direct": 2, "ok": 3, "one": [2, 3], "open": 2, "open_or_create_fil": 2, "our": 3, "out": 3, "packag": 0, "page": 0, "paramet": 2, "parent": 2, "partial": 3, "pass": 3, "password": [2, 3, 4], "past": 3, "path": [2, 3], "pickl": 2, "piqu": 1, "po": 2, "poll": 2, "post": 3, "postcoinctim": 2, "precoinctim": 2, "previous": 3, "probabl": 3, "process_data": 2, "pulseheight": 2, "putev": 3, "py": 2, "pytabl": 2, "python": 4, "quantum": 3, "queue": 2, "rain_rat": 2, "raw": [0, 1], "rcode": 0, "read": [2, 3], "readlin": 3, "recurs": [2, 3], "reduce_data": 2, "reload": 4, "reload_datastor": [0, 4], "request": 3, "reread": 3, "return": 3, "reweav": 3, "rip": 4, "rpc": 0, "run": 4, "script": 2, "search": 0, "send": 3, "serv": 3, "server": [0, 3], "set": 3, "setup": 1, "sever_alarm": 2, "severe_alarm_dist": 2, "shape": 2, "share": [2, 3], "should": 4, "signal": 3, "signifi": 3, "simpl": 4, "simplexmlrpcserv": 4, "sleep": [2, 3], "slv_ch1_adc_gain": 2, "slv_ch1_adc_offset": 2, "slv_ch1_comp_gain": 2, "slv_ch1_comp_offset": 2, "slv_ch1_current": 2, "slv_ch1_gain_neg": 2, "slv_ch1_gain_po": 2, "slv_ch1_high": 2, "slv_ch1_inttim": 2, "slv_ch1_low": 2, "slv_ch1_offset_neg": 2, "slv_ch1_offset_po": 2, "slv_ch1_thres_high": 2, "slv_ch1_thres_low": 2, "slv_ch1_voltag": 2, "slv_ch2_adc_gain": 2, "slv_ch2_adc_offset": 2, "slv_ch2_comp_gain": 2, "slv_ch2_comp_offset": 2, "slv_ch2_current": 2, "slv_ch2_gain_neg": 2, "slv_ch2_gain_po": 2, "slv_ch2_high": 2, "slv_ch2_inttim": 2, "slv_ch2_low": 2, "slv_ch2_offset_neg": 2, "slv_ch2_offset_po": 2, "slv_ch2_thres_high": 2, "slv_ch2_thres_low": 2, "slv_ch2_voltag": 2, "slv_common_offset": 2, "slv_comp_thres_high": 2, "slv_comp_thres_low": 2, "slv_internal_voltag": 2, "slv_max_voltag": 2, "slv_reset": 2, "slv_version": 2, "solar_rad": 2, "solar_radi": 2, "someon": 3, "sourc": [2, 3, 4], "spare_byt": 2, "squelch_set": 2, "start": 3, "start_respons": 3, "startmod": 2, "station": [1, 2, 3, 4], "station_id": [2, 3], "station_list": [2, 3], "std_dev": 2, "storag": [0, 3], "store": [0, 3], "store_data": 3, "store_ev": 2, "store_event_list": [2, 3], "suspici": 3, "sy": [2, 3], "t": 3, "tabl": 1, "temp_insid": 2, "temp_outsid": 2, "temperature_insid": 2, "temperature_outsid": 2, "temporari": 3, "thi": [2, 3, 4], "time": 3, "time32col": 2, "timestamp": [2, 3], "tmp": [2, 3], "todo": 1, "total_r": 2, "trace": 2, "trig_and_or": 2, "trig_extern": 2, "trig_high_sign": 2, "trig_low_sign": 2, "trigger_pattern": 2, "try": 3, "type": 3, "u": 3, "uint16col": 2, "uint32col": 2, "uint64col": 2, "uint8col": 2, "uncorr_angl": 2, "uncorr_dist": 2, "univers": 3, "unknown": 3, "unpickl": 3, "up": 3, "updat": 4, "upload": [0, 1, 3], "uploadcod": 2, "url": 3, "us": [2, 3], "use_filt": 2, "use_filter_threshold": 2, "usual": [2, 3], "uv": 2, "uv_index": 2, "uwsgi": 3, "valu": 2, "var": [2, 3], "variabl": [2, 3], "verifi": 3, "wa": 4, "we": 3, "weatherconfig": 2, "weathererror": 2, "weatherev": 2, "when": [3, 4], "whenev": 3, "which": 2, "wind_chil": 2, "wind_dir": 2, "wind_direct": 2, "wind_spe": 2, "without": 3, "wrapper": 2, "write": 2, "writer": [0, 3], "writer_app": 2, "written": 2, "wrong": 3, "wsgi": [0, 2], "wsgi_app": 0, "www": [2, 3], "xml": 0, "yet": [2, 3]}, "titles": ["Welcome to HiSPARC-datastore\u2019s documentation!", "Introduction", "datastore writer package", "datastore WSGI app package", "XML-RPC-Server"], "titleterms": {"": 0, "app": 3, "applic": 2, "class": 2, "code": 2, "content": 0, "datastor": [0, 2, 3], "document": 0, "event": 2, "hisparc": 0, "indic": 0, "introduct": 1, "modul": [2, 3], "packag": [2, 3], "raw": 2, "rcode": 3, "rpc": 4, "server": 4, "storag": 2, "store": 2, "tabl": [0, 2], "upload": 2, "welcom": 0, "writer": 2, "wsgi": 3, "wsgi_app": 3, "xml": 4}}) \ No newline at end of file diff --git a/writer.html b/writer.html new file mode 100644 index 0000000..7b033d7 --- /dev/null +++ b/writer.html @@ -0,0 +1,505 @@ + + + + + + + + datastore writer package — HiSPARC-datastore 1.0.0 documentation + + + + + + + + + + + + + + +
+
+
+
+ +
+

datastore writer package

+

HiSPARC datastore writer application

+

This script polls /datatore/frome/incoming for incoming data written +by the WSGI app. It then writes the data into the raw datastore.

+

example writer_app.py:

+
"""Wrapper for the writer application"""
+
+import sys
+
+sys.path.append('/var/www/wsgi-bin/datastore/')
+
+from writer import writer_app
+
+configfile = '/var/www/wsgi-bin/datastore/examples/config.ini'
+writer_app.writer(configfile)
+
+
+

configuration is read from a configuation file, shared between the WSGI app +and the writer usually config.ini:

+
[General]
+log=/tmp/hisparc.log
+loglevel=debug
+station_list=/tmp/station_list.csv
+data_dir=/tmp/datastore
+
+[Writer]
+sleep=5
+
+
+
+

Writer application

+

datastore writer application

+

This module empties the station data incoming queue and writes the +data into HDF5 files using PyTables.

+
+
+writer.writer_app.decode_object(o)[source]
+

recursively decode all bytestrings in object

+
+ +
+
+writer.writer_app.process_data(file, data_dir)[source]
+

Read data from a pickled object and store store in raw datastore

+
+ +
+
+writer.writer_app.writer(configfile)[source]
+

hisparc datastore writer application

+

This script polls /datatore/frome/incoming for incoming data written +by the WSGI app. It then store the data into the raw datastore.

+

Configuration is read from the datastore configuation file (usually +config.ini):

+
[General]
+log=/tmp/hisparc.log
+loglevel=debug
+station_list=/tmp/station_list.csv
+data_dir=/tmp/datastore
+
+[Writer]
+sleep=5
+
+
+
+ +
+
+

Storage classes - raw datastore tables

+

Storage docstrings

+
+
+class writer.storage.HisparcComparator
+
+
+columns = {'comparator': UInt8Col(shape=(), dflt=0, pos=5), 'count': UInt16Col(shape=(), dflt=0, pos=6), 'device': UInt8Col(shape=(), dflt=0, pos=4), 'event_id': UInt32Col(shape=(), dflt=0, pos=0), 'ext_timestamp': UInt64Col(shape=(), dflt=0, pos=3), 'nanoseconds': UInt32Col(shape=(), dflt=0, pos=2), 'timestamp': Time32Col(shape=(), dflt=0, pos=1)}
+
+ +
+ +
+
+class writer.storage.HisparcConfiguration
+
+
+columns = {'buffer': Int32Col(shape=(), dflt=-1, pos=20), 'coinctime': Float64Col(shape=(), dflt=0.0, pos=12), 'delay_check': Float64Col(shape=(), dflt=0.0, pos=23), 'delay_error': Float64Col(shape=(), dflt=0.0, pos=24), 'delay_screen': Float64Col(shape=(), dflt=0.0, pos=22), 'detnum': UInt16Col(shape=(), dflt=0, pos=14), 'event_id': UInt32Col(shape=(), dflt=0, pos=0), 'gps_altitude': Float64Col(shape=(), dflt=0.0, pos=4), 'gps_latitude': Float64Col(shape=(), dflt=0.0, pos=2), 'gps_longitude': Float64Col(shape=(), dflt=0.0, pos=3), 'mas_ch1_adc_gain': Float64Col(shape=(), dflt=0.0, pos=49), 'mas_ch1_adc_offset': Float64Col(shape=(), dflt=0.0, pos=50), 'mas_ch1_comp_gain': Float64Col(shape=(), dflt=0.0, pos=53), 'mas_ch1_comp_offset': Float64Col(shape=(), dflt=0.0, pos=54), 'mas_ch1_current': Float64Col(shape=(), dflt=0.0, pos=33), 'mas_ch1_gain_neg': UInt8Col(shape=(), dflt=0, pos=40), 'mas_ch1_gain_pos': UInt8Col(shape=(), dflt=0, pos=39), 'mas_ch1_inttime': Float64Col(shape=(), dflt=0.0, pos=29), 'mas_ch1_offset_neg': UInt8Col(shape=(), dflt=0, pos=44), 'mas_ch1_offset_pos': UInt8Col(shape=(), dflt=0, pos=43), 'mas_ch1_thres_high': Float64Col(shape=(), dflt=0.0, pos=26), 'mas_ch1_thres_low': Float64Col(shape=(), dflt=0.0, pos=25), 'mas_ch1_voltage': Float64Col(shape=(), dflt=0.0, pos=31), 'mas_ch2_adc_gain': Float64Col(shape=(), dflt=0.0, pos=51), 'mas_ch2_adc_offset': Float64Col(shape=(), dflt=0.0, pos=52), 'mas_ch2_comp_gain': Float64Col(shape=(), dflt=0.0, pos=55), 'mas_ch2_comp_offset': Float64Col(shape=(), dflt=0.0, pos=56), 'mas_ch2_current': Float64Col(shape=(), dflt=0.0, pos=34), 'mas_ch2_gain_neg': UInt8Col(shape=(), dflt=0, pos=42), 'mas_ch2_gain_pos': UInt8Col(shape=(), dflt=0, pos=41), 'mas_ch2_inttime': Float64Col(shape=(), dflt=0.0, pos=30), 'mas_ch2_offset_neg': UInt8Col(shape=(), dflt=0, pos=46), 'mas_ch2_offset_pos': UInt8Col(shape=(), dflt=0, pos=45), 'mas_ch2_thres_high': Float64Col(shape=(), dflt=0.0, pos=28), 'mas_ch2_thres_low': Float64Col(shape=(), dflt=0.0, pos=27), 'mas_ch2_voltage': Float64Col(shape=(), dflt=0.0, pos=32), 'mas_common_offset': UInt8Col(shape=(), dflt=0, pos=47), 'mas_comp_thres_high': Float64Col(shape=(), dflt=0.0, pos=36), 'mas_comp_thres_low': Float64Col(shape=(), dflt=0.0, pos=35), 'mas_internal_voltage': UInt8Col(shape=(), dflt=0, pos=48), 'mas_max_voltage': Float64Col(shape=(), dflt=0.0, pos=37), 'mas_reset': BoolCol(shape=(), dflt=False, pos=38), 'mas_version': Int32Col(shape=(), dflt=-1, pos=5), 'password': Int32Col(shape=(), dflt=-1, pos=15), 'postcoinctime': Float64Col(shape=(), dflt=0.0, pos=13), 'precoinctime': Float64Col(shape=(), dflt=0.0, pos=11), 'reduce_data': BoolCol(shape=(), dflt=False, pos=19), 'slv_ch1_adc_gain': Float64Col(shape=(), dflt=0.0, pos=81), 'slv_ch1_adc_offset': Float64Col(shape=(), dflt=0.0, pos=82), 'slv_ch1_comp_gain': Float64Col(shape=(), dflt=0.0, pos=85), 'slv_ch1_comp_offset': Float64Col(shape=(), dflt=0.0, pos=86), 'slv_ch1_current': Float64Col(shape=(), dflt=0.0, pos=65), 'slv_ch1_gain_neg': UInt8Col(shape=(), dflt=0, pos=72), 'slv_ch1_gain_pos': UInt8Col(shape=(), dflt=0, pos=71), 'slv_ch1_inttime': Float64Col(shape=(), dflt=0.0, pos=61), 'slv_ch1_offset_neg': UInt8Col(shape=(), dflt=0, pos=76), 'slv_ch1_offset_pos': UInt8Col(shape=(), dflt=0, pos=75), 'slv_ch1_thres_high': Float64Col(shape=(), dflt=0.0, pos=58), 'slv_ch1_thres_low': Float64Col(shape=(), dflt=0.0, pos=57), 'slv_ch1_voltage': Float64Col(shape=(), dflt=0.0, pos=63), 'slv_ch2_adc_gain': Float64Col(shape=(), dflt=0.0, pos=83), 'slv_ch2_adc_offset': Float64Col(shape=(), dflt=0.0, pos=84), 'slv_ch2_comp_gain': Float64Col(shape=(), dflt=0.0, pos=87), 'slv_ch2_comp_offset': Float64Col(shape=(), dflt=0.0, pos=88), 'slv_ch2_current': Float64Col(shape=(), dflt=0.0, pos=66), 'slv_ch2_gain_neg': UInt8Col(shape=(), dflt=0, pos=74), 'slv_ch2_gain_pos': UInt8Col(shape=(), dflt=0, pos=73), 'slv_ch2_inttime': Float64Col(shape=(), dflt=0.0, pos=62), 'slv_ch2_offset_neg': UInt8Col(shape=(), dflt=0, pos=78), 'slv_ch2_offset_pos': UInt8Col(shape=(), dflt=0, pos=77), 'slv_ch2_thres_high': Float64Col(shape=(), dflt=0.0, pos=60), 'slv_ch2_thres_low': Float64Col(shape=(), dflt=0.0, pos=59), 'slv_ch2_voltage': Float64Col(shape=(), dflt=0.0, pos=64), 'slv_common_offset': UInt8Col(shape=(), dflt=0, pos=79), 'slv_comp_thres_high': Float64Col(shape=(), dflt=0.0, pos=68), 'slv_comp_thres_low': Float64Col(shape=(), dflt=0.0, pos=67), 'slv_internal_voltage': UInt8Col(shape=(), dflt=0, pos=80), 'slv_max_voltage': Float64Col(shape=(), dflt=0.0, pos=69), 'slv_reset': BoolCol(shape=(), dflt=False, pos=70), 'slv_version': Int32Col(shape=(), dflt=-1, pos=6), 'spare_bytes': UInt8Col(shape=(), dflt=0, pos=16), 'startmode': BoolCol(shape=(), dflt=False, pos=21), 'timestamp': Time32Col(shape=(), dflt=0, pos=1), 'trig_and_or': BoolCol(shape=(), dflt=False, pos=10), 'trig_external': UInt32Col(shape=(), dflt=0, pos=9), 'trig_high_signals': UInt32Col(shape=(), dflt=0, pos=8), 'trig_low_signals': UInt32Col(shape=(), dflt=0, pos=7), 'use_filter': BoolCol(shape=(), dflt=False, pos=17), 'use_filter_threshold': BoolCol(shape=(), dflt=False, pos=18)}
+
+ +
+ +
+
+class writer.storage.HisparcError
+

HiSPARC Error messages tables

+
+
+columns = {'event_id': UInt32Col(shape=(), dflt=0, pos=0), 'messages': Int32Col(shape=(), dflt=-1, pos=2), 'timestamp': Time32Col(shape=(), dflt=0, pos=1)}
+
+ +
+ +
+
+class writer.storage.HisparcEvent
+
+
+columns = {'baseline': Int16Col(shape=(4,), dflt=-1, pos=6), 'data_reduction': BoolCol(shape=(), dflt=False, pos=4), 'event_id': UInt32Col(shape=(), dflt=0, pos=0), 'event_rate': Float32Col(shape=(), dflt=0.0, pos=12), 'ext_timestamp': UInt64Col(shape=(), dflt=0, pos=3), 'integrals': Int32Col(shape=(4,), dflt=-1, pos=10), 'n_peaks': Int16Col(shape=(4,), dflt=-1, pos=8), 'nanoseconds': UInt32Col(shape=(), dflt=0, pos=2), 'pulseheights': Int16Col(shape=(4,), dflt=-1, pos=9), 'std_dev': Int16Col(shape=(4,), dflt=-1, pos=7), 'timestamp': Time32Col(shape=(), dflt=0, pos=1), 'traces': Int32Col(shape=(4,), dflt=-1, pos=11), 'trigger_pattern': UInt32Col(shape=(), dflt=0, pos=5)}
+
+ +
+ +
+
+class writer.storage.HisparcSatellite
+
+
+columns = {'event_id': UInt32Col(shape=(), dflt=0, pos=0), 'max_n': UInt16Col(shape=(), dflt=0, pos=4), 'max_signal': UInt16Col(shape=(), dflt=0, pos=7), 'mean_n': Float32Col(shape=(), dflt=0.0, pos=3), 'mean_signal': Float32Col(shape=(), dflt=0.0, pos=6), 'min_n': UInt16Col(shape=(), dflt=0, pos=2), 'min_signal': UInt16Col(shape=(), dflt=0, pos=5), 'timestamp': Time32Col(shape=(), dflt=0, pos=1)}
+
+ +
+ +
+
+class writer.storage.HisparcSingle
+
+
+columns = {'event_id': UInt32Col(shape=(), dflt=0, pos=0), 'mas_ch1_high': Int32Col(shape=(), dflt=-1, pos=3), 'mas_ch1_low': Int32Col(shape=(), dflt=-1, pos=2), 'mas_ch2_high': Int32Col(shape=(), dflt=-1, pos=5), 'mas_ch2_low': Int32Col(shape=(), dflt=-1, pos=4), 'slv_ch1_high': Int32Col(shape=(), dflt=-1, pos=7), 'slv_ch1_low': Int32Col(shape=(), dflt=-1, pos=6), 'slv_ch2_high': Int32Col(shape=(), dflt=-1, pos=9), 'slv_ch2_low': Int32Col(shape=(), dflt=-1, pos=8), 'timestamp': Time32Col(shape=(), dflt=0, pos=1)}
+
+ +
+ +
+
+class writer.storage.LightningConfig
+
+
+columns = {'altitude': Float64Col(shape=(), dflt=0.0, pos=10), 'angle_correction': Float32Col(shape=(), dflt=0.0, pos=16), 'baud_rate': Int16Col(shape=(), dflt=0, pos=3), 'close_alarm_distance': Int32Col(shape=(), dflt=0, pos=12), 'com_port': UInt8Col(shape=(), dflt=0, pos=2), 'daq_mode': BoolCol(shape=(), dflt=False, pos=7), 'database_name': Int32Col(shape=(), dflt=-1, pos=5), 'event_id': UInt32Col(shape=(), dflt=0, pos=0), 'help_url': Int32Col(shape=(), dflt=-1, pos=6), 'latitude': Float64Col(shape=(), dflt=0.0, pos=8), 'longitude': Float64Col(shape=(), dflt=0.0, pos=9), 'minimum_gps_speed': Int32Col(shape=(), dflt=0, pos=15), 'noise_beep': BoolCol(shape=(), dflt=False, pos=14), 'severe_alarm_distance': Int32Col(shape=(), dflt=0, pos=13), 'squelch_seting': Int32Col(shape=(), dflt=0, pos=11), 'station_id': UInt32Col(shape=(), dflt=0, pos=4), 'timestamp': Time32Col(shape=(), dflt=0, pos=1)}
+
+ +
+ +
+
+class writer.storage.LightningError
+
+
+columns = {'event_id': UInt32Col(shape=(), dflt=0, pos=0), 'messages': Int32Col(shape=(), dflt=-1, pos=2), 'timestamp': Time32Col(shape=(), dflt=0, pos=1)}
+
+ +
+ +
+
+class writer.storage.LightningEvent
+
+
+columns = {'corr_angle': Float32Col(shape=(), dflt=0.0, pos=5), 'corr_distance': Int16Col(shape=(), dflt=0, pos=2), 'event_id': UInt32Col(shape=(), dflt=0, pos=0), 'timestamp': Time32Col(shape=(), dflt=0, pos=1), 'uncorr_angle': Float32Col(shape=(), dflt=0.0, pos=4), 'uncorr_distance': Int16Col(shape=(), dflt=0, pos=3)}
+
+ +
+ +
+
+class writer.storage.LightningNoise
+
+
+columns = {'event_id': UInt32Col(shape=(), dflt=0, pos=0), 'timestamp': Time32Col(shape=(), dflt=0, pos=1)}
+
+ +
+ +
+
+class writer.storage.LightningStatus
+
+
+columns = {'close_alarm': BoolCol(shape=(), dflt=False, pos=4), 'close_rate': Int16Col(shape=(), dflt=0, pos=2), 'current_heading': Float32Col(shape=(), dflt=0.0, pos=6), 'event_id': UInt32Col(shape=(), dflt=0, pos=0), 'sever_alarm': BoolCol(shape=(), dflt=False, pos=5), 'timestamp': Time32Col(shape=(), dflt=0, pos=1), 'total_rate': Int16Col(shape=(), dflt=0, pos=3)}
+
+ +
+ +
+
+class writer.storage.WeatherConfig
+
+
+columns = {'altitude': Float64Col(shape=(), dflt=0.0, pos=10), 'barometer': BoolCol(shape=(), dflt=False, pos=15), 'baud_rate': Int16Col(shape=(), dflt=0, pos=3), 'com_port': UInt8Col(shape=(), dflt=0, pos=2), 'daq_mode': BoolCol(shape=(), dflt=False, pos=7), 'database_name': Int32Col(shape=(), dflt=-1, pos=5), 'dew_point': BoolCol(shape=(), dflt=False, pos=23), 'evapotranspiration': BoolCol(shape=(), dflt=False, pos=20), 'event_id': UInt32Col(shape=(), dflt=0, pos=0), 'heat_index': BoolCol(shape=(), dflt=False, pos=22), 'help_url': Int32Col(shape=(), dflt=-1, pos=6), 'humidity_inside': BoolCol(shape=(), dflt=False, pos=13), 'humidity_outside': BoolCol(shape=(), dflt=False, pos=14), 'latitude': Float64Col(shape=(), dflt=0.0, pos=8), 'longitude': Float64Col(shape=(), dflt=0.0, pos=9), 'offset_bar_sea_level': Float32Col(shape=(), dflt=0.0, pos=31), 'offset_inside_humidity': Int16Col(shape=(), dflt=0, pos=27), 'offset_inside_temperature': Float32Col(shape=(), dflt=0.0, pos=25), 'offset_outside_humidity': Int16Col(shape=(), dflt=0, pos=28), 'offset_outside_temperature': Float32Col(shape=(), dflt=0.0, pos=26), 'offset_station_altitude': Float32Col(shape=(), dflt=0.0, pos=30), 'offset_wind_direction': Int16Col(shape=(), dflt=0, pos=29), 'rain_rate': BoolCol(shape=(), dflt=False, pos=21), 'solar_radiation': BoolCol(shape=(), dflt=False, pos=18), 'station_id': UInt32Col(shape=(), dflt=0, pos=4), 'temperature_inside': BoolCol(shape=(), dflt=False, pos=11), 'temperature_outside': BoolCol(shape=(), dflt=False, pos=12), 'timestamp': Time32Col(shape=(), dflt=0, pos=1), 'uv_index': BoolCol(shape=(), dflt=False, pos=19), 'wind_chill': BoolCol(shape=(), dflt=False, pos=24), 'wind_direction': BoolCol(shape=(), dflt=False, pos=16), 'wind_speed': BoolCol(shape=(), dflt=False, pos=17)}
+
+ +
+ +
+
+class writer.storage.WeatherError
+
+
+columns = {'event_id': UInt32Col(shape=(), dflt=0, pos=0), 'messages': Int32Col(shape=(), dflt=-1, pos=2), 'timestamp': Time32Col(shape=(), dflt=0, pos=1)}
+
+ +
+ +
+
+class writer.storage.WeatherEvent
+
+
+columns = {'barometer': Float32Col(shape=(), dflt=0.0, pos=6), 'dew_point': Float32Col(shape=(), dflt=0.0, pos=14), 'evapotranspiration': Float32Col(shape=(), dflt=0.0, pos=11), 'event_id': UInt32Col(shape=(), dflt=0, pos=0), 'heat_index': Int16Col(shape=(), dflt=0, pos=13), 'humidity_inside': Int16Col(shape=(), dflt=0, pos=4), 'humidity_outside': Int16Col(shape=(), dflt=0, pos=5), 'rain_rate': Float32Col(shape=(), dflt=0.0, pos=12), 'solar_rad': Int16Col(shape=(), dflt=0, pos=9), 'temp_inside': Float32Col(shape=(), dflt=0.0, pos=2), 'temp_outside': Float32Col(shape=(), dflt=0.0, pos=3), 'timestamp': Time32Col(shape=(), dflt=0, pos=1), 'uv': Int16Col(shape=(), dflt=0, pos=10), 'wind_chill': Float32Col(shape=(), dflt=0.0, pos=15), 'wind_dir': Int16Col(shape=(), dflt=0, pos=7), 'wind_speed': Int16Col(shape=(), dflt=0, pos=8)}
+
+ +
+ +
+
+writer.storage.get_or_create_cluster_group(file, cluster)[source]
+

Get an existing cluster group or create a new one

+
+
Parameters:
+
    +

  • file – the PyTables data file

    • +

  • cluster – the name of the cluster

    • +
+
+
+
+ +
+
+writer.storage.get_or_create_node(file, cluster, node)[source]
+

Get an existing node or create a new one

+
+
Parameters:
+
    +

  • file – the PyTables data file

    • +

  • cluster – the parent (cluster) node

    • +

  • node – the node (e.g. events, blobs)

    • +
+
+
+
+ +
+
+writer.storage.get_or_create_station_group(file, cluster, station_id)[source]
+

Get an existing station group or create a new one

+
+
Parameters:
+
    +

  • file – the PyTables data file

    • +

  • cluster – the name of the cluster

    • +

  • station_id – the station number

    • +
+
+
+
+ +
+
+writer.storage.open_or_create_file(data_dir, date)[source]
+

Open an existing file or create a new one

+

This function opens an existing PyTables file according to the event +date. If the file does not yet exist, a new one is created.

+
+
Parameters:
+
    +

  • data_dir – the directory containing all data files

    • +

  • date – the event date

    • +
+
+
+
+ +
+
+

Store events module

+
+
+writer.store_events.data_is_blob(uploadcode, blob_types)[source]
+

Determine if data is a variable length binary value (blob)

+
+ +
+
+writer.store_events.store_event(datafile, cluster, station_id, event)[source]
+

Stores an event in the h5 filesystem

+
+
Parameters:
+
    +

  • datafile – the h5 data file

    • +

  • cluster – the name of the cluster to which the station belongs

    • +

  • station_id – the id of the station this event belongs to

    • +

  • event – the event to store

    • +
+
+
+
+ +
+
+writer.store_events.store_event_list(data_dir, station_id, cluster, event_list)[source]
+

Store a list of events

+
+ +
+
+

Upload codes

+
+
+ + +
+
+
+
+ +
+
+ + + + \ No newline at end of file diff --git a/wsgi.html b/wsgi.html new file mode 100644 index 0000000..9407b34 --- /dev/null +++ b/wsgi.html @@ -0,0 +1,249 @@ + + + + + + + + datastore WSGI app package — HiSPARC-datastore 1.0.0 documentation + + + + + + + + + + + + + + +
+
+
+
+ +
+

datastore WSGI app package

+

HiSPARC datastore WSGI application

+

This WSGI app is served by uWSGI on frome and lives at +http://frome.nikef.nl/hisparc/upload.

+

example application.wsgi:

+
import sys
+import functools
+
+sys.path.append('/var/www/wsgi-bin/datastore')
+
+from wsgi import wsgi_app
+
+configfile = '/var/www/wsgi-bin/datastore/examples/config.ini'
+application = functools.partial(wsgi_app.application, configfile=configfile)
+
+
+

configuration is read from a configuation file, shared between the WSGI app +and the writer usually config.ini:

+
[General]
+log=/tmp/hisparc.log
+loglevel=debug
+station_list=/tmp/station_list.csv
+data_dir=/tmp/datastore
+
+[Writer]
+sleep=5
+
+
+
+

wsgi.rcodes module

+

Return codes for the WSGI app.

+

There are four types of return codes:

+
    +

  1. OK (100)

    2. +

  2. putEvent errors (20*)

    +
      +

    • 201: invalid input: checksum of data does not match

      • +

    • 203: wrong password

      • +

    • 206: station unknown (check station-list.csv)

      • +

    • 208: unpickling error

      • +
    +
    3. +

  3. getEvent errors (30*)

    +

    not used by WSGI app

    +
    4. +

  4. Internal server error (40*)

    +
      +

    • 400: internal server error. Invalid post data.

      • +
    +
    5. +
+
+
+

wsgi.wsgi_app module

+
+
+wsgi.wsgi_app.application(environ, start_response, configfile)[source]
+

The hisparc upload application

+

This handler is called by uWSGI whenever someone requests our URL.

+

First, we generate a dictionary of POSTed variables and try to read out the +station_id, password, checksum and data. When we do a readline(), we +already read out the entire datastream. I don’t know if we can first check +on station_id/password combinations before reading out the datastream +without setting up a bidirectional communication channel.

+

When the checksum matches, we unpickle the event_list and pass everything +on to store_event_list.

+
+ +
+
+wsgi.wsgi_app.decode_object(o)[source]
+

recursively decode all bytestrings in object

+
+ +
+
+wsgi.wsgi_app.do_init(configfile)[source]
+

Load configuration and passwords and set up a logger handler

+

This function will do one-time initialization. By using global +variables, we eliminate the need to reread configuration and passwords +on every request.

+

Configuration is read from the datastore configuation file (usually +config.ini):

+
[General]
+log=/tmp/hisparc.log
+loglevel=debug
+station_list=/tmp/station_list.csv
+data_dir=/tmp/datastore
+
+[Writer]
+sleep=5
+
+
+

Station information is read from the station_list config variable. +(station_list.csv on frome)

+
+ +
+
+wsgi.wsgi_app.is_data_suspicious(event_list)[source]
+

Check data for suspiciousness

+

Suspiciousness, a previously unknown quantum number that may signify +the actual birth of the universe and the reweaving of past fates into +current events has come to hunt us and our beloved data.

+

Note: Apr 7, 2019 0:00 is the default time after a cold start without GPS signal. +The DAQ will happily send events even when no GPS signal has been +acquired (yet). Events with timestamp Apr 7, 2019 are most probably caused +by no or bad GPS signal. Such events must be eigenstates of suspiciousness.

+
+ +
+
+wsgi.wsgi_app.store_data(station_id, cluster, event_list)[source]
+

Store verified event data to temporary storage

+
+ +
+
+ + +
+
+
+
+ +
+
+ + + + \ No newline at end of file diff --git a/xml-rpc-server.html b/xml-rpc-server.html new file mode 100644 index 0000000..018d3e5 --- /dev/null +++ b/xml-rpc-server.html @@ -0,0 +1,125 @@ + + + + + + + + XML-RPC-Server — HiSPARC-datastore 1.0.0 documentation + + + + + + + + + + + + + +
+
+
+
+ +
+

XML-RPC-Server

+

Simple XML-RPC Server to run on the datastore server.

+

This daemon should be run on HiSPARC’s datastore server. It will +handle the cluster layouts and station passwords. When an update is +necessary, it will reload the HTTP daemon.

+

The basis for this code was ripped from the python SimpleXMLRPCServer +library documentation and extended.

+
+
+examples.datastore_config_server.reload_datastore()[source]
+

Load datastore config and reload datastore, if necessary

+
+ +
+ + +
+
+
+
+ +
+
+ + + + \ No newline at end of file