Variant Import

backend/promis/backend_api/models.py

@@ -3,6 +3,7 @@
 from django.db import models
 from django.db.models.fields import ( DateTimeField, IntegerField, CharField,
     DateField, FloatField, TextField )
+from django.contrib.postgres.fields import ArrayField
 from django.db.models.fields.related import ForeignKey
 from jsonfield import JSONField
 from django.contrib.gis.db.models import LineStringField
@@ -92,6 +93,9 @@ class Session(models.Model):
     # but ensures distances and intersections are caclulated correctly
     # TODO: srid should eventually be 4979 see #222
     geo_line = LineStringField(dim = 3, srid = 4326, geography=True)
+
+    # TODO: 4D coordinates, see #256
+    altitude = ArrayField(FloatField())
     space_project = ForeignKey('Space_project', null = True)
 
     class Meta:

backend/promis/backend_api/serializer.py

@@ -83,7 +83,9 @@ def get_geo_line(self, obj):
         #return obj.geo_line.wkb.hex()
 
         # TODO: study whether pre-building the list or JSON would speed up things
-        return parsers.wkb(obj.geo_line.wkb) # <- Generator
+        # TODO: ugly hack before #256
+        # geo_line.wkb calls a generator implicitly
+        return ( (*geo[:2], alt) for alt, geo in zip(obj.altitude, parsers.wkb(obj.geo_line.wkb)) )   
 
     def get_timelapse(self, obj):
         # TODO: change to time_start in model for consistency

backend/promis/classes/base_data.py

@@ -41,6 +41,15 @@ def data_start(self):
     def frequency(self):
         return self.measurement.sampling_frequency
 
+    def __len__(self):
+        return len(self.doc)
+
+    def __getitem__(self, idx):
+        return self.doc[idx]
+
+    def data(self, selection = slice(None)):
+        return self.doc[selection]
+
     def timeslice(self, start, end):
         '''
         Returns a slice for accessing data between start and end inclusively.
@@ -77,14 +86,6 @@ class SingleVarTimeSeries(BaseData):
 
     # Underlying document is a simple list
     # delegate sequence protocol there
-    def __len__(self):
-        return len(self.doc)
-
-    def __getitem__(self, idx):
-        return self.doc[idx]
-
-    def data(self, selection = slice(None)):
-        return self.doc[selection]
 
     # TODO: propagate upwards?
     def quicklook(self, points, selection = slice(None)):
@@ -134,26 +135,58 @@ def avg_float(l, n, span):
             yield avg_float(v, int(span * i), span)
 
 
-# TODO: realize
 class ObliqueThreeVarTimeSeriesHF(BaseData):
     """
     [en]: Three high frequency components in oblique coordinate system
     [uk]: Три високочастотні компоненти вектора величини в косоугольній системі координат
     """
+    def quicklook(self, points, selection = slice(None)):
+        return
 
-
-# TODO: realize
 class OrthogonalThreeVarTimeSeriesHF(BaseData):
     """
     [en]: Three high frequency components in orthogonal coordinate system
     [uk]: Три високочастотні компоненти вектора величини в ортогональній системі координат
     """
+    def quicklook(self, points, selection = slice(None)):
+        return
 
+class ObliqueThreeVarTimeSeriesLF(BaseData):
+    """
+    [en]: Three low frequency components in oblique coordinate system
+    [uk]: Три низькочастотні компоненти вектора величини в косоугольній системі координат
+    """
+    def quicklook(self, points, selection = slice(None)):
+        return
+
+class OrthogonalThreeVarTimeSeriesLF(BaseData):
+    """
+    [en]: Three low frequency components in orthogonal coordinate system
+    [uk]: Три низькочастотні компоненти вектора величини в ортогональній системі координат
+    """
+    def quicklook(self, points, selection = slice(None)):
+        return
 
-# TODO: realize
 class OrthogonalTwoVarTimeSeriesHF(BaseData):
     """
     [en]: Two high frequency components in orthogonal coordinate system
     [uk]: Дві високочастотні компоненти вектора величини в ортогональній системі координат
     """
-    pass
+    def quicklook(self, points, selection = slice(None)):
+        return
+
+class ObliqueTwoVarTimeSeriesHF(BaseData):
+    """
+    [en]: Two high frequency components in oblique coordinate system
+    [uk]: Дві високочастотні компоненти вектора величини в косоугольній системі координат
+    """
+    def quicklook(self, points, selection = slice(None)):
+        return
+
+class ScalarValueDifference(BaseData):
+    """
+    [en]: Scalar difference of two values
+    [uk]: Скалярна різниця двух змінних
+    """
+    def quicklook(self, points, selection = slice(None)):
+        return

backend/promis/classes/potential.py

@@ -132,7 +132,10 @@ def fetch(self, daydir):
                                 ez_time_end = time_end
 
                                 # Generator for the orbit
-                                line_gen = ( (y.lon, y.lat, t) for t, y, _ in orbit.generate_orbit(orbit_path, time_start, time_end) )
+                                # TODO: rewrite as generator if possible at all
+                                path = [ (y.lon, y.lat, y.alt, t) for t, y, _ in orbit.generate_orbit(orbit_path, time_start, time_end) ]
+                                line_gen = [ (x, y, t) for x, y, _, t in path ]
+                                alt_gen = [ alt for _, _, alt, _ in path ]
                                 # Converting time to python objects for convenience
                                 # This is the point where onboard time gets converted to the UTC
                                 time_start = unix_time.maketime(time_start)
@@ -143,7 +146,8 @@ def fetch(self, daydir):
                                 # Creating a session object
                                 # TODO: make it more readable
                                 # TODO: srid should be 4979 see #222
-                                ez_sess_obj = model.Session.objects.create(time_begin = time_start, time_end = time_end, geo_line = LineString(*line_gen, srid = 4326), space_project = self.project_obj )
+                                ez_sess_obj = model.Session.objects.create(time_begin = time_start, time_end = time_end, altitude = alt_gen,
+                                        geo_line = LineString(*line_gen, srid = 4326), space_project = self.project_obj )
 
                                 # TODO: record data_id in the object
                                 # TODO: somehow generalise this process maybe

backend/promis/classes/variant.py

@@ -19,12 +19,51 @@
 # permissions and limitations under the Licence.
 #
 
-
+from django.contrib.gis.geos import LineString
 from classes.base_project import BaseProject
 
+import datetime
+import ftp_helper, parsers, unix_time, orbit
+import backend_api.models as model
 
-# TODO: implement obtaining of Variant data
-
+sessions_time = {
+    "597": {
+      'time_start': datetime.datetime(2005, 2, 1, 8, 22, 59).timestamp(),
+      'time_end': datetime.datetime(2005, 2, 1, 8, 43, 29).timestamp()
+    },
+    "1056": {
+      'time_start': datetime.datetime(2005, 3, 2, 17, 23, 10).timestamp(),
+      'time_end': datetime.datetime(2005, 3, 2, 17, 27, 37).timestamp()
+    },
+    "1057": {
+      'time_start': datetime.datetime(2005, 3, 3, 5, 42, 57).timestamp(),
+      'time_end': datetime.datetime(2005, 3, 3, 5, 44, 37).timestamp()
+    },
+    "1071": {
+      'time_start': datetime.datetime(2005, 3, 3, 16, 48, 14).timestamp(),
+      'time_end': datetime.datetime(2005, 3, 3, 16, 52, 41).timestamp()
+    },
+    "1109": {
+      'time_start': datetime.datetime(2005, 3, 6, 11, 3, 13).timestamp(),
+      'time_end': datetime.datetime(2005, 3, 6, 11, 7, 40).timestamp()
+    },
+    "1139": {
+      'time_start': datetime.datetime(2005, 3, 8, 1, 53, 5).timestamp(),
+      'time_end': datetime.datetime(2005, 3, 8, 1, 57, 32).timestamp()
+    },
+    "1278": {
+      'time_start': datetime.datetime(2005, 3, 17, 13, 52, 49).timestamp(),
+      'time_end': datetime.datetime(2005, 3, 17, 13, 57, 16).timestamp()
+    },
+    "1363": {
+      'time_start': datetime.datetime(2005, 3, 23, 1, 13, 6).timestamp(),
+      'time_end': datetime.datetime(2005, 3, 23, 1, 14, 2).timestamp()
+    },
+    "1394": {
+      'time_start': datetime.datetime(2005, 3, 25, 1, 32, 47).timestamp(),
+      'time_end': datetime.datetime(2005, 3, 25, 1, 33, 43).timestamp()
+    }
+}
 
 class Variant(BaseProject):
     '''
@@ -33,4 +72,141 @@ class Variant(BaseProject):
     '''
 
     def check(*args, **kwargs):
-        return []
+        exceptions = {"1363",     # too few anchor points unfortunately
+                      "Read_me.doc",
+                      "Tabl_2.doc",
+                      "Tabl_3.doc"}
+        with ftp_helper.FTPChecker("Variant/Data_Release1/", "ftp.promis.ikd.kiev.ua") as ftp:
+            ftp.exceptions = exceptions
+            # TODO: check that directory exists properly
+            # TODO: any more elegant way? re-yield or smth
+            for v in ftp.check():
+                yield v
+
+    def fetch(self, daydir):
+        with ftp_helper.FTPChecker("Variant/", "ftp.promis.ikd.kiev.ua") as ftp: 
+            ftp.cwd("telemetry")
+            # TODO: timezone hack
+            Δ = 1111714367 - 1111703568
+
+            with ftp.xopen("var{0}.tm".format(daydir)) as fp:
+                orbit_path = { (t + Δ): pt for t, pt in parsers.telemetry(fp, cartesian=False) }
+
+            time_start = int(sessions_time[daydir]['time_start'])
+            time_end = int(sessions_time[daydir]['time_end'])
+
+            path = [ (y.lon, y.lat, y.alt, t) for t, y, _ in orbit.generate_orbit(orbit_path, time_start, time_end, time_delta = 20) ]
+            line_gen = [ (x, y, t) for x, y, _, t in path ]
+            alt_gen = [ alt for _, _, alt, _ in path ]
+
+            time_start = unix_time.maketime(time_start)
+            time_end = unix_time.maketime(time_end)
+            time_dur = time_end - time_start
+            print("\tSession: [ %s, %s ] (%s)." % (time_start.isoformat(), time_end.isoformat(), str(time_dur)) )
+
+            sess_obj = model.Session.objects.create(time_begin = time_start, time_end = time_end, altitude = alt_gen,
+                geo_line = LineString(*line_gen, srid = 4326), space_project = self.project_obj )
+
+            # read per-channel sampling frequencies
+            component_sf = [['Bx', 0], ['By',0], ['Bz',0], ['Ts',0], ['Te',0], 
+                           ['JF~',0], 
+                           ['FC1~',0], ['FC1=',0], ['FC2~',0], ['FC2=',0], 
+                           ['Jyz~',0], ['Bx~',0], ['Jxz~',0], ['By~',0], 
+                           ['E1~',0], ['E1=',0], ['E2~',0], ['E2=',0], ['E3~',0], ['E3=',0]]
+            ftp.cwd("..")
+            ftp.cwd("Data_Release1/{0}".format(daydir))
+
+            with ftp.xopen("{0}.txt".format(daydir)) as fp:
+                for line in fp:
+                    if line.rstrip() == "Sampling frequency, Hz":
+                        break
+                idx = 0
+                for line in fp:
+                    if idx >= len(component_sf):
+                        break
+                    if daydir in ["1057", "1071", "1109", "1278", "1394"] and idx == 3:
+                        idx += 2
+                    if len(line.rstrip()) != 0:
+                        component_sf[idx][1] = float(line)
+                    idx += 1
+
+            # Per-channel dicts of filename and JSON name
+            data = {
+                'ΔE1, ΔE2, ΔE3': {
+                    'param': 'Ex, Ey, Ez (three components of electric field HF)',
+                    'comps' : ['E1~', 'E2~', 'E3~']
+                },
+                'Bx~, By~ (not calibrated)': {
+                    'param': 'Bx, By (two components of magnetic field HF)',
+                    'comps' : ['Bx~', 'By~']
+                },
+                'Jxz~, Jyz~ (not calibrated)': {
+                    'param': 'Jxz, Jyz (two components of current density)',
+                    'comps' : ['Jxz~', 'Jyz~'] 
+                },
+                'ΔE1=, ΔE2=, ΔE3=': {
+                    'param': 'E1=, E2=, E3= (three components of electric field LF)',
+                    'comps': ['E1=', 'E2=', 'E3=']
+                },
+                'FC1~, FC2~': {
+                    'param': 'ΔFC~',
+                    'comps': ['FC1~', 'FC2~']
+                },
+                'JF~': {
+                    'param': 'JF~ (current density)',
+                    'comps': ['JF~']
+                },
+                'Bx, By, Bz': {
+                    'param': 'Bx, By, Bz (three components of magnetic field)',
+                    'comps': ['Bx', 'By', 'Bz']
+                },
+                'FC1=, FC2=': {
+                    'param': 'ΔFC=',
+                    'comps': ['FC1=', 'FC2=']
+                }
+            }
+
+            def get_file(name):
+                try:
+                    with ftp.xopen(name) as fp:
+                        return [ float(ln) for ln in fp ]
+                except:
+                    return []
+
+            for chan_name, chan_files in data.items():
+                chan_obj = model.Channel.objects.language('en').filter(name = chan_name)[0]
+                par_obj = model.Parameter.objects.language('en').filter(name = chan_files['param'])[0]
+                list_of_components = [get_file(name) for name in chan_files['comps']]
+
+                if (len(list_of_components) == 0) or (sum(len(x) for x in list_of_components) == 0):
+                    continue
+
+                # All the components of the single channel are suuposed to have the same sampling frequecny
+                # In fact sometimes some fo them appeared to be 0, that is why max is needed here
+                def find_sf(components):
+                    max_sf = 0
+                    for comp_name in components:
+                        for name, sf in component_sf:
+                            if name == comp_name:
+                                max_sf = max(max_sf, sf)
+                                break
+                    #print("components: " + str(components) + " max_sf: " + str(max_sf))        
+                    return max_sf
+
+                sampling_frequency = find_sf(chan_files['comps'])
+                numValues = int(time_dur.total_seconds() * sampling_frequency)
+
+                def catch_idx_error(component, idx):
+                   try:
+                      return component[idx]
+                   except IndexError:
+                      return 0.0
+
+                # We should better not covert a single component into a tuple. Handling this here
+                if len(list_of_components) > 1:
+                    json_data = [tuple([catch_idx_error(component,idx) for component in list_of_components]) for idx in range(numValues)]
+                else:
+                    json_data = [catch_idx_error(list_of_components[0],idx) for idx in range(numValues)]
+
+                doc_obj = model.Document.objects.create(json_data = json_data )
+                meas = model.Measurement.objects.create(session = sess_obj, parameter = par_obj, channel = chan_obj, channel_doc = doc_obj, parameter_doc = doc_obj, sampling_frequency = sampling_frequency, max_frequency = sampling_frequency, min_frequency = sampling_frequency)

backend/promis/export.py

@@ -23,6 +23,7 @@
 import collections
 import math
 import unix_time
+import itertools
 
 # TODO: currently only one data row
 # TODO: do we need this type or can we just have a tuple?
@@ -100,7 +101,7 @@ def ascii_export(table, datalabel="Data", dataunits="units"):
     yield "{:^10} {:^10} {:^6} {:^6} {:^6} {:^15}".format("Date", "UT", "Lat.", "Lon.", "Alt.", datalabel)
     yield "{:^10} {:^10} {:^6} {:^6} {:^6} {:^15}".format("(YYYYDDD)", "(ms)", "(deg.)", "(deg.)", "(km)", "(%s)" % dataunits)
     for row in table:
-        yield "{:>10} {:>10} {:>6.02f} {:>6.02f} {:>6.02f} {:>15.06f}".format(row.date, row.ut, row.lat, row.lon, row.alt, row.data)
+        yield "{:>10} {:>10} {:>6.02f} {:>6.02f} {:>6.02f}".format(row.date, row.ut, row.lat, row.lon, row.alt) + "".join(" {:>15.06f}".format(data_comp) for data_comp in row.data)
 
 def csv_export(table, datalabel="Data", dataunits="units"):
     """
@@ -115,7 +116,7 @@ def csv_export(table, datalabel="Data", dataunits="units"):
     """
     yield '"{}","{}","{}","{}","{}","{}"'.format("Date (YYYYDDD)", "UT (ms)", "Longitude (deg)", "Latitude (deg)", "Altitude (km)", datalabel + "(%s)" % dataunits)
     for row in table:
-        yield ",".join(str(x) for x in [row.date, row.ut, row.lon, row.lat, row.alt, row.data])
+        yield ",".join(str(x) for x in itertools.chain([row.date, row.ut, row.lon, row.lat, row.alt], row.data))
 
 
 

backend/promis/orbit.py

@@ -44,12 +44,12 @@ def cord_conv(RX, RY, RZ, **kwargs):
     return OrbitPoint(math.degrees(φ), math.degrees(λ), ρ - _earth_radius)
 
 # Generating an orbit point every 1 second, discarding extra point and filling the gaps
-def generate_orbit(datapoints, orbit_start, orbit_end):
+def generate_orbit(datapoints, orbit_start, orbit_end, time_delta = 0):
     datakeys = datapoints.keys()
     time_start, time_end = min(datakeys), max(datakeys)
     orbit_len = orbit_end - orbit_start
 
-    assert(orbit_end <= time_end and orbit_start >= time_start)
+    assert((orbit_end - time_delta) <= time_end and (orbit_start + time_delta) >= time_start)
 
     # Anchor points from which the curve is deduced
     # anchor[t] is a list of 4 known timepoints: 2 before t + 2 after t if possible
@@ -73,7 +73,7 @@ def generate_orbit(datapoints, orbit_start, orbit_end):
         if i in datakeys:
             anchor[0].append(i)
         i += 1
-
+ 
     # Pass 2: Copying the anchor over as long as no new points are encountered
     # If we do encounter a new point, shift the list to the left and add it
     last_anchor = 0