Variant Import

backend/promis/backend_api/fixtures/init_data.json

@@ -58,7 +58,7 @@
             "classes.variant.Variant"
         ],
         "date_start": "2005-02-01",
-        "date_end": "2005-03-25"
+        "date_end": "2005-03-26"
     }
 },
 {

backend/promis/backend_api/models.py

@@ -3,6 +3,7 @@
 from django.db import models
 from django.db.models.fields import ( DateTimeField, IntegerField, CharField,
     DateField, FloatField, TextField )
+from django.contrib.postgres.fields import ArrayField
 from django.db.models.fields.related import ForeignKey
 from jsonfield import JSONField
 from django.contrib.gis.db.models import LineStringField
@@ -92,6 +93,9 @@ class Session(models.Model):
     # but ensures distances and intersections are caclulated correctly
     # TODO: srid should eventually be 4979 see #222
     geo_line = LineStringField(dim = 3, srid = 4326, geography=True)
+
+    # TODO: 4D coordinates, see #256
+    altitude = ArrayField(FloatField())
     space_project = ForeignKey('Space_project', null = True)
 
     class Meta:

backend/promis/backend_api/serializer.py

@@ -83,7 +83,9 @@ def get_geo_line(self, obj):
         #return obj.geo_line.wkb.hex()
 
         # TODO: study whether pre-building the list or JSON would speed up things
-        return parsers.wkb(obj.geo_line.wkb) # <- Generator
+        # TODO: ugly hack before #256
+        # geo_line.wkb calls a generator implicitly
+        return ( (*geo[:2], alt) for alt, geo in zip(obj.altitude, parsers.wkb(obj.geo_line.wkb)) )   
 
     def get_timelapse(self, obj):
         # TODO: change to time_start in model for consistency

backend/promis/classes/potential.py

@@ -132,7 +132,10 @@ def fetch(self, daydir):
                                 ez_time_end = time_end
 
                                 # Generator for the orbit
-                                line_gen = ( (y.lon, y.lat, t) for t, y, _ in orbit.generate_orbit(orbit_path, time_start, time_end) )
+                                # TODO: rewrite as generator if possible at all
+                                path = [ (y.lon, y.lat, y.alt, t) for t, y, _ in orbit.generate_orbit(orbit_path, time_start, time_end) ]
+                                line_gen = [ (x, y, t) for x, y, _, t in path ]
+                                alt_gen = [ alt for _, _, alt, _ in path ]
                                 # Converting time to python objects for convenience
                                 # This is the point where onboard time gets converted to the UTC
                                 time_start = unix_time.maketime(time_start)
@@ -143,7 +146,8 @@ def fetch(self, daydir):
                                 # Creating a session object
                                 # TODO: make it more readable
                                 # TODO: srid should be 4979 see #222
-                                ez_sess_obj = model.Session.objects.create(time_begin = time_start, time_end = time_end, geo_line = LineString(*line_gen, srid = 4326), space_project = self.project_obj )
+                                ez_sess_obj = model.Session.objects.create(time_begin = time_start, time_end = time_end, altitude = alt_gen,
+                                        geo_line = LineString(*line_gen, srid = 4326), space_project = self.project_obj )
 
                                 # TODO: record data_id in the object
                                 # TODO: somehow generalise this process maybe

backend/promis/classes/variant.py

@@ -19,11 +19,11 @@
 # permissions and limitations under the Licence.
 #
 
-
+from django.contrib.gis.geos import LineString
 from classes.base_project import BaseProject
 
-
-# TODO: implement obtaining of Variant data
+import ftp_helper, parsers, unix_time, orbit
+import backend_api.models as model
 
 
 class Variant(BaseProject):
@@ -33,4 +33,78 @@ class Variant(BaseProject):
     '''
 
     def check(*args, **kwargs):
-        return []
+        return [ "1394" ] # TODO: properly check the FTP
+
+    def fetch(self, daydir):
+        with ftp_helper.FTPChecker("Variant/", "ftp.promis.ikd.kiev.ua") as ftp: 
+            ftp.cwd("telemetry")
+            # TODO: timezone hack
+            Δ = 1111714367 - 1111703568
+
+            # TODO: lots of pre-conference half-measures here
+            with ftp.xopen("var{0}.tm".format(daydir)) as fp:
+                orbit_path = { (t + Δ): pt for t, pt in parsers.telemetry(fp, cartesian=False) }
+
+            # TODO: hardcode
+            time_start = 1111714367 # 25-03-2005 01:32:47
+            time_end = time_start + 56
+
+            # TODO: assuming there was only one session
+            path = [ (y.lon, y.lat, y.alt, t) for t, y, _ in orbit.generate_orbit(orbit_path, time_start, time_end) ]
+            line_gen = [ (x, y, t) for x, y, _, t in path ]
+            alt_gen = [ alt for _, _, alt, _ in path ]
+
+            time_start = unix_time.maketime(time_start)
+            time_end = unix_time.maketime(time_end)
+            time_dur = time_end - time_start
+            print("\tSession: [ %s, %s ] (%s)." % (time_start.isoformat(), time_end.isoformat(), str(time_dur)) )
+
+            sess_obj = model.Session.objects.create(time_begin = time_start, time_end = time_end, altitude = alt_gen,
+                geo_line = LineString(*line_gen, srid = 4326), space_project = self.project_obj )
+
+            ftp.cwd("..")
+            ftp.cwd("Data_Release1/{0}".format(daydir))
+            # Per-channel dicts of filename and JSON name
+            data = {
+                'ΔE1, ΔE2, ΔE3': {
+                    'param': 'Ex, Ey, Ez (three components of electric field HF Fs = 31.25 kHz)',
+                    'file': 'E1~'
+                    #'comps' : {
+                    #    'E1~': 'e1',
+                    #    'E2~': 'e2',
+                    #    'E3~': 'e3'
+                    #}
+                },
+                'Bx~, By~ (not calibrated)': {
+                    'param': 'Bx, By (two components of magnetic field HF Fs = 31,25 kHz)',
+                    'file': 'Bx~'
+                    #'comps': {
+                    #    'Bx~': 'bx',
+                    #    'By~': 'by'
+                    #}
+                },
+                'Jxz~, Jyz~ (not calibrated)': {
+                    'param': 'Jxz, Jyz (two components of current density Fs = 31.25 kHz)',
+                    'file': 'Jxz~'
+                    #'comps': {
+                    #    'Jxz~': 'jxz',
+                    #    'Jyz~': 'jyz'
+                    #}
+                }
+            }
+
+            def get_file(name):
+                with ftp.xopen(name) as fp:
+                    return [ float(ln) for ln in fp ]
+
+            for chan_name, chan_files in data.items():
+                chan_obj = model.Channel.objects.language('en').filter(name = chan_name)[0]
+                par_obj = model.Parameter.objects.language('en').filter(name = chan_files['param'])[0]
+                # json_data = { key: get_file(name) for name, key in chan_files['comps'].items() }
+                json_data = get_file(chan_files['file'])
+                doc_obj = model.Document.objects.create(json_data = json_data )
+                meas = model.Measurement.objects.create(session = sess_obj, parameter = par_obj, channel = chan_obj, channel_doc = doc_obj, parameter_doc = doc_obj, sampling_frequency = 31250, max_frequency = 31250, min_frequency = 31250)
+
+
+
+

backend/promis/parsers.py

@@ -28,9 +28,12 @@
 # TODO: replace ValueErrors with meaningful exception classes when integrating
 # or make something like DataImportError(), whatever
 
-def telemetry(fp):
+def telemetry(fp, cartesian=True):
     """
     Parses the telemetry .txt file used in Potential and possibly some other satellites.
+    If cartesian is true, then the parser looks for RX, RY, RZ sections and converts to
+    geographic coordinates.
+    Otherwise the parser expects V_ALT, V_LAT and V_LONG as ready-to use data. 
 
     Yields time, (time, lon, lat, alt) tuples.
     """
@@ -40,7 +43,8 @@ def telemetry(fp):
     # I'm going with route #2 for now, no particular reason, just think it will be more elegant in code
 
     # Pointers to the current reading positions in the file per section
-    sections_index = { "RX": -1 , "RY": -1, "RZ": -1 }
+    # Yes, Variant's ones have a space before the name. Yush!
+    sections_index = { "RX": -1 , "RY": -1, "RZ": -1 } if cartesian else { " V_LAT": -1, " V_LONG": -1, " V_ALT": -1 }
 
     # Regular expression to match the sections above
     sections_rx = "^@(%s)" % "|".join(sections_index)
@@ -67,19 +71,24 @@ def scan_sect(sect):
             fp.seek(sections_index[sect])
 
             # Read a line, exit the iteration if it's the end of input
+            # Variant doesn't have proper spacing, so stop at new section as well
             ln = fp.readline()
-            if ln.rstrip() == "":
+            if ln.rstrip() == "" or ln[0] == '@':
                 break
 
             # Record the new position
             sections_index[sect] = fp.tell()
 
             # Try to parse the data
             # NOTE: expected format: <timestamp> <float value(.)> <human readable date>, ignoring the last one
-            m = re.search("^[0-9]* ([0-9.-]*) (2.*)", ln)
+            m = re.search("^([0-9]+) *([0-9.-]+) *(2.*)?", ln)
             if m:
+                # Potential had human readable time at the end and onboard time in first column
+                # Variant seems(!) to have UNIX time in first column and no third one
+                time_mark = unix_time.str_to_utc(m.group(3)) if m.group(3) else int(m.group(1))
+
                 # Yielding a nested tuple e.g. ( "RX", (1, 432.0) ), will be converted to dict
-                yield ( sect, (unix_time.str_to_utc(m.group(2)), float(m.group(1))) )
+                yield ( sect, (time_mark, float(m.group(2))) )
             else:
                 raise ValueError("Input inconsistency detected")
 
@@ -100,7 +109,13 @@ def scan_point():
                 for k,v in nextpoint.items():
                     nextpoint[k] = v[1]
 
-                yield timemark, orbit.cord_conv(**nextpoint)
+                # Convert if the input is cartesian, otherwise just construct a class 
+                if cartesian:
+                    point_value = orbit.cord_conv(**nextpoint)
+                else:
+                    point_value = orbit.OrbitPoint(*nextpoint.values())
+
+                yield timemark,  point_value
         except StopIteration:
             pass
 

frontend/app/components/CesiumMap.js

@@ -486,7 +486,7 @@ export default class CesiumContainer extends Component {
 
         /* data is [lat, lon, hgt] */
         data.forEach(function(point) { // TODO temporary disabling altitude because it stores time temporarily
-            cartesians.push(Cartesian3.fromDegrees(point[1], point[0], /*point[2] ? point[2] :*/ 250000));
+            cartesians.push(Cartesian3.fromDegrees(point[1], point[0], point[2] ? point[2] *1000 : 250000));
         });
 
         return this.viewer.entities.add({