Initial code for handling GoPro metadata

GPMF_gyro.py

@@ -0,0 +1,39 @@
+# Script to extract gopro metadata into a useful format.
+# Uses python-gpmf by  from https://github.com/rambo/python-gpmf
+
+import gpmf.parse as gpmf_parse
+from gpmf.extract import get_gpmf_payloads_from_file
+import sys
+
+
+class Extractor:
+    def __init__(self, videopath = "hero5.mp4"):
+        self.videopath = videopath
+
+        payloads, parser = get_gpmf_payloads_from_file(videopath)
+
+        for gpmf_data, timestamps in payloads:
+            for element, parents in gpmf_parse.recursive(gpmf_data):
+                try:
+                    value = gpmf_parse.parse_value(element)
+                except ValueError:
+                    value = element.data
+                print("{} {} > {}: {}".format(
+                    timestamps,
+                    ' > '.join([x.decode('ascii') for x in parents]),
+                    element.key.decode('ascii'),
+                    value
+                ))
+
+    def get_gyro(self):
+        return 1
+
+    def get_accl(self):
+        return 1
+
+    def get_video_length(self):
+        return 1
+
+
+if __name__ == "__main__":
+    testing = Extractor()

README.md

@@ -4,4 +4,21 @@ A program built around Python, OpenCV, and PySide2 for video stabilization using
 
 The project consists of three core parts: A utility for the generation of lens undistortion preset, a utility for stabilizing footage using gyro data, and a utility for stretching 4:3 video to 16:9 using non-linear horizontal stretching (similar to GoPro's superview). Only the last part (sorta) works as of right now.
 
-This is very much a work in progress project, but the goal is to use the gyro data logged on drone flight controllers for stabilizing the onboard HD camera. Furthermore, the gyro data embedded in newer GoPro cameras should also be usable for stabilization purposes.
+This is very much a work in progress project, but the goal is to use the gyro data logged on drone flight controllers for stabilizing the onboard HD camera. Furthermore, the gyro data embedded in newer GoPro cameras should also be usable for stabilization purposes.
+
+### Status
+
+Working:
+* Videoplayer based on OpenCV and Pyside2
+* Gyro integration using quaternions
+* Non-linear stretch utility
+* Basic video import/export
+* Camera calibration utility with preset import/export
+
+
+Not working (yet):
+* GoPro/blackbox data import
+* Symmetrical quaternion low-pass filter
+* Camera rotation perspective transform
+* Automatic gyro/video sync
+* Stabilization UI

camera_presets/gopro_calib.JSON

@@ -0,0 +1,40 @@
+{
+    "name": "Camera name",
+    "note": "",
+    "calibrator_version": "0.1.0 pre-alpha",
+    "date": "2020-08-04",
+    "calib_dimension": {
+        "w": 1920,
+        "h": 1440
+    },
+    "num_images": 5,
+    "use_opencv_fisheye": true,
+    "fisheye_params": {
+        "RMS_error": 1.312648728814275,
+        "camera_matrix": [
+            [
+                853.5791051672114,
+                0.0,
+                973.112329623264
+            ],
+            [
+                0.0,
+                859.3536064161069,
+                720.8037689142063
+            ],
+            [
+                0.0,
+                0.0,
+                1.0
+            ]
+        ],
+        "distortion_coeffs": [
+            0.012920277676868729,
+            0.11660847198484613,
+            -0.11973972602821542,
+            0.04344499986692209
+        ]
+    },
+    "use_opencv_standard": false,
+    "calib_params": {}
+}

gpmf/extract.py

@@ -0,0 +1,140 @@
+# The MIT License (MIT)
+# Copyright (c) 2014 Eero af Heurlin
+# https://github.com/rambo/python-gpmf
+
+#!/usr/bin/env python3
+import hachoir.parser
+from hachoir.field import MissingField
+from hachoir.field.string_field import String
+
+
+def get_raw_content(met):
+    """Reads the raw bytes from the stream for this atom/field"""
+    if hasattr(met, 'stream'):
+        stream = met.stream
+    else:
+        stream = met.parent.stream
+    return stream.read(met.absolute_address, met.size)
+
+
+def get_gpmf_payloads_from_file(filepath):
+    """Get payloads from file, returns a tuple with the payloads iterator and the parser instance"""
+    parser = hachoir.parser.createParser(filepath)
+    return (get_payloads(find_gpmd_stbl_atom(parser)), parser)
+
+
+def get_gpmf_payloads(parser):
+    """Shorthand for finding the GPMF atom to be passed to get_payloads"""
+    return get_payloads(find_gpmd_stbl_atom(parser))
+
+
+def get_payloads(stbl):
+    """Get payloads by chunk from stbl, with timing info"""
+    # Locate needed subatoms
+    for subatom in stbl:
+        tag = subatom['tag']
+        if tag.value == 'stsz':
+            stsz = subatom['stsz']
+        if tag.value == 'stco':
+            stco = subatom['stco']
+        if tag.value == 'stts':
+            stts = subatom['stts']
+
+    # Generate start and end timestamps for all chunks
+    timestamps = []
+    for idx in range(stts['count'].value):
+        sample_delta = stts["sample_delta[{}]".format(idx)].value
+        for idx2 in range(stts["sample_count[{}]".format(idx)].value):
+            if idx == 0 and idx2 == 0:
+                sampletimes = (0, sample_delta)
+            else:
+                sampletimes = (timestamps[-1][1], timestamps[-1][1] + sample_delta)
+            timestamps.append(sampletimes)
+
+    # Read chunks, yield with timing data
+    num_samples = stsz['count'].value
+    for idx in range(num_samples):
+        offset = stco["chunk_offset[{}]".format(idx)].value
+        size = stsz["sample_size[{}]".format(idx)].value
+        data = stbl.stream.read(offset * 8, size * 8)[1]
+        yield (data, timestamps[idx])
+
+
+def get_stream_data(stbl):
+    """Get raw payload bytes from stbl atom offsets"""
+    ret_bytes = b''
+    for payload in get_payloads(stbl):
+        ret_bytes += payload[0]
+    return ret_bytes
+
+
+def find_gpmd_stbl_atom(parser):
+    """Find the stbl atom"""
+    minf_atom = find_gpmd_minf_atom(parser)
+    if not minf_atom:
+        return None
+    try:
+        for minf_field in minf_atom:
+            tag = minf_field['tag']
+            if tag.value != 'stbl':
+                continue
+            return minf_field['stbl']
+    except MissingField:
+        pass
+
+
+def find_gpmd_minf_atom(parser):
+    """Find minf atom for GPMF media"""
+    def recursive_search(atom):
+        try:
+            subtype = atom['hdlr/subtype']
+            if subtype.value == 'meta':
+                meta_atom = atom.parent
+                # print(meta_atom)
+                for subatom in meta_atom:
+                    tag = subatom['tag']
+                    if tag.value != 'minf':
+                        continue
+                    minf_atom = subatom['minf']
+                    #print("  {}".format(minf_atom))
+                    for minf_field in minf_atom:
+                        tag = minf_field['tag']
+                        #print("    {}".format(tag))
+                        if tag.value != 'gmhd':
+                            continue
+                        if b'gpmd' in minf_field['data'].value:
+                            return minf_atom
+        except MissingField:
+            pass
+        try:
+            for x in atom:
+                ret = recursive_search(x)
+                if ret:
+                    return ret
+        except KeyError as e:
+            pass
+        return None
+    return recursive_search(parser)
+
+
+def recursive_print(input):
+    """Recursively print hachoir parsed state"""
+    print(repr(input))
+    if isinstance(input, String):
+        print("  {}".format(input.display))
+    try:
+        for x in input:
+            recursive_print(x)
+    except KeyError as e:
+        pass
+
+
+if __name__ == '__main__':
+    import sys
+    parser = hachoir.parser.createParser(sys.argv[1])
+    with open(sys.argv[2], 'wb') as fp:
+        fp.write(
+            get_stream_data(
+                find_gpmd_stbl_atom(parser)
+            )
+        )

gpmf/parse.py

@@ -0,0 +1,143 @@
+# The MIT License (MIT)
+# Copyright (c) 2014 Eero af Heurlin
+# https://github.com/rambo/python-gpmf
+
+#!/usr/bin/env python3
+"""Parses the FOURCC data in GPMF stream into fields"""
+import struct
+
+import construct
+import dateutil.parser
+
+TYPES = construct.Enum(
+    construct.Byte,
+    int8_t=ord(b'b'),
+    uint8_t=ord(b'B'),
+    char=ord(b'c'),
+    int16_t=ord(b's'),
+    uint16_t=ord(b'S'),
+    int32_t=ord(b'l'),
+    uint32_t=ord(b'L'),
+    float=ord(b'f'),
+    double=ord(b'd'),
+    fourcc=ord(b'F'),
+    uuid=ord(b'G'),
+    int64_t=ord(b'j'),
+    uint64_t=ord(b'J'),
+    Q1516=ord(b'q'),
+    Q3132=ord(b'Q'),
+    utcdate=ord(b'U'),
+    complex=ord(b'?'),
+    nested=0x0,
+)
+
+FOURCC = construct.Struct(
+    "key" / construct.Bytes(4),
+    "type" / construct.Byte,
+    "size" / construct.Byte,
+    "repeat" / construct.Int16ub,
+    "data" / construct.Aligned(4, construct.Bytes(construct.this.size * construct.this.repeat))
+)
+
+
+def parse_value(element):
+    """Parses element value"""
+    type_parsed = TYPES.parse(bytes([element.type]))
+    #print("DEBUG: type_parsed={}, element.repeat={}, element.size={}, len(element.data): {}".format(type_parsed, element.repeat, element.size, len(element.data)))
+
+    # Special cases
+    if type_parsed == 'char' and element.key == b'GPSU':
+        return parse_goprodate(element)
+    if type_parsed == 'utcdate':
+        return parse_goprodate(element)
+
+    # Basic number types
+    struct_key = None
+    struct_repeat = element.repeat
+    if type_parsed == 'int32_t':
+        struct_key = 'l'
+        # It seems gopro is "creative" with grouped values and size vs repeat...
+        if element.size > 4:
+            struct_repeat = int(element.repeat * (element.size / 4))
+    if type_parsed == 'uint32_t':
+        struct_key = 'L'
+        if element.size > 4:
+            struct_repeat = int(element.repeat * (element.size / 4))
+
+    if type_parsed == 'int16_t':
+        struct_key = 'h'
+        if element.size > 2:
+            struct_repeat = int(element.repeat * (element.size / 2))
+    if type_parsed == 'uint16_t':
+        struct_key = 'H'
+        if element.size > 2:
+            struct_repeat = int(element.repeat * (element.size / 2))
+
+    if type_parsed == 'float':
+        struct_key = 'f'
+        if element.size > 4:
+            struct_repeat = int(element.repeat * (element.size / 4))
+
+    if not struct_key:
+        raise ValueError("{} does not have value parser yet".format(type_parsed))
+
+    struct_format = ">{}".format(''.join([struct_key for x in range(struct_repeat)]))
+    #print("DEBUG: struct_format={}".format(struct_format))
+    try:
+        value_parsed = struct.unpack(struct_format, element.data)
+    except struct.error as e:
+        #print("ERROR: {}".format(e))
+        #print("DEBUG: struct_format={}, data (len: {}) was: {}".format(struct_format, len(element.data), element.data))
+        raise ValueError("Struct unpack failed: {}".format(e))
+
+    # Single value
+    if len(value_parsed) == 1:
+        return value_parsed[0]
+    # Grouped values
+    if len(value_parsed) > element.repeat:
+        n = int(len(value_parsed) / element.repeat)
+        return [value_parsed[i:i + n] for i in range(0, len(value_parsed), n)]
+    return list(value_parsed)
+
+
+def parse_goprodate(element):
+    """Parses the gopro date string from element to Python datetime"""
+    goprotime = element.data.decode('UTF-8')
+    return dateutil.parser.parse("{}-{}-{}T{}:{}:{}Z".format(
+        2000 + int(goprotime[:2]),  # years
+        int(goprotime[2:4]),        # months
+        int(goprotime[4:6]),        # days
+        int(goprotime[6:8]),        # hours
+        int(goprotime[8:10]),       # minutes
+        float(goprotime[10:])       # seconds
+    ))
+
+
+def recursive(data, parents=tuple()):
+    """Recursive parser returns depth-first traversing generator yielding fields and list of their parent keys"""
+    elements = construct.GreedyRange(FOURCC).parse(data)
+    for element in elements:
+        if element.type == 0:
+            subparents = parents + (element.key,)
+            for subyield in recursive(element.data, subparents):
+                yield subyield
+        else:
+            yield (element, parents)
+
+
+if __name__ == '__main__':
+    import sys
+    from extract import get_gpmf_payloads_from_file
+    payloads, parser = get_gpmf_payloads_from_file(sys.argv[1])
+    for gpmf_data, timestamps in payloads:
+        for element, parents in recursive(gpmf_data):
+            try:
+                value = parse_value(element)
+            except ValueError:
+                value = element.data
+            print("{} {} > {}: {}".format(
+                timestamps,
+                ' > '.join([x.decode('ascii') for x in parents]),
+                element.key.decode('ascii'),
+                value
+            ))