Code and instructions for reproducing results.

README.md

@@ -1,2 +1,82 @@
-# hhi-stmrftracking
-Compressed-Domain Video Object Tracking using Markov Random Fields with Graph Cuts Optimization
+# Compressed-Domain Video Object Tracking using Markov Random Fields with Graph Cuts Optimization
+
+Research group at Fraunhofer HHI: <https://www.hhi.fraunhofer.de/en/departments/vca/research-groups/multimedia-communications.html>
+
+The application performs video object tracking with compressed-domain processing of H.264/AVC video bitstreams applying Markov Random Fields and Graph Cuts methods.
+
+---
+## Compiling
+##### Compile FFmpeg decoder for motion vector extraction
+1. Clone repository into `libs` folder
+```sh
+cd libs
+git clone https://github.com/bombardellif/FFmpeg.git
+cd FFmpeg
+mkdir bin
+```
+2. [Install dependencies and compile](https://trac.ffmpeg.org/wiki/CompilationGuide/Ubuntu) (assuming Linux Ubuntu)
+```sh
+sudo apt install autoconf \
+    automake \
+    build-essential \
+    cmake \
+    libass-dev \
+    libfreetype6-dev \
+    libtheora-dev \
+    libtool \
+    libvorbis-dev \
+    mercurial \
+    pkg-config \
+    texinfo \
+    zlib1g-dev \
+    libx264-dev \
+    yasm
+PATH="./bin:$PATH" PKG_CONFIG_PATH="./ffmpeg_build/lib/pkgconfig" ./configure
+--prefix=./ffmpeg_build --bindir=./bin --extra-cflags=-I./ffmpeg_build/include
+--extra-ldflags=-L./ffmpeg_build/lib --enable-gpl --enable-libx264 --enablenonfree --enable-shared
+PATH="./bin:$PATH" make -j4 && make install
+```
+##### Install application dependencies
+1. Install Python and dependencies
+```sh
+sudo apt install python3-pip python3-dev python3-tk g++ gfortran liblapack-dev \
+    liblapacke-dev libatlas-dev libopenblas-dev python3-cffi-backend \
+    python3-cairo-dev libffi-dev python-opencv libopencv-dev
+cd ../../
+sudo pip3 install -r requirements.txt
+```
+2. Compile the sub modules
+```sh
+cd utils
+python3 setup.py build_ext --inplace
+cd ../mincut
+python3 setup.py build_ext --inplace
+cd ../decoder
+python3 setup.py build_ext --inplace
+```
+3. Run the application (help output)
+```sh
+cd ..
+LD_LIBRARY_PATH=libs/FFmpeg/ffmpeg_build/lib python3 -m hhi_stmrftracking.main -h
+```
+
+---
+## Datasets
+
+#### [VOT 2016 Challenge](http://www.votchallenge.net/vot2016/) Dataset
+
+Download the pictures and ground truth at <http://www.votchallenge.net/vot2016/dataset.html>.
+
+Copy the shell script `docs/encode-all.sh` inside the downloaded folder and execute it to encode the pictures in H.264/AVC.
+
+#### Derf Dataset ([ST-MRF by Khatoonabadi and Bajić](http://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=6272352&tag=1))
+
+Download the ST-MRF code and dataset at <https://www.researchgate.net/publication/258938622_ST-MRF_tracking>. Videos are already encoded.
+
+---
+## Example
+With the dataset copied to the `data` folder. The last parameter is the ground truth for the first picture of the video sequence.
+```
+LD_LIBRARY_PATH=libs/FFmpeg/ffmpeg_build/lib python3 -m hhi_stmrftracking.main -d data/in/gymnastics4.264 -e data/gt-gymnastics4/ data/in/gymnastics4.264 data/in/gt-gymnastics4.png
+```
+The expected output can be found in `docs/example_gymnastics4.264.eval.mp4`.

decoder/decoder.pyx

@@ -0,0 +1,65 @@
+from libcpp cimport bool
+from libc.stddef cimport size_t
+from libc.stdint cimport int16_t
+from cpython cimport pycapsule
+import numpy as np
+cimport numpy as cnp
+
+cimport mvextract
+from streambuffer cimport StreamBuffer
+from utils cimport arraywrapper
+
+# Global mapping of id to stream buffer pointer
+buffer_map = {}
+EMPTY_ARRAY = np.empty(0, dtype=np.int16)
+
+def init():
+    mvextract.init()
+
+def read_videostream(tracking_id, py_url):
+    cdef StreamBuffer* buff
+    cdef bytes py_byte_url = py_url.encode()
+    cdef const char* url = py_byte_url
+    ret = False
+    if tracking_id not in buffer_map:
+        buff = mvextract.read_videostream(url)
+        if buff is not NULL:
+            buffer_map[tracking_id] = pycapsule.PyCapsule_New(<void*>buff,
+                                                              NULL, NULL)
+            ret = True
+    return ret
+
+def stop(tracking_id):
+    cdef StreamBuffer* buff
+    if tracking_id in buffer_map:
+        buff = <StreamBuffer*>pycapsule.PyCapsule_GetPointer(
+            buffer_map[tracking_id],
+            NULL)
+        mvextract.stop(buff)
+
+def get_next_frame(tracking_id):
+    cdef StreamBuffer* buff
+    cdef int16_t (*data)[4]
+    cdef size_t size
+    cdef int shape[2]
+    cdef bool status = False
+    cdef cnp.ndarray array = EMPTY_ARRAY
+    if tracking_id in buffer_map:
+        buff = <StreamBuffer*>pycapsule.PyCapsule_GetPointer(
+            buffer_map[tracking_id],
+            NULL)
+        status = mvextract.get_next_frame(buff, &data, &size)
+        shape[0],shape[1] = (<int>size),4
+        if status and data is not NULL:
+            array = arraywrapper.create_ndarray(<void*>data, shape,
+                                                cnp.NPY_INT16)
+    return status, array
+
+def destroy(tracking_id):
+    cdef StreamBuffer* buff
+    if tracking_id in buffer_map:
+        buff = <StreamBuffer*>pycapsule.PyCapsule_GetPointer(
+            buffer_map[tracking_id],
+            NULL)
+        mvextract.destroy(buff)
+        del buffer_map[tracking_id]

decoder/mvextract.pxd

@@ -0,0 +1,17 @@
+
+from libcpp cimport bool
+from libc.stddef cimport size_t
+from libc.stdint cimport int16_t
+
+from streambuffer cimport StreamBuffer
+
+cdef extern from "mvextract/mvextract.h":
+    bool init()
+
+    StreamBuffer* read_videostream(const char url[])
+
+    void stop(StreamBuffer* buff)
+
+    bool get_next_frame(StreamBuffer* buff, int16_t (**out)[4], size_t* size)
+
+    void destroy(StreamBuffer* buff)

decoder/mvextract/Makefile

@@ -0,0 +1,19 @@
+CC      = gcc
+CFLAGS  = -Wall -std=c11 -g
+# LDFLAGS = -lswscale -lavdevice -lavformat -lavcodec -lswresample -lavutil -lpthread -lbz2 -lz -lc -lrt
+LDFLAGS = -lavutil -lavformat -lavcodec -lswresample -lz -lm -lpthread
+FFMPEG_INC = -I/home/bombardelli/dev/FFmpeg/ffmpeg_build/include
+FFMPEG_LIBS = -L/home/bombardelli/dev/FFmpeg/ffmpeg_build/lib
+
+SRCS=$(wildcard *.c)
+OBJS=$(SRCS:.c=.o)
+all: $(OBJS)
+
+test: $(OBJS)
+	$(CC) $(CFLAGS) -o $@ $^ $(FFMPEG_LIBS) $(LDFLAGS)
+
+%.o: %.c
+	$(CC) $(CFLAGS) $(FFMPEG_INC) -c $<
+
+clean:
+	rm -rf *.o

decoder/mvextract/concurrentqueue.c

@@ -0,0 +1,47 @@
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <stdatomic.h>
+
+#include "concurrentqueue.h"
+
+static inline size_t inc(size_t a) {
+    return (a+1) % QUEUE_CAPACITY;
+}
+
+bool concurrentqueue_push(ConcurrentQueue* self, int16_t (*data)[4], size_t size) {
+    size_t curr_tail = atomic_load(&(self->tail));
+    size_t next_tail = inc(curr_tail);
+    // If queue is not full
+    if (next_tail != atomic_load(&(self->head))) {
+        self->buffer[curr_tail] = (QueueNode){.data = data, .size = size};
+        // increment the tail
+        atomic_store(&(self->tail), next_tail);
+        return true;
+    }
+    return false;
+}
+
+bool concurrentqueue_pop(ConcurrentQueue* self, int16_t (**out)[4], size_t* size) {
+    size_t curr_head = atomic_load(&(self->head));
+    // if queue is not empty
+    if (curr_head != atomic_load(&(self->tail))) {
+        // Set output values
+        *out = self->buffer[curr_head].data;
+        *size = self->buffer[curr_head].size;
+        // Increase the head towards the tail
+        atomic_store(&(self->head), inc(curr_head));
+        return true;
+    }
+    return false;
+}
+
+void concurrentqueue_free(ConcurrentQueue* self) {
+    // Free operation is not thread safe. Exclusive access must be guaranteed
+    for (size_t head = self->head, tail = self->tail;
+    head != tail;
+    head = inc(head)) {
+        free(self->buffer[head].data);
+    }
+}

decoder/mvextract/concurrentqueue.h

@@ -0,0 +1,25 @@
+#ifndef __CONCURRENTQUEUE_H__
+#define __CONCURRENTQUEUE_H__
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#define QUEUE_CAPACITY 256
+
+typedef struct QueueNode {
+    int16_t (*data)[4];
+    size_t size;
+} QueueNode;
+
+typedef struct ConcurrentQueue {
+    _Atomic size_t head;
+    _Atomic size_t tail;
+    QueueNode buffer[QUEUE_CAPACITY];
+} ConcurrentQueue;
+
+bool concurrentqueue_push(ConcurrentQueue* self, int16_t (*data)[4], size_t size);
+bool concurrentqueue_pop(ConcurrentQueue* self, int16_t (**out)[4], size_t* size);
+void concurrentqueue_free(ConcurrentQueue* self);
+
+#endif