Chris/initial fix

.gitignore

@@ -1,8 +1,10 @@
 # Datasets folder
 datasets/*
+src/data/
 
 # Outputs folder
-outputs/
+outputs/ELG_i60x36_f60x36_n32_m2/
+outputs/ELG_i180x108_f60x36_n64_m3/
 
 # 3rd party files
 src/3rdparty
@@ -14,3 +16,9 @@ __pycache__/
 # Python package caches
 dist/
 *.egg-info/
+
+src/commands.md
+
+.vscode/
+
+

setup.py

@@ -20,6 +20,7 @@
             'pandas',
             'ujson',
             'dlib',
+            'tensorflow>=1,<2',
 
             # Install the most appropriate version of Tensorflow
             # Ref. https://www.tensorflow.org/install/

src/elg_demo.py

@@ -15,6 +15,10 @@
 from models import ELG
 import util.gaze
 
+finish_thread = False
+is_shown = False
+image = None
+
 if __name__ == '__main__':
 
     # Set global log level
@@ -69,6 +73,7 @@
                                  eye_image_shape=(36, 60))
 
         # Define model
+        # CH Question: Why do videos vs webcams use different parameters?
         if args.from_video:
             model = ELG(
                 session, train_data={'videostream': data_source},
@@ -135,16 +140,18 @@ def _record_frame():
         # Begin visualization thread
         inferred_stuff_queue = queue.Queue()
 
+        # Constructs the video visualization of the output
+        # It receives the neural network output  from the inferred_stuff_queue
+
         def _visualize_output():
+            global is_shown
+            global image
+
             last_frame_index = 0
             last_frame_time = time.time()
             fps_history = []
             all_gaze_histories = []
 
-            if args.fullscreen:
-                cv.namedWindow('vis', cv.WND_PROP_FULLSCREEN)
-                cv.setWindowProperty('vis', cv.WND_PROP_FULLSCREEN, cv.WINDOW_FULLSCREEN)
-
             while True:
                 # If no output to visualize, show unannotated frame
                 if inferred_stuff_queue.empty():
@@ -153,12 +160,12 @@ def _visualize_output():
                         next_frame = data_source._frames[next_frame_index]
                         if 'faces' in next_frame and len(next_frame['faces']) == 0:
                             if not args.headless:
-                                cv.imshow('vis', next_frame['bgr'])
+                                image = next_frame['bgr'].copy()
+                                is_shown = True
                             if args.record_video:
                                 video_out_queue.put_nowait(next_frame_index)
                             last_frame_index = next_frame_index
-                    if cv.waitKey(1) & 0xFF == ord('q'):
-                        return
+
                     continue
 
                 # Get output from neural network and visualize
@@ -172,6 +179,7 @@ def _visualize_output():
 
                     # Decide which landmarks are usable
                     heatmaps_amax = np.amax(output['heatmaps'][j, :].reshape(-1, 18), axis=0)
+                    print(heatmaps_amax)
                     can_use_eye = np.all(heatmaps_amax > 0.7)
                     can_use_eyelid = np.all(heatmaps_amax[0:8] > 0.75)
                     can_use_iris = np.all(heatmaps_amax[8:16] > 0.8)
@@ -337,15 +345,15 @@ def _dstr(title, before_id, after_id):
                                    fontFace=cv.FONT_HERSHEY_DUPLEX, fontScale=0.79,
                                    color=(255, 255, 255), thickness=1, lineType=cv.LINE_AA)
                         if not args.headless:
-                            cv.imshow('vis', bgr)
+                            image = bgr.copy()
+                            is_shown = True
                         last_frame_index = frame_index
 
                         # Record frame?
                         if args.record_video:
                             video_out_queue.put_nowait(frame_index)
 
-                        # Quit?
-                        if cv.waitKey(1) & 0xFF == ord('q'):
+                        if finish_thread:
                             return
 
                         # Print timings
@@ -366,9 +374,14 @@ def _dstr(title, before_id, after_id):
         visualize_thread.daemon = True
         visualize_thread.start()
 
+        if args.fullscreen:
+            cv.namedWindow('vis', cv.WND_PROP_FULLSCREEN)
+            cv.setWindowProperty('vis', cv.WND_PROP_FULLSCREEN, cv.WINDOW_FULLSCREEN)
+
         # Do inference forever
         infer = model.inference_generator()
         while True:
+
             output = next(infer)
             for frame_index in np.unique(output['frame_index']):
                 if frame_index not in data_source._frames:
@@ -378,8 +391,17 @@ def _dstr(title, before_id, after_id):
                     frame['time']['inference'] += output['inference_time']
                 else:
                     frame['time']['inference'] = output['inference_time']
+
+            print('Output keys\n', output.keys())
+
             inferred_stuff_queue.put_nowait(output)
 
+            if is_shown:
+                cv.imshow('vis', image)
+                is_shown = False
+                if cv.waitKey(1) & 0xFF == ord('q'):
+                    finish_thread = True
+
             if not visualize_thread.isAlive():
                 break
 

src/util/get_weights.py

@@ -0,0 +1,17 @@
+import wget
+import os
+
+urls = ["https://ait.ethz.ch/projects/2018/landmarks-gaze/downloads/ELG_i180x108_f60x36_n64_m3.zip",
+        "https://ait.ethz.ch/projects/2018/landmarks-gaze/downloads/ELG_i60x36_f60x36_n32_m2.zip"]
+
+
+def get_weights(dest_dir):
+    for url in urls:
+        file_name = os.path.basename(url)
+        dest_path = os.path.join(dest_dir, file_name)
+        if os.path.exists(dest_path):
+            print("Weight file {} already downloaded".format(file_name))
+        else:
+            print("Downloading weights from {} to {}".format(url, dest_path))
+            os.makedirs(dest_dir, exist_ok=True)
+            wget.download(url, out=dest_path)