From cbadd29d062d57ec84422a9abd2d5ac5f3742f8d Mon Sep 17 00:00:00 2001
From: Chris Heinrich <cpheinrich@gmail.com>
Date: Mon, 6 Jul 2020 11:12:23 -0700
Subject: [PATCH 1/2] initial fix
---
.gitignore | 2 ++
src/elg_demo.py | 35 +++++++++++++++++++++++++----------
2 files changed, 27 insertions(+), 10 deletions(-)
diff --git a/.gitignore b/.gitignore
index 88ea4d7..d29200a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,6 @@
# Datasets folder
datasets/*
+src/data/
# Outputs folder
outputs/
@@ -14,3 +15,4 @@ __pycache__/
# Python package caches
dist/
*.egg-info/
+
diff --git a/src/elg_demo.py b/src/elg_demo.py
index b1a3ab7..abb1a7e 100644
--- a/src/elg_demo.py
+++ b/src/elg_demo.py
@@ -15,6 +15,10 @@
from models import ELG
import util.gaze
+finish_thread = False
+is_shown = False
+image = None
+
if __name__ == '__main__':
# Set global log level
@@ -136,15 +140,14 @@ def _record_frame():
inferred_stuff_queue = queue.Queue()
def _visualize_output():
+ global is_shown
+ global image
+
last_frame_index = 0
last_frame_time = time.time()
fps_history = []
all_gaze_histories = []
- if args.fullscreen:
- cv.namedWindow('vis', cv.WND_PROP_FULLSCREEN)
- cv.setWindowProperty('vis', cv.WND_PROP_FULLSCREEN, cv.WINDOW_FULLSCREEN)
-
while True:
# If no output to visualize, show unannotated frame
if inferred_stuff_queue.empty():
@@ -153,12 +156,12 @@ def _visualize_output():
next_frame = data_source._frames[next_frame_index]
if 'faces' in next_frame and len(next_frame['faces']) == 0:
if not args.headless:
- cv.imshow('vis', next_frame['bgr'])
+ image = next_frame['bgr'].copy()
+ is_shown = True
if args.record_video:
video_out_queue.put_nowait(next_frame_index)
last_frame_index = next_frame_index
- if cv.waitKey(1) & 0xFF == ord('q'):
- return
+
continue
# Get output from neural network and visualize
@@ -172,6 +175,7 @@ def _visualize_output():
# Decide which landmarks are usable
heatmaps_amax = np.amax(output['heatmaps'][j, :].reshape(-1, 18), axis=0)
+ print(heatmaps_amax)
can_use_eye = np.all(heatmaps_amax > 0.7)
can_use_eyelid = np.all(heatmaps_amax[0:8] > 0.75)
can_use_iris = np.all(heatmaps_amax[8:16] > 0.8)
@@ -337,15 +341,15 @@ def _dstr(title, before_id, after_id):
fontFace=cv.FONT_HERSHEY_DUPLEX, fontScale=0.79,
color=(255, 255, 255), thickness=1, lineType=cv.LINE_AA)
if not args.headless:
- cv.imshow('vis', bgr)
+ image = bgr.copy()
+ is_shown = True
last_frame_index = frame_index
# Record frame?
if args.record_video:
video_out_queue.put_nowait(frame_index)
- # Quit?
- if cv.waitKey(1) & 0xFF == ord('q'):
+ if finish_thread:
return
# Print timings
@@ -366,9 +370,14 @@ def _dstr(title, before_id, after_id):
visualize_thread.daemon = True
visualize_thread.start()
+ if args.fullscreen:
+ cv.namedWindow('vis', cv.WND_PROP_FULLSCREEN)
+ cv.setWindowProperty('vis', cv.WND_PROP_FULLSCREEN, cv.WINDOW_FULLSCREEN)
+
# Do inference forever
infer = model.inference_generator()
while True:
+
output = next(infer)
for frame_index in np.unique(output['frame_index']):
if frame_index not in data_source._frames:
@@ -380,6 +389,12 @@ def _dstr(title, before_id, after_id):
frame['time']['inference'] = output['inference_time']
inferred_stuff_queue.put_nowait(output)
+ if is_shown:
+ cv.imshow('vis', image)
+ is_shown = False
+ if cv.waitKey(1) & 0xFF == ord('q'):
+ finish_thread = True
+
if not visualize_thread.isAlive():
break
From d74dc35ae7c19f46f63e61f6b32bf264701fdd83 Mon Sep 17 00:00:00 2001
From: Chris Heinrich <cpheinrich@gmail.com>
Date: Tue, 7 Jul 2020 18:18:16 -0700
Subject: [PATCH 2/2] some small fixes
---
.gitignore | 8 +++++++-
setup.py | 1 +
src/elg_demo.py | 7 +++++++
src/util/get_weights.py | 17 +++++++++++++++++
4 files changed, 32 insertions(+), 1 deletion(-)
create mode 100644 src/util/get_weights.py
diff --git a/.gitignore b/.gitignore
index d29200a..20a824d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,7 +3,8 @@ datasets/*
src/data/
# Outputs folder
-outputs/
+outputs/ELG_i60x36_f60x36_n32_m2/
+outputs/ELG_i180x108_f60x36_n64_m3/
# 3rd party files
src/3rdparty
@@ -16,3 +17,8 @@ __pycache__/
dist/
*.egg-info/
+src/commands.md
+
+.vscode/
+
+
diff --git a/setup.py b/setup.py
index 5f8b8f2..527d6a8 100644
--- a/setup.py
+++ b/setup.py
@@ -20,6 +20,7 @@
'pandas',
'ujson',
'dlib',
+ 'tensorflow>=1,<2',
# Install the most appropriate version of Tensorflow
# Ref. https://www.tensorflow.org/install/
diff --git a/src/elg_demo.py b/src/elg_demo.py
index abb1a7e..4634329 100644
--- a/src/elg_demo.py
+++ b/src/elg_demo.py
@@ -73,6 +73,7 @@
eye_image_shape=(36, 60))
# Define model
+ # CH Question: Why do videos vs webcams use different parameters?
if args.from_video:
model = ELG(
session, train_data={'videostream': data_source},
@@ -139,6 +140,9 @@ def _record_frame():
# Begin visualization thread
inferred_stuff_queue = queue.Queue()
+ # Constructs the video visualization of the output
+ # It receives the neural network output from the inferred_stuff_queue
+
def _visualize_output():
global is_shown
global image
@@ -387,6 +391,9 @@ def _dstr(title, before_id, after_id):
frame['time']['inference'] += output['inference_time']
else:
frame['time']['inference'] = output['inference_time']
+
+ print('Output keys\n', output.keys())
+
inferred_stuff_queue.put_nowait(output)
if is_shown:
diff --git a/src/util/get_weights.py b/src/util/get_weights.py
new file mode 100644
index 0000000..2b5ebd7
--- /dev/null
+++ b/src/util/get_weights.py
@@ -0,0 +1,17 @@
+import wget
+import os
+
+urls = ["https://ait.ethz.ch/projects/2018/landmarks-gaze/downloads/ELG_i180x108_f60x36_n64_m3.zip",
+ "https://ait.ethz.ch/projects/2018/landmarks-gaze/downloads/ELG_i60x36_f60x36_n32_m2.zip"]
+
+
+def get_weights(dest_dir):
+ for url in urls:
+ file_name = os.path.basename(url)
+ dest_path = os.path.join(dest_dir, file_name)
+ if os.path.exists(dest_path):
+ print("Weight file {} already downloaded".format(file_name))
+ else:
+ print("Downloading weights from {} to {}".format(url, dest_path))
+ os.makedirs(dest_dir, exist_ok=True)
+ wget.download(url, out=dest_path)