-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathprepare_all.py
245 lines (190 loc) · 8.4 KB
/
prepare_all.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
"""
create dataset
"""
from options import *
import pickle
import pandas as pd
import numpy as np
import math
import csv
def prepare_data():
"""
1. create a dataset in the following format:
{
vid: [[sa1, sa2, sb, sy], ...]
where each sa1 consists of n_workers action feature vectors, each of which is in the shape of (17,),
each sa2 consists of n_workers activity feature vectors, each of which is in the shape of (12,),
each sb consists of n_workers bounding boxes, each of which is in the shape of (4,), and
each sy consists of n_workers a2 indexes, each of which is an integer.
...
}
2. save it with
pickle.dump(dataset, open(os.path.join(param_fdp_labels, "all_x1_x2_xb_y.pickle"), "wb"))
"""
# *.a1 and *.a2 have all actions and activity results
# recognized using C3D based on tracklet information
# without creating images on local hard drivers, but in memory!
# for vid_ in sorted(os.listdir(param_fdp_videos)):
fp_test_list = os.path.join(param_fdp_data, "splits", "testlist01.txt")
f = open(fp_test_list)
try:
lines = f.readlines()
# for line in lines:
# print(line)
finally:
f.close()
gts_all = {}
for line in lines:
# activity/vid-segid-workerid
fds = line.split('/')
act = fds[0]
act_id = activities.index(act)
cnames = fds[1].split('-')
vid = cnames[0]
seg_no = int(cnames[1])
worker_id = int(cnames[2][:-1])
if vid in gts_all:
gts_all[vid].append([act_id, seg_no, worker_id])
else:
gts_all[vid] = [[act_id, seg_no, worker_id]]
for k, v in gts_all.items():
gts_all[k] = pd.DataFrame(gts_all[k], columns=['a2_ind', 'seg_no', 'wkr_id'])
dataset = {}
summary2 = []
for vid_ in sorted(os.listdir(param_fdp_videos)):
vid = vid_[:-4]
gts = gts_all[vid]
# load segments from *.a1 files. Although this information can be obtained from
# *.seg files, action recognition results cannot be assessed from them.
vid = vid_[:-4]
fp_a1 = os.path.join(param_fdp_segments, vid + ".a1")
with open(fp_a1, 'rb') as f_a1:
a1s = pickle.load(f_a1)
# *.a2 has the same data structure with *.a1, only with different labels.
# a1 falls into [0, 16], while a2 falls into [0, 11]
fp_a2 = os.path.join(param_fdp_segments, vid + ".a2")
with open(fp_a2, 'rb') as f_a2:
a2s = pickle.load(f_a2)
assert (len(a1s) == len(a2s))
subset = []
last_seg_no = a1s[0]['segment'][0]
n_segs = len(a1s)
sx1, sx2, sxb, sy = [], [], [], []
for k, [a1, a2] in enumerate(zip(a1s, a2s)):
# find the tracklets via filtering in dataframe
gt = gts[(gts['seg_no'] == a1['segment'][0]) & (gts['wkr_id'] == a1['segment'][1])]
# if found one
if len(gt) == 1:
# create a new sample if the seg_no changes
if a1['segment'][0] != last_seg_no:
# there should be at least two workers
if len(sx1) > 1:
subset.append([last_seg_no, sx1, sx2, sxb, sy])
print("Added a sample with {} workers at segment {}".format(len(sx1), last_seg_no))
last_seg_no = a1['segment'][0]
sx1, sx2, sxb, sy = [], [], [], []
sx1.append(a1['scores']) # a1_feature
sx2.append(a2['scores']) # a2_feature
sxb.append(a1['segment'][2:6]) # box
sy.append(gt.iloc[0, 0]) # a2_ind
# come to the last seg
if k == n_segs - 1:
# there should be at least two workers
if len(sx1) > 1:
subset.append([last_seg_no, sx1, sx2, sxb, sy])
print("Added a sample with {} workers at segment {}".format(len(sx1), last_seg_no))
print("=========================")
# else:
# print("tracklet segno_workerid: {}_{} not found".format(a1['segment'][0], a1['segment'][1]))
for _, _, _, sxb, sy in subset:
for box, y in zip(sxb, sy):
if y in dataset:
dataset[y].append([box[2]-box[0], box[3]-box[1]])
else:
dataset[y] = [[box[2]-box[0], box[3]-box[1]]]
# k nearest neighbors
for k in [1, 2, 3, 4, 5, 6, 7, 8, 9, 100]:
print("Create {} nearest neighbors".format(k))
# activity features
for pattern in range(3):
print("Create level {} activity features".format(pattern))
# by far, we have all necessary data for vid-level preparation
X = []
Y = []
count_tracklets = 0
count_edges = 0
for seg_no, sx1, sx2, sxb, sy in subset:
if len(sxb) > 0:
sub_edges = []
sub_edge_features = []
for i in range(len(sxb)):
sps = np.zeros(len(sxb))
for j in range(len(sxb)):
if i != j:
sp = sprel(sxb[i], sxb[j])
sps[j] = sp
# the largest kk neighbors
kk = min(k, len(sxb))
if len(sxb) > 0:
ends = sps.argsort()[-kk:][::-1]
for ae in ends:
if sps[ae] > 0:
sub_edges.append(np.array([i, ae], dtype=int))
ef = np.zeros(NA2)
ef[sy[ae]] = sps[ae]
sub_edge_features.append(ef)
if pattern == 0:
xs = [np.array(x) for x in sx1]
elif pattern == 1:
xs = [np.array(x) for x in sx2]
else:
xs = [np.hstack((np.array(x1), np.array(x2))) for x1, x2 in zip(sx1, sx2)]
X.append((np.array(xs), np.array(sub_edges), np.array(sub_edge_features)))
Y.append(np.array(sy))
count_tracklets += len(sy)
count_edges += len(sub_edges)
print("Found {} edges in segment {}".format(len(sub_edges), seg_no))
pickle.dump([X, Y], open(os.path.join(param_fdp_results, "{}_{}_{}.npd".format(k, pattern, vid)), "wb"))
summary2.append([vid, k, pattern, len(X), count_tracklets, count_edges])
summary = []
for k in range(NA2):
wh = np.array(dataset[k])
avg = np.average(wh, axis=0)
std = np.std(wh, axis=0)
summary.append([activities[k], len(dataset[k]), avg[0], avg[1], std[0], std[1]])
with open('summary.csv', 'w') as myfile:
wr = csv.writer(myfile)
for r in summary:
wr.writerow(r)
wr.writerow([""])
wr.writerow(["vid", "k", "act_feature_level", "segments", "tracklets", "edges"])
for r in summary2:
wr.writerow(r)
print("Done")
def sprel(bi, bj):
"""
Spatial relevance of two objects defined by their bounding boxes
:param bi:
:param bj:
:return:
"""
xx1 = np.maximum(bi[0], bj[0])
yy1 = np.maximum(bi[1], bj[1])
xx2 = np.minimum(bi[2], bj[2])
yy2 = np.minimum(bi[3], bj[3])
w = np.maximum(0.0, xx2 - xx1)
h = np.maximum(0.0, yy2 - yy1)
intersection = w * h
if intersection > 0:
area_i = (bi[2] - bi[0]) * (bi[3] - bi[1])
area_j = (bj[2] - bj[0]) * (bj[3] - bj[1])
return 0.5 * (1. + intersection / np.minimum(area_i, area_j))
else:
side_i = np.minimum((bi[2] - bi[0]), (bi[3] - bi[1]))
side_j = np.minimum((bj[2] - bj[0]), (bj[3] - bj[1]))
dist = np.minimum(math.fabs(xx2 - xx1), math.fabs(yy2 - yy1)) \
if (xx2 - xx1) * (yy2 - yy1) < 0 \
else math.sqrt((xx2 - xx1)**2 + (yy2 - yy1)**2)
return 0.5 * (side_i + side_j) / (side_i + side_j + dist)
if __name__ == '__main__':
prepare_data()