From 7b6bd3154fe419d523d739e63932ead86d93a457 Mon Sep 17 00:00:00 2001 From: Justin Ruan Date: Sat, 16 Dec 2023 16:35:39 +0000 Subject: [PATCH 01/11] support computing hota metrics --- motmetrics/distances.py | 8 +-- motmetrics/metrics.py | 55 +++++++++++++++++++ motmetrics/mot.py | 62 ++++++++++++---------- motmetrics/utils.py | 115 ++++++++++++++++++++++++++++++++++++++++ 4 files changed, 210 insertions(+), 30 deletions(-) diff --git a/motmetrics/distances.py b/motmetrics/distances.py index 93dfe1e..ccd4723 100644 --- a/motmetrics/distances.py +++ b/motmetrics/distances.py @@ -80,7 +80,7 @@ def boxiou(a, b): math_util.quiet_divide(i_vol, u_vol)) -def iou_matrix(objs, hyps, max_iou=1.): +def iou_matrix(objs, hyps, max_iou=1., return_dist=True): """Computes 'intersection over union (IoU)' distance matrix between object and hypothesis rectangles. The IoU is computed as @@ -119,5 +119,7 @@ def iou_matrix(objs, hyps, max_iou=1.): assert objs.shape[1] == 4 assert hyps.shape[1] == 4 iou = boxiou(objs[:, None], hyps[None, :]) - dist = 1 - iou - return np.where(dist > max_iou, np.nan, dist) + if return_dist: + dist = 1 - iou + return np.where(dist > max_iou, np.nan, dist) + return iou diff --git a/motmetrics/metrics.py b/motmetrics/metrics.py index e16f16c..362f49e 100644 --- a/motmetrics/metrics.py +++ b/motmetrics/metrics.py @@ -495,6 +495,21 @@ def num_predictions(df, pred_frequencies): simple_add_func.append(num_predictions) +def num_gt_ids(df): + """Number of unique object ids encountered.""" + return df.full["OId"].dropna().unique().shape[0] + + +simple_add_func.append(num_gt_ids) + + +def num_dt_ids(df): + return df.full["HId"].dropna().unique().shape[0] + + +simple_add_func.append(num_dt_ids) + + def track_ratios(df, obj_frequencies): """Ratio of assigned to total appearance count per unique object id.""" tracked = df.noraw[df.noraw.Type != "MISS"]["OId"].value_counts() @@ -597,6 +612,40 @@ def recall_m(partials, num_detections, num_objects): return math_util.quiet_divide(num_detections, num_objects) +def deta_alpha(df, num_detections, num_objects, num_false_positives): + r"""DeTA under specific threshold $\alpha$ + Source: https://jonathonluiten.medium.com/how-to-evaluate-tracking-with-the-hota-metrics-754036d183e1 + """ + del df # unused + return math_util.quiet_divide(num_detections, max(1, num_objects + num_false_positives)) + + +def assa_alpha(df, num_detections, num_gt_ids, num_dt_ids): + r"""AssA under specific threshold $\alpha$ + Source: https://github.com/JonathonLuiten/TrackEval/blob/12c8791b303e0a0b50f753af204249e622d0281a/trackeval/metrics/hota.py#L107-L108 + """ + match_count_array = np.zeros((num_gt_ids, num_dt_ids)) + gt_id_counts = np.zeros((num_gt_ids, 1)) + tracker_id_counts = np.zeros((1, num_dt_ids)) + for idx in range(len(df.noraw)): + oid, hid = df.noraw.iloc[idx, 1], df.noraw.iloc[idx, 2] + if df.noraw.iloc[idx, 0] in ["SWITCH", "MATCH"]: + match_count_array[int(oid) - 1, int(hid) - 1] += 1 + if oid == oid: # check non nan + gt_id_counts[int(oid) - 1] += 1 + if hid == hid: + tracker_id_counts[0, int(hid) - 1] += 1 + + ass_a = match_count_array / np.maximum(1, gt_id_counts + tracker_id_counts - match_count_array) + return math_util.quiet_divide((ass_a * match_count_array).sum(), max(1, num_detections)) + + +def hota_alpha(df, deta_alpha, assa_alpha): + r"""HOTA under specific threshold $\alpha$""" + del df + return (deta_alpha * assa_alpha) ** 0.5 + + class DataFrameMap: # pylint: disable=too-few-public-methods def __init__(self, full, raw, noraw, extra): self.full = full @@ -783,6 +832,8 @@ def create(): m.register(num_detections, formatter="{:d}".format) m.register(num_objects, formatter="{:d}".format) m.register(num_predictions, formatter="{:d}".format) + m.register(num_gt_ids, formatter="{:d}".format) + m.register(num_dt_ids, formatter="{:d}".format) m.register(num_unique_objects, formatter="{:d}".format) m.register(track_ratios) m.register(mostly_tracked, formatter="{:d}".format) @@ -801,6 +852,10 @@ def create(): m.register(idp, formatter="{:.1%}".format) m.register(idr, formatter="{:.1%}".format) m.register(idf1, formatter="{:.1%}".format) + + m.register(deta_alpha, formatter="{:.1%}".format) + m.register(assa_alpha, formatter="{:.1%}".format) + m.register(hota_alpha, formatter="{:.1%}".format) return m diff --git a/motmetrics/mot.py b/motmetrics/mot.py index ec7ae59..a4bd3e9 100644 --- a/motmetrics/mot.py +++ b/motmetrics/mot.py @@ -134,7 +134,7 @@ def _append_to_events(self, typestr, oid, hid, distance): self._events['HId'].append(hid) self._events['D'].append(distance) - def update(self, oids, hids, dists, frameid=None, vf=''): + def update(self, oids, hids, dists, frameid=None, vf='', similartiy_matrix=None, th=None): """Updates the accumulator with frame specific objects/detections. This method generates events based on the following algorithm [1]: @@ -202,6 +202,12 @@ def update(self, oids, hids, dists, frameid=None, vf=''): self._append_to_indices(frameid, next(eid)) self._append_to_events('RAW', np.nan, np.nan, np.nan) + # Postcompute the distance matrix if necessary. (e.g., HOTA) + cost_for_matching = dists.copy() + if similartiy_matrix is not None and th is not None: + dists = 1 - similartiy_matrix + dists = np.where(similartiy_matrix < th - np.finfo("float").eps, np.nan, dists) + # There must be at least one RAW event per object and hypothesis. # Record all finite distances as RAW events. valid_i, valid_j = np.where(np.isfinite(dists)) @@ -224,34 +230,36 @@ def update(self, oids, hids, dists, frameid=None, vf=''): if oids.size * hids.size > 0: # 1. Try to re-establish tracks from correspondences in last update - for i in range(oids.shape[0]): - # No need to check oids_masked[i] here. - if not (oids[i] in self.m and self.last_match[oids[i]] == self.last_update_frameid): - continue - - hprev = self.m[oids[i]] - j, = np.where(~hids_masked & (hids == hprev)) - if j.shape[0] == 0: - continue - j = j[0] + # ignore this if post processing is performed (e.g., HOTA) + if similartiy_matrix is None or th is None: + for i in range(oids.shape[0]): + # No need to check oids_masked[i] here. + if not (oids[i] in self.m and self.last_match[oids[i]] == self.last_update_frameid): + continue + + hprev = self.m[oids[i]] + j, = np.where(~hids_masked & (hids == hprev)) + if j.shape[0] == 0: + continue + j = j[0] + + if np.isfinite(dists[i, j]): + o = oids[i] + h = hids[j] + oids_masked[i] = True + hids_masked[j] = True + self.m[oids[i]] = hids[j] - if np.isfinite(dists[i, j]): - o = oids[i] - h = hids[j] - oids_masked[i] = True - hids_masked[j] = True - self.m[oids[i]] = hids[j] - - self._append_to_indices(frameid, next(eid)) - self._append_to_events('MATCH', oids[i], hids[j], dists[i, j]) - self.last_match[o] = frameid - self.hypHistory[h] = frameid + self._append_to_indices(frameid, next(eid)) + self._append_to_events('MATCH', oids[i], hids[j], dists[i, j]) + self.last_match[o] = frameid + self.hypHistory[h] = frameid # 2. Try to remaining objects/hypotheses dists[oids_masked, :] = np.nan dists[:, hids_masked] = np.nan - rids, cids = linear_sum_assignment(dists) + rids, cids = linear_sum_assignment(cost_for_matching) for i, j in zip(rids, cids): if not np.isfinite(dists[i, j]): @@ -265,10 +273,10 @@ def update(self, oids, hids, dists, frameid=None, vf=''): # self.m[o] != h and # abs(frameid - self.last_occurrence[o]) <= self.max_switch_time) switch_condition = ( - o in self.m and - self.m[o] != h and - o in self.last_occurrence and # Ensure the object ID 'o' is initialized in last_occurrence - abs(frameid - self.last_occurrence[o]) <= self.max_switch_time + o in self.m and + self.m[o] != h and + o in self.last_occurrence and # Ensure the object ID 'o' is initialized in last_occurrence + abs(frameid - self.last_occurrence[o]) <= self.max_switch_time ) is_switch = switch_condition ###################################################################### diff --git a/motmetrics/utils.py b/motmetrics/utils.py index 520573a..d9082ff 100644 --- a/motmetrics/utils.py +++ b/motmetrics/utils.py @@ -18,6 +18,121 @@ from motmetrics.preprocess import preprocessResult +def compute_global_aligment_score( + allframeids, fid_to_fgt, fid_to_fdt, num_gt_id, num_det_id, dist_func +): + """Taken from https://github.com/JonathonLuiten/TrackEval/blob/12c8791b303e0a0b50f753af204249e622d0281a/trackeval/metrics/hota.py""" + potential_matches_count = np.zeros((num_gt_id, num_det_id)) + gt_id_count = np.zeros((num_gt_id, 1)) + tracker_id_count = np.zeros((1, num_det_id)) + + for fid in allframeids: + oids = np.empty(0) + hids = np.empty(0) + if fid in fid_to_fgt: + fgt = fid_to_fgt[fid] + oids = fgt.index.get_level_values("Id") + if fid in fid_to_fdt: + fdt = fid_to_fdt[fid] + hids = fdt.index.get_level_values("Id") + if len(oids) > 0 and len(hids) > 0: + gt_ids = np.array(oids.values) - 1 + dt_ids = np.array(hids.values) - 1 + similarity = dist_func(fgt.values, fdt.values, return_dist=False) + + sim_iou_denom = ( + similarity.sum(0)[np.newaxis, :] + similarity.sum(1)[:, np.newaxis] - similarity + ) + sim_iou = np.zeros_like(similarity) + sim_iou_mask = sim_iou_denom > 0 + np.finfo("float").eps + sim_iou[sim_iou_mask] = similarity[sim_iou_mask] / sim_iou_denom[sim_iou_mask] + potential_matches_count[gt_ids[:, np.newaxis], dt_ids[np.newaxis, :]] += sim_iou + + # Calculate the total number of dets for each gt_id and tracker_id. + gt_id_count[gt_ids] += 1 + tracker_id_count[0, dt_ids] += 1 + global_alignment_score = potential_matches_count / ( + gt_id_count + tracker_id_count - potential_matches_count + ) + return global_alignment_score + + +def compare_to_groundtruth_reweighting(gt, dt, dist="iou", distfields=None, distth=(0.5)): + # pylint: disable=too-many-locals + if distfields is None: + distfields = ["X", "Y", "Width", "Height"] + + def compute_iou(a, b, return_dist): + return iou_matrix(a, b, max_iou=distth, return_dist=return_dist) + + def compute_euc(a, b, *args, **kwargs): + return np.sqrt(norm2squared_matrix(a, b, max_d2=distth**2)) + + def compute_seuc(a, b, *args, **kwargs): + return norm2squared_matrix(a, b, max_d2=distth) + + if dist.upper() == "IOU": + compute_dist = compute_iou + elif dist.upper() == "EUC": + compute_dist = compute_euc + import warnings + + warnings.warn( + f"'euc' flag changed its behavior. The euclidean distance is now used instead of the squared euclidean distance. Make sure the used threshold (distth={distth}) is not squared. Use 'euclidean' flag to avoid this warning." + ) + elif dist.upper() == "EUCLIDEAN": + compute_dist = compute_euc + elif dist.upper() == "SEUC": + compute_dist = compute_seuc + else: + raise f'Unknown distance metric {dist}. Use "IOU", "EUCLIDEAN", or "SEUC"' + + return_single = False + if isinstance(distth, float): + distth = [distth] + return_single = True + + acc_list = [MOTAccumulator() for _ in range(len(distth))] + + num_gt_id = gt.index.get_level_values("Id").max() + num_det_id = dt.index.get_level_values("Id").max() + + # We need to account for all frames reported either by ground truth or + # detector. In case a frame is missing in GT this will lead to FPs, in + # case a frame is missing in detector results this will lead to FNs. + allframeids = gt.index.union(dt.index).levels[0] + + gt = gt[distfields] + dt = dt[distfields] + fid_to_fgt = dict(iter(gt.groupby("FrameId"))) + fid_to_fdt = dict(iter(dt.groupby("FrameId"))) + + global_alignment_score = compute_global_aligment_score( + allframeids, fid_to_fgt, fid_to_fdt, num_gt_id, num_det_id, compute_dist + ) + + for fid in allframeids: + oids = np.empty(0) + hids = np.empty(0) + weighted_dists = np.empty((0, 0)) + if fid in fid_to_fgt: + fgt = fid_to_fgt[fid] + oids = fgt.index.get_level_values("Id") + if fid in fid_to_fdt: + fdt = fid_to_fdt[fid] + hids = fdt.index.get_level_values("Id") + if len(oids) > 0 and len(hids) > 0: + gt_ids = np.array(oids.values) - 1 + dt_ids = np.array(hids.values) - 1 + dists = compute_dist(fgt.values, fdt.values, return_dist=False) + weighted_dists = ( + dists * global_alignment_score[gt_ids[:, np.newaxis], dt_ids[np.newaxis, :]] + ) + for acc, th in zip(acc_list, distth): + acc.update(oids, hids, 1 - weighted_dists, frameid=fid, similartiy_matrix=dists, th=th) + return acc_list[0] if return_single else acc_list + + def compare_to_groundtruth(gt, dt, dist='iou', distfields=None, distth=0.5): """Compare groundtruth and detector results. From 16e88daac55acebba0f561c7ee1764619a7592bb Mon Sep 17 00:00:00 2001 From: Justin Ruan Date: Sat, 16 Dec 2023 16:37:03 +0000 Subject: [PATCH 02/11] fix code style for flake8 --- motmetrics/metrics.py | 2 +- motmetrics/mot.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/motmetrics/metrics.py b/motmetrics/metrics.py index 362f49e..920090d 100644 --- a/motmetrics/metrics.py +++ b/motmetrics/metrics.py @@ -852,7 +852,7 @@ def create(): m.register(idp, formatter="{:.1%}".format) m.register(idr, formatter="{:.1%}".format) m.register(idf1, formatter="{:.1%}".format) - + m.register(deta_alpha, formatter="{:.1%}".format) m.register(assa_alpha, formatter="{:.1%}".format) m.register(hota_alpha, formatter="{:.1%}".format) diff --git a/motmetrics/mot.py b/motmetrics/mot.py index a4bd3e9..39771f9 100644 --- a/motmetrics/mot.py +++ b/motmetrics/mot.py @@ -478,7 +478,7 @@ def merge_event_dataframes(dfs, update_frame_indices=True, update_oids=True, upd copy['HId'] = copy['HId'].map(lambda x: hid_map[x], na_action='ignore') infos['hid_map'] = hid_map - r = pd.concat([r,copy]) + r = pd.concat([r, copy]) mapping_infos.append(infos) if return_mappings: From 96258e8d8cb719df6f44d6481bd6636b8184ac33 Mon Sep 17 00:00:00 2001 From: Justin Ruan Date: Sun, 17 Dec 2023 05:31:29 +0000 Subject: [PATCH 03/11] update docstring --- motmetrics/distances.py | 3 +++ motmetrics/metrics.py | 3 ++- motmetrics/utils.py | 26 ++++++++++++++++++++++++++ 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/motmetrics/distances.py b/motmetrics/distances.py index ccd4723..8ad7113 100644 --- a/motmetrics/distances.py +++ b/motmetrics/distances.py @@ -104,11 +104,14 @@ def iou_matrix(objs, hyps, max_iou=1., return_dist=True): Maximum tolerable overlap distance. Object / hypothesis points with larger distance are set to np.nan signalling do-not-pair. Defaults to 0.5 + return_dist : bool + If true, return distance matrix. If false, return similarity (IoU) matrix. Returns ------- C : NxK array Distance matrix containing pairwise distances or np.nan. + if `return_dist` is False, then the matrix contains the pairwise IoU. """ if np.size(objs) == 0 or np.size(hyps) == 0: diff --git a/motmetrics/metrics.py b/motmetrics/metrics.py index 920090d..efc4543 100644 --- a/motmetrics/metrics.py +++ b/motmetrics/metrics.py @@ -496,7 +496,7 @@ def num_predictions(df, pred_frequencies): def num_gt_ids(df): - """Number of unique object ids encountered.""" + """Number of unique gt ids.""" return df.full["OId"].dropna().unique().shape[0] @@ -504,6 +504,7 @@ def num_gt_ids(df): def num_dt_ids(df): + """Number of unique dt ids.""" return df.full["HId"].dropna().unique().shape[0] diff --git a/motmetrics/utils.py b/motmetrics/utils.py index d9082ff..c47c4c8 100644 --- a/motmetrics/utils.py +++ b/motmetrics/utils.py @@ -58,6 +58,32 @@ def compute_global_aligment_score( def compare_to_groundtruth_reweighting(gt, dt, dist="iou", distfields=None, distth=(0.5)): + """Compare groundtruth and detector results with global alignment score. + + This method assumes both results are given in terms of DataFrames with at least the following fields + - `FrameId` First level index used for matching ground-truth and test frames. + - `Id` Secondary level index marking available object / hypothesis ids + + Depending on the distance to be used relevant distfields need to be specified. + + Params + ------ + gt : pd.DataFrame + Dataframe for ground-truth + test : pd.DataFrame + Dataframe for detector results + + Kwargs + ------ + dist : str, optional + String identifying distance to be used. Defaults to intersection over union ('iou'). Euclidean + distance ('euclidean') and squared euclidean distance ('seuc') are also supported. + distfields: array, optional + Fields relevant for extracting distance information. Defaults to ['X', 'Y', 'Width', 'Height'] + distth: Union(float, array_like), optional + Maximum tolerable distance. Pairs exceeding this threshold are marked 'do-not-pair'. + If a list of thresholds is given, multiple accumulators are returned. + """ # pylint: disable=too-many-locals if distfields is None: distfields = ["X", "Y", "Width", "Height"] From 334130d18e3b7870bd32343760348ab105485105 Mon Sep 17 00:00:00 2001 From: Justin Ruan Date: Sun, 17 Dec 2023 08:15:02 +0000 Subject: [PATCH 04/11] add unit test for hota metrics --- motmetrics/tests/test_metrics.py | 46 ++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/motmetrics/tests/test_metrics.py b/motmetrics/tests/test_metrics.py index 504fd28..891154e 100644 --- a/motmetrics/tests/test_metrics.py +++ b/motmetrics/tests/test_metrics.py @@ -537,3 +537,49 @@ def my_motp(df: mm.metrics.DataFrameMap): ) print(summary) + + +def test_hota(): + TUD_golden_ans = { # From TrackEval + "TUD-Campus": {"hota": 0.3913974378451139, "deta": 0.418047030142763, "assa": 0.36912068120832836}, + "TUD-Stadtmitte": {"hota": 0.3978490169927877, "deta": 0.3922675723693166, "assa": 0.4088407518112996} + } + + DATA_DIR = "motmetrics/data" + + def compute_motchallenge(dname): + df_gt = mm.io.loadtxt(os.path.join(dname, "gt.txt")) + df_test = mm.io.loadtxt(os.path.join(dname, "test.txt")) + th_list = np.arange(0.05, 0.99, 0.05) + res_list = mm.utils.compare_to_groundtruth_reweighting(df_gt, df_test, "iou", distth=th_list) + return res_list + + accs = [compute_motchallenge(os.path.join(DATA_DIR, d)) for d in TUD_golden_ans.keys()] + mh = mm.metrics.create() + + for dataset_idx, dname in enumerate(TUD_golden_ans.keys()): + deta = [] + assa = [] + hota = [] + for alpha_idx in range(len(accs[dataset_idx])): + summary = mh.compute_many( + [accs[dataset_idx][alpha_idx]], + metrics=[ + "deta_alpha", + "assa_alpha", + "hota_alpha", + ], + names=[dname], + generate_overall=False, + ) + deta.append(float(summary["deta_alpha"].iloc[0])) + assa.append(float(summary["assa_alpha"].iloc[0])) + hota.append(float(summary["hota_alpha"].iloc[0])) + + deta = sum(deta) / len(deta) + assa = sum(assa) / len(assa) + hota = sum(hota) / len(hota) + + assert deta == approx(TUD_golden_ans[dname]["deta"]) + assert assa == approx(TUD_golden_ans[dname]["assa"]) + assert hota == approx(TUD_golden_ans[dname]["hota"]) From 1426f558f1c06c853334911ff4f01532ad44a249 Mon Sep 17 00:00:00 2001 From: Justin Ruan Date: Sun, 18 Feb 2024 15:46:03 +0000 Subject: [PATCH 05/11] add overall computation for hota --- motmetrics/metrics.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/motmetrics/metrics.py b/motmetrics/metrics.py index efc4543..01081e4 100644 --- a/motmetrics/metrics.py +++ b/motmetrics/metrics.py @@ -9,14 +9,12 @@ # pylint: disable=redefined-outer-name -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function -from collections import OrderedDict import inspect import logging import time +from collections import OrderedDict import numpy as np import pandas as pd @@ -621,6 +619,13 @@ def deta_alpha(df, num_detections, num_objects, num_false_positives): return math_util.quiet_divide(num_detections, max(1, num_objects + num_false_positives)) +def deta_alpha_m(partials): + res = 0 + for v in partials: + res += v["deta_alpha"] + return math_util.quiet_divide(res, len(partials)) + + def assa_alpha(df, num_detections, num_gt_ids, num_dt_ids): r"""AssA under specific threshold $\alpha$ Source: https://github.com/JonathonLuiten/TrackEval/blob/12c8791b303e0a0b50f753af204249e622d0281a/trackeval/metrics/hota.py#L107-L108 @@ -641,12 +646,26 @@ def assa_alpha(df, num_detections, num_gt_ids, num_dt_ids): return math_util.quiet_divide((ass_a * match_count_array).sum(), max(1, num_detections)) +def assa_alpha_m(partials): + res = 0 + for v in partials: + res += v["assa_alpha"] + return math_util.quiet_divide(res, len(partials)) + + def hota_alpha(df, deta_alpha, assa_alpha): r"""HOTA under specific threshold $\alpha$""" del df return (deta_alpha * assa_alpha) ** 0.5 +def hota_alpha_m(partials): + res = 0 + for v in partials: + res += v["hota_alpha"] + return math_util.quiet_divide(res, len(partials)) + + class DataFrameMap: # pylint: disable=too-few-public-methods def __init__(self, full, raw, noraw, extra): self.full = full From 553125123d1ce18b89cef8307bc4c8d1eee7ecb9 Mon Sep 17 00:00:00 2001 From: Justin Ruan Date: Sun, 18 Feb 2024 17:05:13 +0000 Subject: [PATCH 06/11] fix invalid denominator --- motmetrics/utils.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/motmetrics/utils.py b/motmetrics/utils.py index c47c4c8..74ff37d 100644 --- a/motmetrics/utils.py +++ b/motmetrics/utils.py @@ -7,9 +7,7 @@ """Functions for populating event accumulators.""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function +from __future__ import absolute_import, division, print_function import numpy as np @@ -52,7 +50,7 @@ def compute_global_aligment_score( gt_id_count[gt_ids] += 1 tracker_id_count[0, dt_ids] += 1 global_alignment_score = potential_matches_count / ( - gt_id_count + tracker_id_count - potential_matches_count + np.maximum(1, gt_id_count + tracker_id_count - potential_matches_count) ) return global_alignment_score From 7749b9f4ea350febe3564f1b467a357e0b18ea92 Mon Sep 17 00:00:00 2001 From: Justin Ruan Date: Sun, 18 Feb 2024 17:05:48 +0000 Subject: [PATCH 07/11] fix unconinious id when computing assa --- motmetrics/metrics.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/motmetrics/metrics.py b/motmetrics/metrics.py index 01081e4..eeaef77 100644 --- a/motmetrics/metrics.py +++ b/motmetrics/metrics.py @@ -630,9 +630,12 @@ def assa_alpha(df, num_detections, num_gt_ids, num_dt_ids): r"""AssA under specific threshold $\alpha$ Source: https://github.com/JonathonLuiten/TrackEval/blob/12c8791b303e0a0b50f753af204249e622d0281a/trackeval/metrics/hota.py#L107-L108 """ - match_count_array = np.zeros((num_gt_ids, num_dt_ids)) - gt_id_counts = np.zeros((num_gt_ids, 1)) - tracker_id_counts = np.zeros((1, num_dt_ids)) + max_gt_ids = int(df.noraw.OId.max()) + max_dt_ids = int(df.noraw.HId.max()) + + match_count_array = np.zeros((max_gt_ids, max_dt_ids)) + gt_id_counts = np.zeros((max_gt_ids, 1)) + tracker_id_counts = np.zeros((1, max_dt_ids)) for idx in range(len(df.noraw)): oid, hid = df.noraw.iloc[idx, 1], df.noraw.iloc[idx, 2] if df.noraw.iloc[idx, 0] in ["SWITCH", "MATCH"]: From 98a571f44284e22284452f21b8cd744638d6820a Mon Sep 17 00:00:00 2001 From: Justin Ruan Date: Wed, 22 May 2024 06:58:55 +0000 Subject: [PATCH 08/11] update README for hota metrics --- Readme.md | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/Readme.md b/Readme.md index 4de0b9f..52121f6 100644 --- a/Readme.md +++ b/Readme.md @@ -71,6 +71,9 @@ print(mh.list_metrics_markdown()) | pred_frequencies | `pd.Series` Total number of occurrences of individual predictions over all frames. | | track_ratios | `pd.Series` Ratio of assigned to total appearance count per unique object id. | | id_global_assignment | `dict` ID measures: Global min-cost assignment for ID measures. | +| deta_alpha | HOTA: Detection Accuracy (DetA) for a given threshold. | +| assa_alpha | HOTA: Association Accuracy (AssA) for a given threshold. | +| hota_alpha | HOTA: Higher Order Tracking Accuracy (HOTA) for a given threshold. | @@ -362,6 +365,57 @@ OVERALL 80.0% 80.0% 80.0% 80.0% 80.0% 4 2 2 0 2 2 1 1 50.0% 0.275 """ ``` +#### [Underdeveloped] Computing HOTA metrics + +Computing HOTA metrics is also possible. However, it cannot be used with the `Accumulator` class directly, as HOTA requires to computing a reweighting matrix from all the frames at the beginning. Here is an example of how to use it: + +```python +import numpy as np +import motmetrics as mm + + +def compute_motchallenge(dir_name): + # `gt.txt` and `test.txt` should be prepared in MOT15 format + df_gt = mm.io.loadtxt(os.path.join(dir_name, "gt.txt")) + df_test = mm.io.loadtxt(os.path.join(dir_name, "test.txt")) + # Require different thresholds for matching + th_list = np.arange(0.05, 0.99, 0.05) + res_list = mm.utils.compare_to_groundtruth_reweighting(df_gt, df_test, "iou", distth=th_list) + return res_list + +# `data_dir` is the directory containing the gt.txt and test.txt files +acc = compute_motchallenge("data_dir") +mh = mm.metrics.create() + +deta, assa, hota = [], [], [] +for alpha_idx in range(len(acc)): # Loop over different alpha values + summary = mh.compute_many( + [acc[alpha_idx]], + metrics=[ + "deta_alpha", + "assa_alpha", + "hota_alpha", + ], + ) + deta.append(float(summary["deta_alpha"].iloc[0])) + assa.append(float(summary["assa_alpha"].iloc[0])) + hota.append(float(summary["hota_alpha"].iloc[0])) + +deta = sum(deta) / len(deta) +assa = sum(assa) / len(assa) +hota = sum(hota) / len(hota) +print(f"{dname}: HOTA: {hota * 100:.3f} | AssA: {assa * 100:.3f} | DetA: {deta * 100:.3f}") + +""" +# motmetrics/data/TUD-Campus +TUD-Campus: HOTA: 39.140 | AssA: 36.912 | DetA: 41.805 +# motmetrics/data/TUD-Stadtmitte +TUD-Stadtmitte: HOTA: 39.785 | AssA: 40.884 | DetA: 39.227 +""" +``` + +**Merging this for-loop into a single function is a work in progress.** + ### Computing distances Up until this point we assumed the pairwise object/hypothesis distances to be known. Usually this is not the case. You are mostly given either rectangles or points (centroids) of related objects. To compute a distance matrix from them you can use `motmetrics.distance` module as shown below. From e6959a76834e8b6da5dbea0f342f85805e5f8355 Mon Sep 17 00:00:00 2001 From: Justin Ruan Date: Wed, 22 May 2024 07:47:05 +0000 Subject: [PATCH 09/11] update README for hota metrics without for loop --- Readme.md | 42 +++++++++++++++++++----------------------- 1 file changed, 19 insertions(+), 23 deletions(-) diff --git a/Readme.md b/Readme.md index 52121f6..99dcc84 100644 --- a/Readme.md +++ b/Readme.md @@ -387,35 +387,31 @@ def compute_motchallenge(dir_name): acc = compute_motchallenge("data_dir") mh = mm.metrics.create() -deta, assa, hota = [], [], [] -for alpha_idx in range(len(acc)): # Loop over different alpha values - summary = mh.compute_many( - [acc[alpha_idx]], - metrics=[ - "deta_alpha", - "assa_alpha", - "hota_alpha", - ], - ) - deta.append(float(summary["deta_alpha"].iloc[0])) - assa.append(float(summary["assa_alpha"].iloc[0])) - hota.append(float(summary["hota_alpha"].iloc[0])) - -deta = sum(deta) / len(deta) -assa = sum(assa) / len(assa) -hota = sum(hota) / len(hota) -print(f"{dname}: HOTA: {hota * 100:.3f} | AssA: {assa * 100:.3f} | DetA: {deta * 100:.3f}") - +summary = mh.compute_many( + accs[dataset_idx], + metrics=[ + "deta_alpha", + "assa_alpha", + "hota_alpha", + ], + generate_overall=True, # `Overall` is the average we need only +) +strsummary = mm.io.render_summary( + summary.iloc[[-1], :], # Use list to preserve `DataFrame` type + formatters=mh.formatters, + namemap={"hota_alpha": "HOTA", "assa_alpha": "ASSA", "deta_alpha": "DETA"}, +) +print(strsummary) """ # motmetrics/data/TUD-Campus -TUD-Campus: HOTA: 39.140 | AssA: 36.912 | DetA: 41.805 + DETA ASSA HOTA +OVERALL 41.8% 36.9% 39.1% # motmetrics/data/TUD-Stadtmitte -TUD-Stadtmitte: HOTA: 39.785 | AssA: 40.884 | DetA: 39.227 + DETA ASSA HOTA +OVERALL 39.2% 40.9% 39.8% """ ``` -**Merging this for-loop into a single function is a work in progress.** - ### Computing distances Up until this point we assumed the pairwise object/hypothesis distances to be known. Usually this is not the case. You are mostly given either rectangles or points (centroids) of related objects. To compute a distance matrix from them you can use `motmetrics.distance` module as shown below. From 013562fb76b9fb23c7123cd0c474d49dc8ab0398 Mon Sep 17 00:00:00 2001 From: Justin Ruan Date: Wed, 22 May 2024 08:06:03 +0000 Subject: [PATCH 10/11] fix typo --- Readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Readme.md b/Readme.md index 99dcc84..da7e299 100644 --- a/Readme.md +++ b/Readme.md @@ -388,7 +388,7 @@ acc = compute_motchallenge("data_dir") mh = mm.metrics.create() summary = mh.compute_many( - accs[dataset_idx], + acc, metrics=[ "deta_alpha", "assa_alpha", From 926217af5b1a1d33fd34e8db0d40f7cf9e0f6197 Mon Sep 17 00:00:00 2001 From: Justin Ruan Date: Fri, 24 May 2024 11:52:22 +0000 Subject: [PATCH 11/11] add detail instruction for hota --- Readme.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Readme.md b/Readme.md index da7e299..43f9a08 100644 --- a/Readme.md +++ b/Readme.md @@ -370,6 +370,7 @@ OVERALL 80.0% 80.0% 80.0% 80.0% 80.0% 4 2 2 0 2 2 1 1 50.0% 0.275 Computing HOTA metrics is also possible. However, it cannot be used with the `Accumulator` class directly, as HOTA requires to computing a reweighting matrix from all the frames at the beginning. Here is an example of how to use it: ```python +import os import numpy as np import motmetrics as mm @@ -403,10 +404,10 @@ strsummary = mm.io.render_summary( ) print(strsummary) """ -# motmetrics/data/TUD-Campus +# data_dir=motmetrics/data/TUD-Campus DETA ASSA HOTA OVERALL 41.8% 36.9% 39.1% -# motmetrics/data/TUD-Stadtmitte +# data_dir=motmetrics/data/TUD-Stadtmitte DETA ASSA HOTA OVERALL 39.2% 40.9% 39.8% """