From 3b74ade97cdb96f23628076e185b24916eedbec8 Mon Sep 17 00:00:00 2001 From: "mikel.brostrom" Date: Wed, 6 Mar 2024 21:29:47 +0100 Subject: [PATCH 01/17] trajectory plotting for deepocsort --- boxmot/trackers/deepocsort/deep_ocsort.py | 4 ++- tracking/track.py | 39 +++++++++++++++-------- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/boxmot/trackers/deepocsort/deep_ocsort.py b/boxmot/trackers/deepocsort/deep_ocsort.py index 8820a7ea8b..0fb4478fcd 100644 --- a/boxmot/trackers/deepocsort/deep_ocsort.py +++ b/boxmot/trackers/deepocsort/deep_ocsort.py @@ -1,6 +1,7 @@ # Mikel Broström 🔥 Yolo Tracking 🧾 AGPL-3.0 license import numpy as np +from collections import deque from boxmot.appearance.reid_multibackend import ReIDDetectMultiBackend from boxmot.motion.cmc import get_cmc_method @@ -183,11 +184,12 @@ def __init__(self, det, delta_t=3, emb=None, alpha=0, new_kf=False): # Used for OCR self.last_observation = np.array([-1, -1, -1, -1, -1]) # placeholder # Used to output track after min_hits reached - self.history_observations = [] + self.features = deque([], maxlen=50) # Used for velocity self.observations = dict() self.velocity = None self.delta_t = delta_t + self.history_observations = deque([], maxlen=50) self.emb = emb diff --git a/tracking/track.py b/tracking/track.py index 715e304228..33fa14c091 100644 --- a/tracking/track.py +++ b/tracking/track.py @@ -1,6 +1,8 @@ # Mikel Broström 🔥 Yolo Tracking 🧾 AGPL-3.0 license import argparse +import cv2 +import numpy as np from functools import partial from pathlib import Path @@ -16,6 +18,7 @@ __tr.check_packages(('ultralytics @ git+https://github.com/mikel-brostrom/ultralytics.git', )) # install from ultralytics import YOLO +from ultralytics.utils.plotting import Annotator, colors from ultralytics.data.utils import VID_FORMATS from ultralytics.utils.plotting import save_one_box @@ -95,23 +98,31 @@ def run(args): # store custom args in predictor yolo.predictor.custom_args = args - for frame_idx, r in enumerate(results): + for idx, r in enumerate(results): + + annotator = Annotator(r.orig_img) if r.boxes.data.shape[1] == 7: - if args.save_id_crops: - for d in r.boxes: - print('args.save_id_crops', d.data) - save_one_box( - d.xyxy, - r.orig_img.copy(), - file=( - yolo.predictor.save_dir / 'crops' / - str(int(d.cls.cpu().numpy().item())) / - str(int(d.id.cpu().numpy().item())) / f'{frame_idx}.jpg' - ), - BGR=True - ) + annotator = Annotator(r.orig_img) + for inn, b in enumerate(r.boxes): + box = b.xyxy[0] + c = b.cls + i = b.id + annotator.box_label(box, str(i), color=colors(int(i))) + + img = annotator.result() + + a = yolo.predictor.trackers[0].active_tracks[inn] + for o in a.history_observations: + thickness = int(np.sqrt(float (idx + 1)) * 2) + cv2.circle(img, (int((o[0] + o[2]) / 2), int((o[1] + o[3]) / 2)), 2, color=colors(int(i)), thickness=thickness) + + cv2.imshow('BoxMOT', img) + if cv2.waitKey(1) & 0xFF == ord(' '): + break + + def parse_opt(): From 4c64fe71a7d632bbf9e615d332b25f9791c0af1c Mon Sep 17 00:00:00 2001 From: "mikel.brostrom" Date: Wed, 6 Mar 2024 21:35:47 +0100 Subject: [PATCH 02/17] deque instead of list --- boxmot/trackers/ocsort/ocsort.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/boxmot/trackers/ocsort/ocsort.py b/boxmot/trackers/ocsort/ocsort.py index 30ab557ceb..8c089ffb5a 100644 --- a/boxmot/trackers/ocsort/ocsort.py +++ b/boxmot/trackers/ocsort/ocsort.py @@ -4,6 +4,8 @@ This script is adopted from the SORT script by Alex Bewley alex@bewley.ai """ import numpy as np +from collections import deque + from boxmot.motion.kalman_filters.ocsort_kf import KalmanFilter from boxmot.utils.association import associate, linear_assignment @@ -125,7 +127,7 @@ def __init__(self, bbox, cls, det_ind, delta_t=3): """ self.last_observation = np.array([-1, -1, -1, -1, -1]) # placeholder self.observations = dict() - self.history_observations = [] + self.history_observations = deque([], maxlen=50) self.velocity = None self.delta_t = delta_t @@ -160,7 +162,7 @@ def update(self, bbox, cls, det_ind): self.history_observations.append(bbox) self.time_since_update = 0 - self.history = [] + self.history = deque([], maxlen=50) self.hits += 1 self.hit_streak += 1 self.kf.update(convert_bbox_to_z(bbox)) From 8bdf01711ad51d767fbdabfaad14a8b7f2ed6676 Mon Sep 17 00:00:00 2001 From: "mikel.brostrom" Date: Wed, 6 Mar 2024 21:36:00 +0100 Subject: [PATCH 03/17] added trajectory plotting --- tracking/track.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tracking/track.py b/tracking/track.py index 33fa14c091..468563362f 100644 --- a/tracking/track.py +++ b/tracking/track.py @@ -115,7 +115,7 @@ def run(args): a = yolo.predictor.trackers[0].active_tracks[inn] for o in a.history_observations: - thickness = int(np.sqrt(float (idx + 1)) * 2) + thickness = int(np.sqrt(float (inn + 1)) * 2) cv2.circle(img, (int((o[0] + o[2]) / 2), int((o[1] + o[3]) / 2)), 2, color=colors(int(i)), thickness=thickness) cv2.imshow('BoxMOT', img) From e828894bca557cf406e4a5a8916b7ed068f6189f Mon Sep 17 00:00:00 2001 From: "mikel.brostrom" Date: Wed, 6 Mar 2024 21:36:53 +0100 Subject: [PATCH 04/17] history list now queue --- boxmot/trackers/deepocsort/deep_ocsort.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/boxmot/trackers/deepocsort/deep_ocsort.py b/boxmot/trackers/deepocsort/deep_ocsort.py index 0fb4478fcd..873d9bd676 100644 --- a/boxmot/trackers/deepocsort/deep_ocsort.py +++ b/boxmot/trackers/deepocsort/deep_ocsort.py @@ -171,7 +171,7 @@ def __init__(self, det, delta_t=3, emb=None, alpha=0, new_kf=False): self.time_since_update = 0 self.id = KalmanBoxTracker.count KalmanBoxTracker.count += 1 - self.history = [] + self.history = deque([], maxlen=50) self.hits = 0 self.hit_streak = 0 self.age = 0 From 3a24b648d998ca648b7222463c9939b504929074 Mon Sep 17 00:00:00 2001 From: "mikel.brostrom" Date: Thu, 7 Mar 2024 08:09:49 +0100 Subject: [PATCH 05/17] moved trajectory plotting to basetracker --- boxmot/trackers/basetracker.py | 37 ++++++++++++++++++++++++++++++++++ tracking/track.py | 5 +---- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/boxmot/trackers/basetracker.py b/boxmot/trackers/basetracker.py index be834e815b..32a3ef3573 100644 --- a/boxmot/trackers/basetracker.py +++ b/boxmot/trackers/basetracker.py @@ -1,3 +1,9 @@ +import numpy as np +import cv2 as cv +import hashlib +import colorsys + + class BaseTracker(object): def __init__(self, det_thresh=0.3, max_age=30, min_hits=3, iou_threshold=0.3): self.det_thresh = det_thresh @@ -10,3 +16,34 @@ def __init__(self, det_thresh=0.3, max_age=30, min_hits=3, iou_threshold=0.3): def update(self, dets, img, embs=None): raise NotImplementedError("The update method needs to be implemented by the subclass.") + + def id_to_color(self, id, saturation=0.75, value=0.95): + # Hash the ID to get a consistent unique value + hash_object = hashlib.sha256(str(id).encode()) + hash_digest = hash_object.hexdigest() + + # Convert the first few characters of the hash to an integer + # and map it to a value between 0 and 1 for the hue + hue = int(hash_digest[:8], 16) / 0xffffffff + + # Convert HSV to RGB + rgb = colorsys.hsv_to_rgb(hue, saturation, value) + + # Convert RGB from 0-1 range to 0-255 range and format as hexadecimal + rgb_255 = tuple(int(component * 255) for component in rgb) + hex_color = '#%02x%02x%02x' % rgb_255 + # Strip the '#' character and convert the string to RGB integers + rgb = tuple(int(hex_color.strip('#')[i:i+2], 16) for i in (0, 2, 4)) + + # Convert RGB to BGR for OpenCV + bgr = rgb[::-1] + + return bgr + + def plot_trajectory(self, img, id): + for a in self.active_tracks: + for i, o in enumerate(a.history_observations): + thickness = int(np.sqrt(float (i + 1)) * 2) + cv.circle(img, (int((o[0] + o[2]) / 2), int((o[1] + o[3]) / 2)), 2, color=self.id_to_color(int(id)), thickness=thickness) + + diff --git a/tracking/track.py b/tracking/track.py index 468563362f..1a3a8c3125 100644 --- a/tracking/track.py +++ b/tracking/track.py @@ -113,10 +113,7 @@ def run(args): img = annotator.result() - a = yolo.predictor.trackers[0].active_tracks[inn] - for o in a.history_observations: - thickness = int(np.sqrt(float (inn + 1)) * 2) - cv2.circle(img, (int((o[0] + o[2]) / 2), int((o[1] + o[3]) / 2)), 2, color=colors(int(i)), thickness=thickness) + a = yolo.predictor.trackers[0].plot_trajectory(img, i) cv2.imshow('BoxMOT', img) if cv2.waitKey(1) & 0xFF == ord(' '): From db90154306a886a12f67cfe2bdf379f8c6520d24 Mon Sep 17 00:00:00 2001 From: "mikel.brostrom" Date: Thu, 7 Mar 2024 08:13:42 +0100 Subject: [PATCH 06/17] coditional trajectory n bboxes display --- tracking/track.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tracking/track.py b/tracking/track.py index 1a3a8c3125..5a6c05e935 100644 --- a/tracking/track.py +++ b/tracking/track.py @@ -115,9 +115,10 @@ def run(args): a = yolo.predictor.trackers[0].plot_trajectory(img, i) - cv2.imshow('BoxMOT', img) - if cv2.waitKey(1) & 0xFF == ord(' '): - break + if args.show is True: + cv2.imshow('BoxMOT', img) + if cv2.waitKey(1) & 0xFF == ord(' '): + break From a1fd2ffdc83dadfe169a2fc018fe8f6bd6539b3a Mon Sep 17 00:00:00 2001 From: "mikel.brostrom" Date: Thu, 7 Mar 2024 16:22:02 +0100 Subject: [PATCH 07/17] use boxmot plotting functionality --- boxmot/trackers/basetracker.py | 39 +++++++++++++++++++++++++++++----- tracking/track.py | 27 ++++++----------------- 2 files changed, 40 insertions(+), 26 deletions(-) diff --git a/boxmot/trackers/basetracker.py b/boxmot/trackers/basetracker.py index 32a3ef3573..cdc76344eb 100644 --- a/boxmot/trackers/basetracker.py +++ b/boxmot/trackers/basetracker.py @@ -40,10 +40,39 @@ def id_to_color(self, id, saturation=0.75, value=0.95): return bgr - def plot_trajectory(self, img, id): + def plot_trajectory(self, img): + + thickness = 2 + fontscale = 0.5 + for a in self.active_tracks: - for i, o in enumerate(a.history_observations): - thickness = int(np.sqrt(float (i + 1)) * 2) - cv.circle(img, (int((o[0] + o[2]) / 2), int((o[1] + o[3]) / 2)), 2, color=self.id_to_color(int(id)), thickness=thickness) + if a.history_observations: + o = a.history_observations[-1] + img = cv.rectangle( + img, + (int(o[0]), int(o[1])), + (int(o[2]), int(o[3])), + self.id_to_color(a.id), + thickness + ) + img = cv.putText( + img, + f'id: {a.id}, conf: {a.conf:.2f}, c: {a.cls}', + (int(o[0]), int(o[1]) - 10), + cv.FONT_HERSHEY_SIMPLEX, + fontscale, + self.id_to_color(a.id), + thickness + ) + for e, o in enumerate(a.history_observations): + thickness = int(np.sqrt(float (e + 1)) * 1.2) + cv.circle( + img, + (int((o[0] + o[2]) / 2), + int((o[1] + o[3]) / 2)), + 2, + color=self.id_to_color(int(a.id)), + thickness=thickness + ) + return img - diff --git a/tracking/track.py b/tracking/track.py index 5a6c05e935..024b7723e3 100644 --- a/tracking/track.py +++ b/tracking/track.py @@ -66,7 +66,7 @@ def run(args): conf=args.conf, iou=args.iou, agnostic_nms=args.agnostic_nms, - show=args.show, + show=True, stream=True, device=args.device, show_conf=args.show_conf, @@ -100,27 +100,12 @@ def run(args): for idx, r in enumerate(results): - annotator = Annotator(r.orig_img) + img = yolo.predictor.trackers[0].plot_trajectory(r.orig_img) - if r.boxes.data.shape[1] == 7: - - annotator = Annotator(r.orig_img) - for inn, b in enumerate(r.boxes): - box = b.xyxy[0] - c = b.cls - i = b.id - annotator.box_label(box, str(i), color=colors(int(i))) - - img = annotator.result() - - a = yolo.predictor.trackers[0].plot_trajectory(img, i) - - if args.show is True: - cv2.imshow('BoxMOT', img) - if cv2.waitKey(1) & 0xFF == ord(' '): - break - - + if args.show is True: + cv2.imshow('BoxMOT', img) + if cv2.waitKey(1) & 0xFF == ord(' '): + break def parse_opt(): From f06f31d58167dac954e2322b009d0c9c354828ab Mon Sep 17 00:00:00 2001 From: "mikel.brostrom" Date: Thu, 7 Mar 2024 16:23:42 +0100 Subject: [PATCH 08/17] show boxmot results in img --- tracking/track.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tracking/track.py b/tracking/track.py index 024b7723e3..ea4e3f2780 100644 --- a/tracking/track.py +++ b/tracking/track.py @@ -66,7 +66,7 @@ def run(args): conf=args.conf, iou=args.iou, agnostic_nms=args.agnostic_nms, - show=True, + show=False, stream=True, device=args.device, show_conf=args.show_conf, From 12e213eaef4862f8e3442371d7eecec26b4df4bc Mon Sep 17 00:00:00 2001 From: "mikel.brostrom" Date: Thu, 7 Mar 2024 17:51:19 +0100 Subject: [PATCH 09/17] fix trajectory plotting for botsort --- boxmot/trackers/basetracker.py | 26 +++++++++--------- boxmot/trackers/botsort/bot_sort.py | 41 ++++++++++++++++------------- 2 files changed, 36 insertions(+), 31 deletions(-) diff --git a/boxmot/trackers/basetracker.py b/boxmot/trackers/basetracker.py index cdc76344eb..52521827e1 100644 --- a/boxmot/trackers/basetracker.py +++ b/boxmot/trackers/basetracker.py @@ -44,9 +44,10 @@ def plot_trajectory(self, img): thickness = 2 fontscale = 0.5 - for a in self.active_tracks: + if a.history_observations: + o = a.history_observations[-1] img = cv.rectangle( img, @@ -57,22 +58,23 @@ def plot_trajectory(self, img): ) img = cv.putText( img, - f'id: {a.id}, conf: {a.conf:.2f}, c: {a.cls}', + f'id: {int(a.id)}, conf: {a.conf:.2f}, c: {int(a.cls)}', (int(o[0]), int(o[1]) - 10), cv.FONT_HERSHEY_SIMPLEX, fontscale, self.id_to_color(a.id), thickness ) - for e, o in enumerate(a.history_observations): - thickness = int(np.sqrt(float (e + 1)) * 1.2) - cv.circle( - img, - (int((o[0] + o[2]) / 2), - int((o[1] + o[3]) / 2)), - 2, - color=self.id_to_color(int(a.id)), - thickness=thickness - ) + if len(a.history_observations) > 3: + for e, o in enumerate(a.history_observations): + trajectory_thickness = int(np.sqrt(float (e + 1)) * 1.2) + cv.circle( + img, + (int((o[0] + o[2]) / 2), + int((o[1] + o[3]) / 2)), + 2, + color=self.id_to_color(int(a.id)), + thickness=trajectory_thickness + ) return img diff --git a/boxmot/trackers/botsort/bot_sort.py b/boxmot/trackers/botsort/bot_sort.py index a343bfba52..2f0457c6ec 100644 --- a/boxmot/trackers/botsort/bot_sort.py +++ b/boxmot/trackers/botsort/bot_sort.py @@ -21,14 +21,15 @@ class STrack(BaseTrack): def __init__(self, det, feat=None, feat_history=50): # wait activate self.xywh = xyxy2xywh(det[0:4]) # (x1, y1, x2, y2) --> (xc, yc, w, h) - self.score = det[4] + self.conf = det[4] self.cls = det[5] self.det_ind = det[6] self.kalman_filter = None self.mean, self.covariance = None, None self.is_activated = False self.cls_hist = [] # (cls id, freq) - self.update_cls(self.cls, self.score) + self.update_cls(self.cls, self.conf) + self.history_observations = deque([], maxlen=50) self.tracklet_len = 0 @@ -49,23 +50,23 @@ def update_features(self, feat): self.features.append(feat) self.smooth_feat /= np.linalg.norm(self.smooth_feat) - def update_cls(self, cls, score): + def update_cls(self, cls, conf): if len(self.cls_hist) > 0: max_freq = 0 found = False for c in self.cls_hist: if cls == c[0]: - c[1] += score + c[1] += conf found = True if c[1] > max_freq: max_freq = c[1] self.cls = c[0] if not found: - self.cls_hist.append([cls, score]) + self.cls_hist.append([cls, conf]) self.cls = cls else: - self.cls_hist.append([cls, score]) + self.cls_hist.append([cls, conf]) self.cls = cls def predict(self): @@ -138,11 +139,11 @@ def re_activate(self, new_track, frame_id, new_id=False): self.frame_id = frame_id if new_id: self.id = self.next_id() - self.score = new_track.score + self.conf = new_track.conf self.cls = new_track.cls self.det_ind = new_track.det_ind - self.update_cls(new_track.cls, new_track.score) + self.update_cls(new_track.cls, new_track.conf) def update(self, new_track, frame_id): """ @@ -155,6 +156,8 @@ def update(self, new_track, frame_id): self.frame_id = frame_id self.tracklet_len += 1 + self.history_observations.append(self.xyxy) + self.mean, self.covariance = self.kalman_filter.update( self.mean, self.covariance, new_track.xywh ) @@ -165,10 +168,10 @@ def update(self, new_track, frame_id): self.state = TrackState.Tracked self.is_activated = True - self.score = new_track.score + self.conf = new_track.conf self.cls = new_track.cls self.det_ind = new_track.det_ind - self.update_cls(new_track.cls, new_track.score) + self.update_cls(new_track.cls, new_track.conf) @property def xyxy(self): @@ -281,17 +284,17 @@ def update(self, dets, img, embs=None): else: detections = [] - """ Add newly detected tracklets to tracked_stracks""" + """ Add newly detected tracklets to active_tracks""" unconfirmed = [] - tracked_stracks = [] # type: list[STrack] + active_tracks = [] # type: list[STrack] for track in self.active_tracks: if not track.is_activated: unconfirmed.append(track) else: - tracked_stracks.append(track) + active_tracks.append(track) - """ Step 2: First association, with high score detection boxes""" - strack_pool = joint_stracks(tracked_stracks, self.lost_stracks) + """ Step 2: First association, with high conf detection boxes""" + strack_pool = joint_stracks(active_tracks, self.lost_stracks) # Predict the current location with KF STrack.multi_predict(strack_pool) @@ -301,7 +304,7 @@ def update(self, dets, img, embs=None): STrack.multi_gmc(strack_pool, warp) STrack.multi_gmc(unconfirmed, warp) - # Associate with high score detection boxes + # Associate with high conf detection boxes ious_dists = iou_distance(strack_pool, detections) ious_dists_mask = ious_dists > self.proximity_thresh if self.fuse_first_associate: @@ -329,7 +332,7 @@ def update(self, dets, img, embs=None): track.re_activate(det, self.frame_count, new_id=False) refind_stracks.append(track) - """ Step 3: Second association, with low score detection boxes""" + """ Step 3: Second association, with low conf detection boxes""" if len(dets_second) > 0: """Detections""" detections_second = [STrack(dets_second) for dets_second in dets_second] @@ -386,7 +389,7 @@ def update(self, dets, img, embs=None): """ Step 4: Init new stracks""" for inew in u_detection: track = detections[inew] - if track.score < self.new_track_thresh: + if track.conf < self.new_track_thresh: continue track.activate(self.kalman_filter, self.frame_count) @@ -418,7 +421,7 @@ def update(self, dets, img, embs=None): output = [] output.extend(t.xyxy) output.append(t.id) - output.append(t.score) + output.append(t.conf) output.append(t.cls) output.append(t.det_ind) outputs.append(output) From 2df5b53643f6a9dcdc2600157fc6e9d636ee1fef Mon Sep 17 00:00:00 2001 From: "mikel.brostrom" Date: Thu, 7 Mar 2024 18:10:08 +0100 Subject: [PATCH 10/17] fix bytetrack plotting --- boxmot/trackers/basetracker.py | 1 + boxmot/trackers/botsort/bot_sort.py | 3 +- boxmot/trackers/bytetrack/basetrack.py | 2 +- boxmot/trackers/bytetrack/byte_tracker.py | 40 +++++++++++------------ 4 files changed, 23 insertions(+), 23 deletions(-) diff --git a/boxmot/trackers/basetracker.py b/boxmot/trackers/basetracker.py index 52521827e1..0a3d7507f0 100644 --- a/boxmot/trackers/basetracker.py +++ b/boxmot/trackers/basetracker.py @@ -44,6 +44,7 @@ def plot_trajectory(self, img): thickness = 2 fontscale = 0.5 + for a in self.active_tracks: if a.history_observations: diff --git a/boxmot/trackers/botsort/bot_sort.py b/boxmot/trackers/botsort/bot_sort.py index 2f0457c6ec..34a83b73f7 100644 --- a/boxmot/trackers/botsort/bot_sort.py +++ b/boxmot/trackers/botsort/bot_sort.py @@ -1,8 +1,7 @@ # Mikel Broström 🔥 Yolo Tracking 🧾 AGPL-3.0 license -from collections import deque - import numpy as np +from collections import deque from boxmot.appearance.reid_multibackend import ReIDDetectMultiBackend from boxmot.motion.cmc.sof import SOF diff --git a/boxmot/trackers/bytetrack/basetrack.py b/boxmot/trackers/bytetrack/basetrack.py index d9a71477e8..45f5c15fa5 100644 --- a/boxmot/trackers/bytetrack/basetrack.py +++ b/boxmot/trackers/bytetrack/basetrack.py @@ -22,7 +22,7 @@ class BaseTrack(object): history = OrderedDict() features = [] curr_feature = None - score = 0 + conf = 0 start_frame = 0 frame_id = 0 time_since_update = 0 diff --git a/boxmot/trackers/bytetrack/byte_tracker.py b/boxmot/trackers/bytetrack/byte_tracker.py index b4ef8ed512..9ded8ce88b 100644 --- a/boxmot/trackers/bytetrack/byte_tracker.py +++ b/boxmot/trackers/bytetrack/byte_tracker.py @@ -1,6 +1,7 @@ # Mikel Broström 🔥 Yolo Tracking 🧾 AGPL-3.0 license import numpy as np +from collections import deque from boxmot.motion.kalman_filters.bytetrack_kf import KalmanFilter from boxmot.trackers.bytetrack.basetrack import BaseTrack, TrackState @@ -18,13 +19,14 @@ def __init__(self, det): self.xywh = xyxy2xywh(det[0:4]) # (x1, y1, x2, y2) --> (xc, yc, w, h) self.tlwh = xywh2tlwh(self.xywh) # (xc, yc, w, h) --> (t, l, w, h) self.xyah = tlwh2xyah(self.tlwh) - self.score = det[4] + self.conf = det[4] self.cls = det[5] self.det_ind = det[6] self.kalman_filter = None self.mean, self.covariance = None, None self.is_activated = False self.tracklet_len = 0 + self.history_observations = deque([], maxlen=50) def predict(self): mean_state = self.mean.copy() @@ -52,7 +54,7 @@ def multi_predict(stracks): def activate(self, kalman_filter, frame_id): """Start a new tracklet""" self.kalman_filter = kalman_filter - self.track_id = self.next_id() + self.id = self.next_id() self.mean, self.covariance = self.kalman_filter.initiate(self.xyah) self.tracklet_len = 0 @@ -72,8 +74,8 @@ def re_activate(self, new_track, frame_id, new_id=False): self.is_activated = True self.frame_id = frame_id if new_id: - self.track_id = self.next_id() - self.score = new_track.score + self.id = self.next_id() + self.conf = new_track.conf self.cls = new_track.cls self.det_ind = new_track.det_ind @@ -87,7 +89,7 @@ def update(self, new_track, frame_id): """ self.frame_id = frame_id self.tracklet_len += 1 - # self.cls = cls + self.history_observations.append(self.xyxy) self.mean, self.covariance = self.kalman_filter.update( self.mean, self.covariance, new_track.xyah @@ -95,7 +97,7 @@ def update(self, new_track, frame_id): self.state = TrackState.Tracked self.is_activated = True - self.score = new_track.score + self.conf = new_track.conf self.cls = new_track.cls self.det_ind = new_track.det_ind @@ -179,7 +181,7 @@ def update(self, dets, im=None, embs=None): else: tracked_stracks.append(track) - """ Step 2: First association, with high score detection boxes""" + """ Step 2: First association, with high conf detection boxes""" strack_pool = joint_stracks(tracked_stracks, self.lost_stracks) # Predict the current location with KF STrack.multi_predict(strack_pool) @@ -200,8 +202,8 @@ def update(self, dets, im=None, embs=None): track.re_activate(det, self.frame_count, new_id=False) refind_stracks.append(track) - """ Step 3: Second association, with low score detection boxes""" - # association the untrack to the low score detections + """ Step 3: Second association, with low conf detection boxes""" + # association the untrack to the low conf detections if len(dets_second) > 0: """Detections""" detections_second = [STrack(det_second) for det_second in dets_second] @@ -247,7 +249,7 @@ def update(self, dets, im=None, embs=None): """ Step 4: Init new stracks""" for inew in u_detection: track = detections[inew] - if track.score < self.det_thresh: + if track.conf < self.det_thresh: continue track.activate(self.kalman_filter, self.frame_count) activated_starcks.append(track) @@ -257,8 +259,6 @@ def update(self, dets, im=None, embs=None): track.mark_removed() removed_stracks.append(track) - # print('Ramained match {} s'.format(t4-t3)) - self.active_tracks = [ t for t in self.active_tracks if t.state == TrackState.Tracked ] @@ -271,14 +271,14 @@ def update(self, dets, im=None, embs=None): self.active_tracks, self.lost_stracks = remove_duplicate_stracks( self.active_tracks, self.lost_stracks ) - # get scores of lost tracks + # get confs of lost tracks output_stracks = [track for track in self.active_tracks if track.is_activated] outputs = [] for t in output_stracks: output = [] output.extend(t.xyxy) - output.append(t.track_id) - output.append(t.score) + output.append(t.id) + output.append(t.conf) output.append(t.cls) output.append(t.det_ind) outputs.append(output) @@ -286,17 +286,17 @@ def update(self, dets, im=None, embs=None): return outputs -# track_id, class_id, conf +# id, class_id, conf def joint_stracks(tlista, tlistb): exists = {} res = [] for t in tlista: - exists[t.track_id] = 1 + exists[t.id] = 1 res.append(t) for t in tlistb: - tid = t.track_id + tid = t.id if not exists.get(tid, 0): exists[tid] = 1 res.append(t) @@ -306,9 +306,9 @@ def joint_stracks(tlista, tlistb): def sub_stracks(tlista, tlistb): stracks = {} for t in tlista: - stracks[t.track_id] = t + stracks[t.id] = t for t in tlistb: - tid = t.track_id + tid = t.id if stracks.get(tid, 0): del stracks[tid] return list(stracks.values()) From 7206a0be19f2926d1ee2876535e59761273f51ff Mon Sep 17 00:00:00 2001 From: "mikel.brostrom" Date: Thu, 7 Mar 2024 18:10:21 +0100 Subject: [PATCH 11/17] fix bytetrack plotting --- boxmot/utils/matching.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/boxmot/utils/matching.py b/boxmot/utils/matching.py index 258180bf3c..0a8c16afe3 100644 --- a/boxmot/utils/matching.py +++ b/boxmot/utils/matching.py @@ -203,9 +203,9 @@ def fuse_iou(cost_matrix, tracks, detections): iou_dist = iou_distance(tracks, detections) iou_sim = 1 - iou_dist fuse_sim = reid_sim * (1 + iou_sim) / 2 - det_scores = np.array([det.score for det in detections]) - det_scores = np.expand_dims(det_scores, axis=0).repeat(cost_matrix.shape[0], axis=0) - # fuse_sim = fuse_sim * (1 + det_scores) / 2 + det_confs = np.array([det.conf for det in detections]) + det_confs = np.expand_dims(det_confs, axis=0).repeat(cost_matrix.shape[0], axis=0) + # fuse_sim = fuse_sim * (1 + det_confs) / 2 fuse_cost = 1 - fuse_sim return fuse_cost @@ -214,9 +214,9 @@ def fuse_score(cost_matrix, detections): if cost_matrix.size == 0: return cost_matrix iou_sim = 1 - cost_matrix - det_scores = np.array([det.score for det in detections]) - det_scores = np.expand_dims(det_scores, axis=0).repeat(cost_matrix.shape[0], axis=0) - fuse_sim = iou_sim * det_scores + det_confs = np.array([det.conf for det in detections]) + det_confs = np.expand_dims(det_confs, axis=0).repeat(cost_matrix.shape[0], axis=0) + fuse_sim = iou_sim * det_confs fuse_cost = 1 - fuse_sim return fuse_cost From a9c037e9ca88cfa26abf7b6087321bffe01a5867 Mon Sep 17 00:00:00 2001 From: "mikel.brostrom" Date: Thu, 7 Mar 2024 18:46:35 +0100 Subject: [PATCH 12/17] refactor plotting --- boxmot/trackers/basetracker.py | 71 ++++++++++++++++++++-------------- 1 file changed, 41 insertions(+), 30 deletions(-) diff --git a/boxmot/trackers/basetracker.py b/boxmot/trackers/basetracker.py index 0a3d7507f0..901077c040 100644 --- a/boxmot/trackers/basetracker.py +++ b/boxmot/trackers/basetracker.py @@ -40,42 +40,53 @@ def id_to_color(self, id, saturation=0.75, value=0.95): return bgr - def plot_trajectory(self, img): + def plot_box_on_img(self, img, box, conf, cls, id): thickness = 2 fontscale = 0.5 + img = cv.rectangle( + img, + (int(box[0]), int(box[1])), + (int(box[2]), int(box[3])), + self.id_to_color(id), + thickness + ) + img = cv.putText( + img, + f'id: {int(id)}, conf: {conf:.2f}, c: {int(cls)}', + (int(box[0]), int(box[1]) - 10), + cv.FONT_HERSHEY_SIMPLEX, + fontscale, + self.id_to_color(id), + thickness + ) + return img + + + def plot_trackers_trajectories(self, img, observations, id): + + for i, box in enumerate(observations): + trajectory_thickness = int(np.sqrt(float (i + 1)) * 1.2) + img = cv.circle( + img, + (int((box[0] + box[2]) / 2), + int((box[1] + box[3]) / 2)), + 2, + color=self.id_to_color(int(id)), + thickness=trajectory_thickness + ) + return img + + + def plot_trajectory(self, img): + for a in self.active_tracks: - if a.history_observations: + if len(a.history_observations) > 2: + box = a.history_observations[-1] + img = self.plot_box_on_img(img, box, a.conf, a.cls, a.id) + img = self.plot_trackers_trajectories(img, a.history_observations, a.id) - o = a.history_observations[-1] - img = cv.rectangle( - img, - (int(o[0]), int(o[1])), - (int(o[2]), int(o[3])), - self.id_to_color(a.id), - thickness - ) - img = cv.putText( - img, - f'id: {int(a.id)}, conf: {a.conf:.2f}, c: {int(a.cls)}', - (int(o[0]), int(o[1]) - 10), - cv.FONT_HERSHEY_SIMPLEX, - fontscale, - self.id_to_color(a.id), - thickness - ) - if len(a.history_observations) > 3: - for e, o in enumerate(a.history_observations): - trajectory_thickness = int(np.sqrt(float (e + 1)) * 1.2) - cv.circle( - img, - (int((o[0] + o[2]) / 2), - int((o[1] + o[3]) / 2)), - 2, - color=self.id_to_color(int(a.id)), - thickness=trajectory_thickness - ) return img From 649015476c95c8a6f997090864e2a07680cb3170 Mon Sep 17 00:00:00 2001 From: "mikel.brostrom" Date: Thu, 7 Mar 2024 18:47:08 +0100 Subject: [PATCH 13/17] histories now deques --- boxmot/trackers/hybridsort/hybridsort.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/boxmot/trackers/hybridsort/hybridsort.py b/boxmot/trackers/hybridsort/hybridsort.py index c4d34fbb9c..ed7076028e 100644 --- a/boxmot/trackers/hybridsort/hybridsort.py +++ b/boxmot/trackers/hybridsort/hybridsort.py @@ -161,7 +161,7 @@ def __init__( self.time_since_update = 0 self.id = KalmanBoxTracker.count KalmanBoxTracker.count += 1 - self.history = [] + self.history = deque([], maxlen=50) self.hits = 0 self.hit_streak = 0 self.age = 0 @@ -178,7 +178,7 @@ def __init__( self.last_observation = np.array([-1, -1, -1, -1, -1]) # placeholder self.last_observation_save = np.array([-1, -1, -1, -1, -1]) self.observations = dict() - self.history_observations = [] + self.history_observations = deque([], maxlen=50) self.velocity_lt = None self.velocity_rt = None self.velocity_lb = None From 64be2a24c6877c023d93c4e5c4f1367f0a7858a4 Mon Sep 17 00:00:00 2001 From: "mikel.brostrom" Date: Thu, 7 Mar 2024 18:47:47 +0100 Subject: [PATCH 14/17] cleanup --- tracking/track.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tracking/track.py b/tracking/track.py index ea4e3f2780..e9988034c1 100644 --- a/tracking/track.py +++ b/tracking/track.py @@ -98,7 +98,7 @@ def run(args): # store custom args in predictor yolo.predictor.custom_args = args - for idx, r in enumerate(results): + for r in results: img = yolo.predictor.trackers[0].plot_trajectory(r.orig_img) From c1de31c8e9dc5592fbb5fdfacb9b16e24297d3cf Mon Sep 17 00:00:00 2001 From: "mikel.brostrom" Date: Thu, 7 Mar 2024 18:53:08 +0100 Subject: [PATCH 15/17] docstring for basetracker --- boxmot/trackers/basetracker.py | 89 +++++++++++++++++++++++++++++++--- 1 file changed, 82 insertions(+), 7 deletions(-) diff --git a/boxmot/trackers/basetracker.py b/boxmot/trackers/basetracker.py index 901077c040..c5cd6d6a2b 100644 --- a/boxmot/trackers/basetracker.py +++ b/boxmot/trackers/basetracker.py @@ -5,7 +5,21 @@ class BaseTracker(object): - def __init__(self, det_thresh=0.3, max_age=30, min_hits=3, iou_threshold=0.3): + def __init__(self, det_thresh: float = 0.3, max_age: int = 30, min_hits: int = 3, iou_threshold: float = 0.3): + """ + Initialize the BaseTracker object with detection threshold, maximum age, minimum hits, + and Intersection Over Union (IOU) threshold for tracking objects in video frames. + + Parameters: + - det_thresh (float): Detection threshold for considering detections. + - max_age (int): Maximum age of a track before it is considered lost. + - min_hits (int): Minimum number of detection hits before a track is considered confirmed. + - iou_threshold (float): IOU threshold for determining match between detection and tracks. + + Attributes: + - frame_count (int): Counter for the frames processed. + - active_tracks (list): List to hold active tracks, may be used differently in subclasses. + """ self.det_thresh = det_thresh self.max_age = max_age self.min_hits = min_hits @@ -14,10 +28,34 @@ def __init__(self, det_thresh=0.3, max_age=30, min_hits=3, iou_threshold=0.3): self.frame_count = 0 self.active_tracks = [] # This might be handled differently in derived classes - def update(self, dets, img, embs=None): + def update(self, dets: np.ndarray, img: np.ndarray, embs: np.ndarray = None) -> None: + """ + Abstract method to update the tracker with new detections for a new frame. This method + should be implemented by subclasses. + + Parameters: + - dets (np.ndarray): Array of detections for the current frame. + - img (np.ndarray): The current frame as an image array. + - embs (np.ndarray, optional): Embeddings associated with the detections, if any. + + Raises: + - NotImplementedError: If the subclass does not implement this method. + """ raise NotImplementedError("The update method needs to be implemented by the subclass.") - def id_to_color(self, id, saturation=0.75, value=0.95): + def id_to_color(self, id: int, saturation: float = 0.75, value: float = 0.95) -> tuple: + """ + Generates a consistent unique BGR color for a given ID using hashing. + + Parameters: + - id (int): Unique identifier for which to generate a color. + - saturation (float): Saturation value for the color in HSV space. + - value (float): Value (brightness) for the color in HSV space. + + Returns: + - tuple: A tuple representing the BGR color. + """ + # Hash the ID to get a consistent unique value hash_object = hashlib.sha256(str(id).encode()) hash_digest = hash_object.hexdigest() @@ -40,7 +78,20 @@ def id_to_color(self, id, saturation=0.75, value=0.95): return bgr - def plot_box_on_img(self, img, box, conf, cls, id): + def plot_box_on_img(self, img: np.ndarray, box: tuple, conf: float, cls: int, id: int) -> np.ndarray: + """ + Draws a bounding box with ID, confidence, and class information on an image. + + Parameters: + - img (np.ndarray): The image array to draw on. + - box (tuple): The bounding box coordinates as (x1, y1, x2, y2). + - conf (float): Confidence score of the detection. + - cls (int): Class ID of the detection. + - id (int): Unique identifier for the detection. + + Returns: + - np.ndarray: The image array with the bounding box drawn on it. + """ thickness = 2 fontscale = 0.5 @@ -64,8 +115,21 @@ def plot_box_on_img(self, img, box, conf, cls, id): return img - def plot_trackers_trajectories(self, img, observations, id): - + def plot_trackers_trajectories(self, img: np.ndarray, observations: list, id: int) -> np.ndarray: + """ + Draws the trajectories of tracked objects based on historical observations. Each point + in the trajectory is represented by a circle, with the thickness increasing for more + recent observations to visualize the path of movement. + + Parameters: + - img (np.ndarray): The image array on which to draw the trajectories. + - observations (list): A list of bounding box coordinates representing the historical + observations of a tracked object. Each observation is in the format (x1, y1, x2, y2). + - id (int): The unique identifier of the tracked object for color consistency in visualization. + + Returns: + - np.ndarray: The image array with the trajectories drawn on it. + """ for i, box in enumerate(observations): trajectory_thickness = int(np.sqrt(float (i + 1)) * 1.2) img = cv.circle( @@ -79,8 +143,19 @@ def plot_trackers_trajectories(self, img, observations, id): return img - def plot_trajectory(self, img): + def plot_trajectory(self, img: np.ndarray) -> np.ndarray: + """ + Visualizes the trajectories of all active tracks on the image. For each track, + it draws the latest bounding box and the path of movement if the history of + observations is longer than two. This helps in understanding the movement patterns + of each tracked object. + + Parameters: + - img (np.ndarray): The image array on which to draw the trajectories and bounding boxes. + Returns: + - np.ndarray: The image array with trajectories and bounding boxes of all active tracks. + """ for a in self.active_tracks: if a.history_observations: if len(a.history_observations) > 2: From 03ab61e0ae90dc078d996cf9dd0dc7544d419c78 Mon Sep 17 00:00:00 2001 From: "mikel.brostrom" Date: Thu, 7 Mar 2024 18:59:11 +0100 Subject: [PATCH 16/17] exit on " " and "q" --- tracking/track.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tracking/track.py b/tracking/track.py index e9988034c1..311aca84d8 100644 --- a/tracking/track.py +++ b/tracking/track.py @@ -104,7 +104,8 @@ def run(args): if args.show is True: cv2.imshow('BoxMOT', img) - if cv2.waitKey(1) & 0xFF == ord(' '): + key = cv2.waitKey(1) & 0xFF + if key == ord(' ') or key == ord('q'): break From db048b83f42848bcd1c55654b6aad3c06162207c Mon Sep 17 00:00:00 2001 From: "mikel.brostrom" Date: Thu, 7 Mar 2024 19:06:46 +0100 Subject: [PATCH 17/17] make the trajectories plotting toggleable --- boxmot/trackers/basetracker.py | 5 +++-- tracking/track.py | 4 +++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/boxmot/trackers/basetracker.py b/boxmot/trackers/basetracker.py index c5cd6d6a2b..633861819c 100644 --- a/boxmot/trackers/basetracker.py +++ b/boxmot/trackers/basetracker.py @@ -143,7 +143,7 @@ def plot_trackers_trajectories(self, img: np.ndarray, observations: list, id: in return img - def plot_trajectory(self, img: np.ndarray) -> np.ndarray: + def plot_results(self, img: np.ndarray, show_trajectories: bool) -> np.ndarray: """ Visualizes the trajectories of all active tracks on the image. For each track, it draws the latest bounding box and the path of movement if the history of @@ -161,7 +161,8 @@ def plot_trajectory(self, img: np.ndarray) -> np.ndarray: if len(a.history_observations) > 2: box = a.history_observations[-1] img = self.plot_box_on_img(img, box, a.conf, a.cls, a.id) - img = self.plot_trackers_trajectories(img, a.history_observations, a.id) + if show_trajectories: + img = self.plot_trackers_trajectories(img, a.history_observations, a.id) return img diff --git a/tracking/track.py b/tracking/track.py index 311aca84d8..385ababe5e 100644 --- a/tracking/track.py +++ b/tracking/track.py @@ -100,7 +100,7 @@ def run(args): for r in results: - img = yolo.predictor.trackers[0].plot_trajectory(r.orig_img) + img = yolo.predictor.trackers[0].plot_results(r.orig_img, args.show_trajectories) if args.show is True: cv2.imshow('BoxMOT', img) @@ -148,6 +148,8 @@ def parse_opt(): help='either show all or only bboxes') parser.add_argument('--show-conf', action='store_false', help='hide confidences when show') + parser.add_argument('--show-trajectories', action='store_true', + help='show confidences') parser.add_argument('--save-txt', action='store_true', help='save tracking results in a txt file') parser.add_argument('--save-id-crops', action='store_true',