From 0c2ca56b84df73d5a5290aab309f50be37177be4 Mon Sep 17 00:00:00 2001 From: hannalee2 Date: Fri, 2 Aug 2024 15:46:11 -0700 Subject: [PATCH 01/18] Fixed the average error calcuration --- parallax/coords_transformation.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/parallax/coords_transformation.py b/parallax/coords_transformation.py index 6103145..302b044 100644 --- a/parallax/coords_transformation.py +++ b/parallax/coords_transformation.py @@ -64,9 +64,17 @@ def func(self, x, measured_pts, global_pts, reflect_z=False): def avg_error(self, x, measured_pts, global_pts, reflect_z=False): """Calculates the total error for the optimization.""" error_values = self.func(x, measured_pts, global_pts, reflect_z) - mean_squared_error = np.mean(error_values**2) - average_error = np.sqrt(mean_squared_error) - return average_error + + # Calculate the L2 error for each point + l2_errors = np.zeros(len(global_pts)) + for i in range(len(global_pts)): + error_vector = error_values[i * 3: (i + 1) * 3] + l2_errors[i] = np.linalg.norm(error_vector) + + # Calculate the average L2 error + average_l2_error = np.mean(l2_errors) + + return average_l2_error def fit_params(self, measured_pts, global_pts): """Fits parameters to minimize the error defined in func""" From 535342f9cd64a994ed4ec686aa6950a29895313e Mon Sep 17 00:00:00 2001 From: hannalee2 Date: Mon, 5 Aug 2024 12:21:12 -0700 Subject: [PATCH 02/18] Change printing msg after probel calibration --- parallax/bundle_adjustment.py | 4 ++-- parallax/probe_calibration.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/parallax/bundle_adjustment.py b/parallax/bundle_adjustment.py index 600c142..526fe47 100644 --- a/parallax/bundle_adjustment.py +++ b/parallax/bundle_adjustment.py @@ -184,7 +184,7 @@ def optimize(self, print_result=True): self.opt_points = opt_params[12 * n_cams:].reshape(n_pts, 3) if print_result: - print(f"\n************** Optimization completed. **************************") + print(f"\n*********** Optimization completed **************") # Compute initial residuals initial_residuals = self.residuals(initial_params) initial_residuals_sum = np.sum(initial_residuals**2) @@ -196,7 +196,7 @@ def optimize(self, print_result=True): opt_residuals_sum = np.sum(opt_residuals**2) average_residual = opt_residuals_sum / len(self.bal_problem.observations) print(f"** After BA, Average residual of reproj: {np.round(average_residual, 2)} **") - print(f"******************************************************************") + print(f"****************************************************") logger.debug(f"Optimized camera parameters: {self.opt_camera_params}") diff --git a/parallax/probe_calibration.py b/parallax/probe_calibration.py index 002730c..beea5f9 100644 --- a/parallax/probe_calibration.py +++ b/parallax/probe_calibration.py @@ -38,7 +38,6 @@ class ProbeCalibration(QObject): calib_complete_x = pyqtSignal(str) calib_complete_y = pyqtSignal(str) calib_complete_z = pyqtSignal(str) - #calib_complete = pyqtSignal(str, object) calib_complete = pyqtSignal(str, object, np.ndarray) transM_info = pyqtSignal(str, object, np.ndarray, float, object) From 61d2048a58cfacc074aa8860817304fa2230cd5c Mon Sep 17 00:00:00 2001 From: hannalee2 Date: Mon, 5 Aug 2024 16:23:55 -0700 Subject: [PATCH 03/18] Implementeted 3D visualization (basic function) --- parallax/point_mesh.py | 221 ++++++++++++++++++++++++++++++++++ parallax/probe_calibration.py | 75 +++++++----- pyproject.toml | 3 +- ui/point_mesh.ui | 100 +++++++++++++++ 4 files changed, 366 insertions(+), 33 deletions(-) create mode 100644 parallax/point_mesh.py create mode 100644 ui/point_mesh.ui diff --git a/parallax/point_mesh.py b/parallax/point_mesh.py new file mode 100644 index 0000000..0a4dc69 --- /dev/null +++ b/parallax/point_mesh.py @@ -0,0 +1,221 @@ +import os +import mplcursors +import pandas as pd +import matplotlib.pyplot as plt +from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FigureCanvas +from PyQt5.QtWidgets import QWidget, QPushButton +from PyQt5.uic import loadUi + +package_dir = os.path.dirname(os.path.abspath(__file__)) +ui_dir = os.path.join(os.path.dirname(package_dir), "ui") + +class PointMesh(QWidget): + def __init__(self, model, file_path, sn): + super().__init__() + self.model = model + self.file_path = file_path + self.sn = sn + + self.R, self.R_BA = {}, {} + self.T, self.T_BA = {}, {} + self.S, self.S_BA = {}, {} + + self.points_dict = {} + self.colors = {} + self.cursors = {} # Dictionary to keep track of mplcursors for each key + + def show(self): + self._parse_csv() + self._init_ui() + super().show() # Show the widget + + def set_transM(self, transM, scale): + self.R[self.sn] = transM[:3, :3] + self.T[self.sn] = transM[:3, 3] + self.S[self.sn] = scale[:3] + + def set_transM_BA(self, transM, scale): + self.R_BA[self.sn] = transM[:3, :3] + self.T_BA[self.sn] = transM[:3, 3] + self.S_BA[self.sn] = scale[:3] + + def _parse_csv(self): + self.df = pd.read_csv(self.file_path) + + self.local_pts_org = self.df[['local_x', 'local_y', 'local_z']].values + self.local_pts = self._local_to_global(self.local_pts_org, self.R[self.sn], self.T[self.sn], self.S[self.sn]) + self.points_dict['local_pts'] = self.local_pts + + self.global_pts = self.df[['global_x', 'global_y', 'global_z']].values + self.points_dict['global_pts'] = self.global_pts + + if self.model.bundle_adjustment: + self.m_global_pts = self.df[['m_global_x', 'm_global_y', 'm_global_z']].values + self.points_dict['m_global_pts'] = self.m_global_pts + + self.opt_global_pts = self.df[['opt_x', 'opt_y', 'opt_z']].values + self.points_dict['opt_global_pts'] = self.opt_global_pts + + self.local_pts_BA = self._local_to_global(self.local_pts_org, self.R_BA[self.sn], self.T_BA[self.sn], self.S_BA[self.sn]) + self.points_dict['local_pts_BA'] = self.local_pts_BA + + # Assign unique colors to each key + color_list = ['r', 'g', 'b', 'c', 'm', 'y', 'k'] + for i, key in enumerate(self.points_dict.keys()): + self.colors[key] = color_list[i % len(color_list)] + + def _local_to_global(self, local_pts, R, t, scale=None): + if scale is not None: + local_pts = local_pts * scale + global_coords_exp = R @ local_pts.T + t.reshape(-1, 1) + return global_coords_exp.T + + def _init_ui(self): + self.ui = loadUi(os.path.join(ui_dir, "point_mesh.ui"), self) + + self.figure = plt.figure(figsize=(15, 10)) + self.canvas = FigureCanvas(self.figure) + + # Add the canvas to the first column of verticalLayout1 + self.ui.verticalLayout1.addWidget(self.canvas) + + # Add buttons to the existing verticalLayout2 + self.buttons = {} + for key in self.points_dict.keys(): + button_name = self._get_button_name(key) + button = QPushButton(f'{button_name}') + button.setCheckable(True) # Make the button checkable + button.setMaximumWidth(200) + button.clicked.connect(lambda checked, key=key: self._update_plot(key, checked)) + self.ui.verticalLayout2.addWidget(button) + self.buttons[key] = button + + # Set initial state of buttons + if self.model.bundle_adjustment: + keys_to_check = ['local_pts_BA', 'opt_global_pts'] + else: + keys_to_check = ['local_pts', 'global_pts'] + + for key in keys_to_check: + self.buttons[key].setChecked(True) + self._draw_specific_points(key) + + # Update the legend + self._update_legend() + + """ + # Add zoom in and zoom out buttons + self.zoom_in_button = QPushButton('Zoom In') + self.zoom_in_button.clicked.connect(self._zoom_in) + self.ui.verticalLayout1.addWidget(self.zoom_in_button) + + self.zoom_out_button = QPushButton('Zoom Out') + self.zoom_out_button.clicked.connect(self._zoom_out) + self.ui.verticalLayout1.addWidget(self.zoom_out_button) + """ + + def _get_button_name(self, key): + if key == 'local_pts': + return 'stage' + elif key == 'local_pts_BA': + return 'stage (BA)' + elif key == 'global_pts': + return 'global' + elif key == 'm_global_pts': + return 'global (mean)' + elif key == 'opt_global_pts': + return 'global (BA)' + else: + return key # Default to the key if no match + + def _update_plot(self, key, checked): + if checked: + self._draw_specific_points(key) + else: + self._remove_points_from_plot(key) + self._update_legend() + self.canvas.draw() + + def _remove_points_from_plot(self, key): + # Remove the points and lines corresponding to the given key + label_name = self._get_button_name(key) + artists_to_remove = [artist for artist in self.ax.lines + self.ax.collections if artist.get_label() == label_name] + for artist in artists_to_remove: + artist.remove() + + self._remove_points_info(key) + + def _remove_points_info(self, key='all'): + # Remove the associated cursor if it exists + if key == 'all': + # Remove all cursors + for key in list(self.cursors.keys()): + cursor = self.cursors[key] + cursor.remove() + del self.cursors[key] # Clear the dictionary after removing all cursors + + else: + if key in self.cursors: + cursor = self.cursors[key] + cursor.remove() + del self.cursors[key] + + def _draw_specific_points(self, key): + if not hasattr(self, 'ax'): + self.ax = self.figure.add_subplot(111, projection='3d') # Initialize self.ax if not already + pts = self.points_dict[key] + scatter = self._add_points_to_plot(self.ax, pts, key, s=0.5) + self._add_lines_to_plot(self.ax, pts, key, linewidth=0.5) + + # Add mplcursors for hover functionality + cursor = mplcursors.cursor(scatter, hover=True) + + def on_add(sel): + # Compute the annotation text + text = f'({int(pts[sel.index, 0])}, {int(pts[sel.index, 1])}, {int(pts[sel.index, 2])})' + sel.annotation.set_text(text) + # Set the background color to match the line color + sel.annotation.get_bbox_patch().set_facecolor(self.colors[key]) + sel.annotation.get_bbox_patch().set_alpha(1.0) # Set transparency + cursor.connect("add", on_add) + + # Save the cursor to remove later if necessary + self.cursors[key] = cursor + + def _add_points_to_plot(self, ax, pts, name, s=1): + label_name = self._get_button_name(name) + scatter = ax.scatter(pts[:, 0], pts[:, 1], pts[:, 2], s=s, c=self.colors[name], label=label_name) + return scatter + + def _add_lines_to_plot(self, ax, pts, name, linewidth=0.5): + ax.plot(pts[:, 0], pts[:, 1], pts[:, 2], color=self.colors[name], linewidth=linewidth, label=self._get_button_name(name)) + + def _update_legend(self): + handles, labels = self.ax.get_legend_handles_labels() + by_label = dict(zip(labels, handles)) + self.ax.legend(by_label.values(), by_label.keys(), loc='upper left') + + def _zoom_in(self): + self._zoom(0.9) + #self._remove_points_info(key='all') # Remove all cursors + + def _zoom_out(self): + self._zoom(1.1) + #self._remove_points_info(key='all') # Remove all cursors + + def _zoom(self, scale_factor): + xlim = self.ax.get_xlim() + ylim = self.ax.get_ylim() + zlim = self.ax.get_zlim() + + self.ax.set_xlim([x * scale_factor for x in xlim]) + self.ax.set_ylim([y * scale_factor for y in ylim]) + self.ax.set_zlim([z * scale_factor for z in zlim]) + + self.canvas.draw() + + def wheelEvent(self, event): + if event.angleDelta().y() > 0: + self._zoom_in() + else: + self._zoom_out() \ No newline at end of file diff --git a/parallax/probe_calibration.py b/parallax/probe_calibration.py index beea5f9..9d8f8a3 100644 --- a/parallax/probe_calibration.py +++ b/parallax/probe_calibration.py @@ -13,6 +13,7 @@ from PyQt5.QtCore import QObject, pyqtSignal from .coords_transformation import RotationTransformation from .bundle_adjustment import BALProblem, BALOptimizer +from .point_mesh import PointMesh # Set logger name logger = logging.getLogger(__name__) @@ -57,10 +58,10 @@ def __init__(self, model, stage_listener): self.inliers = [] self.stage = None - """ self.threshold_min_max = 250 self.threshold_min_max_z = 200 self.LR_err_L2_threshold = 200 + self.threshold_avg_error = 500 self.threshold_matrix = np.array( [ [0.002, 0.002, 0.002, 0.0], @@ -82,7 +83,8 @@ def __init__(self, model, stage_listener): [0.0, 0.0, 0.0, 0.0], ] ) - + """ + self.model_LR, self.transM_LR, self.transM_LR_prev = None, None, None self.origin, self.R, self.scale = None, None, np.array([1, 1, 1]) self.avg_err = None @@ -581,7 +583,6 @@ def _print_formatted_transM(self): print(f" [{S[0]:.5f}, {S[1]:.5f}, {S[2]:.5f}]") print("==> Average L2 between stage and global: ", self.avg_err) - def update(self, stage, debug_info=None): """ Main method to update calibration with a new stage position and check if calibration is complete. @@ -609,35 +610,45 @@ def update(self, stage, debug_info=None): self._update_info_ui() # update transformation matrix and overall LR in UI ret = self._is_enough_points() # if ret, send the signal if ret: - # save the filtered points to a new file - self.file_name = f"points_{self.stage.sn}.csv" - self._get_transM(filtered_df, save_to_csv=True, file_name=self.file_name) - - # TODO - Bundle Adjustment - print("\n\n=========================================================") - print("Before BA") - self._print_formatted_transM() - print("=========================================================") - self._update_info_ui(disp_avg_error=True, save_to_csv=True, \ - file_name = f"transM_{self.stage.sn}.csv") - - if self.model.bundle_adjustment: - ret = self.run_bundle_adjustment(self.file_name) - if ret: - print("\n=========================================================") - print("After BA") - self._print_formatted_transM() - print("=========================================================") - self._update_info_ui(disp_avg_error=True, save_to_csv=True, \ - file_name = f"transM_BA_{self.stage.sn}.csv") - else: - return - - # Emit the signal to indicate that calibration is complete - self.calib_complete.emit(self.stage.sn, self.transM_LR, self.scale) - logger.debug( - f"complete probe calibration {self.stage.sn}, {self.transM_LR}, {self.scale}" - ) + self.complete_calibration(filtered_df) + + def complete_calibration(self, filtered_df): + # save the filtered points to a new file + self.file_name = f"points_{self.stage.sn}.csv" + self._get_transM(filtered_df, save_to_csv=True, file_name=self.file_name) + + print("\n\n=========================================================") + print("Before BA") + self._print_formatted_transM() + print("=========================================================") + self._update_info_ui(disp_avg_error=True, save_to_csv=True, \ + file_name = f"transM_{self.stage.sn}.csv") + + # Init point_mesh + point_mesh = PointMesh(self.model, self.file_name, self.stage.sn) + point_mesh.set_transM(self.transM_LR, self.scale) # Set transM + + if self.model.bundle_adjustment: + ret = self.run_bundle_adjustment(self.file_name) + if ret: + print("\n=========================================================") + print("After BA") + self._print_formatted_transM() + print("=========================================================") + self._update_info_ui(disp_avg_error=True, save_to_csv=True, \ + file_name = f"transM_BA_{self.stage.sn}.csv") + point_mesh.set_transM_BA(self.transM_LR, self.scale) # set TransM_BA + else: + return + + # Emit the signal to indicate that calibration is complete + self.calib_complete.emit(self.stage.sn, self.transM_LR, self.scale) + logger.debug( + f"complete probe calibration {self.stage.sn}, {self.transM_LR}, {self.scale}" + ) + + # Draw trajectory + point_mesh.show() def run_bundle_adjustment(self, file_path): bal_problem = BALProblem(self.model, file_path) diff --git a/pyproject.toml b/pyproject.toml index 9881a3e..689174a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,8 @@ dependencies = [ "pandas", "scikit-image", "requests", - "scikit-learn" + "scikit-learn", + "mplcursors" ] [tool.setuptools] diff --git a/ui/point_mesh.ui b/ui/point_mesh.ui new file mode 100644 index 0000000..4f63616 --- /dev/null +++ b/ui/point_mesh.ui @@ -0,0 +1,100 @@ + + + plot_widget + + + + 0 + 0 + 1273 + 800 + + + + + 1200 + 800 + + + + + 4000 + 3000 + + + + Form + + + QWidget{ + background-color: rgb(0,0,0); + color: #FFFFFF; +} + +QPushButton{ + background-color: black; +} + +QPushButton:pressed { + background-color: rgb(224, 0, 0); +} + +QPushButton:hover { + background-color: rgb(100, 30, 30); +} + +QPushButton:checked { + color: gray; + background-color: #ffaaaa; +} + + + + + 20 + 10 + 1571 + 981 + + + + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + + + + + Qt::Vertical + + + + 20 + 40 + + + + + + + + + + + + From 46478569a4923aa4a356c7cc19d5775af6b769a2 Mon Sep 17 00:00:00 2001 From: hannalee2 Date: Tue, 6 Aug 2024 14:39:03 -0700 Subject: [PATCH 04/18] 3D trajectory visualization implemented --- parallax/point_mesh.py | 16 +++- parallax/probe_calibration.py | 46 +++++----- parallax/stage_widget.py | 24 +++++- ui/probe_calib.ui | 155 ++++++++++++++++++++++------------ 4 files changed, 154 insertions(+), 87 deletions(-) diff --git a/parallax/point_mesh.py b/parallax/point_mesh.py index 0a4dc69..66bf061 100644 --- a/parallax/point_mesh.py +++ b/parallax/point_mesh.py @@ -7,7 +7,9 @@ from PyQt5.uic import loadUi package_dir = os.path.dirname(os.path.abspath(__file__)) +debug_dir = os.path.join(os.path.dirname(package_dir), "debug") ui_dir = os.path.join(os.path.dirname(package_dir), "ui") +csv_file = os.path.join(debug_dir, "points.csv") class PointMesh(QWidget): def __init__(self, model, file_path, sn): @@ -23,6 +25,15 @@ def __init__(self, model, file_path, sn): self.points_dict = {} self.colors = {} self.cursors = {} # Dictionary to keep track of mplcursors for each key + + self.figure = plt.figure(figsize=(15, 10)) + self.ax = self.figure.add_subplot(111, projection='3d') + + """ + def initialize(self): + if not hasattr(self, 'ax'): + # Initialize self.ax if not already + print("self.ax initialized")""" def show(self): self._parse_csv() @@ -41,6 +52,7 @@ def set_transM_BA(self, transM, scale): def _parse_csv(self): self.df = pd.read_csv(self.file_path) + self.df = self.df[self.df["sn"] == self.sn] # filter by sn self.local_pts_org = self.df[['local_x', 'local_y', 'local_z']].values self.local_pts = self._local_to_global(self.local_pts_org, self.R[self.sn], self.T[self.sn], self.S[self.sn]) @@ -72,8 +84,6 @@ def _local_to_global(self, local_pts, R, t, scale=None): def _init_ui(self): self.ui = loadUi(os.path.join(ui_dir, "point_mesh.ui"), self) - - self.figure = plt.figure(figsize=(15, 10)) self.canvas = FigureCanvas(self.figure) # Add the canvas to the first column of verticalLayout1 @@ -161,8 +171,6 @@ def _remove_points_info(self, key='all'): del self.cursors[key] def _draw_specific_points(self, key): - if not hasattr(self, 'ax'): - self.ax = self.figure.add_subplot(111, projection='3d') # Initialize self.ax if not already pts = self.points_dict[key] scatter = self._add_points_to_plot(self.ax, pts, key, s=0.5) self._add_lines_to_plot(self.ax, pts, key, linewidth=0.5) diff --git a/parallax/probe_calibration.py b/parallax/probe_calibration.py index 9d8f8a3..e520224 100644 --- a/parallax/probe_calibration.py +++ b/parallax/probe_calibration.py @@ -54,6 +54,7 @@ def __init__(self, model, stage_listener): self.stage_listener = stage_listener self.stage_listener.probeCalibRequest.connect(self.update) self.stages = {} + self.point_mesh = {} self.df = None self.inliers = [] self.stage = None @@ -106,7 +107,8 @@ def reset_calib(self, sn=None): 'max_z': float("-inf"), 'signal_emitted_x': False, 'signal_emitted_y': False, - 'signal_emitted_z': False + 'signal_emitted_z': False, + 'calib_completed': False } else: self.stages = {} @@ -363,7 +365,9 @@ def _update_min_max_x_y_z(self): self.stages[sn] = { 'min_x': float("inf"), 'max_x': float("-inf"), 'min_y': float("inf"), 'max_y': float("-inf"), - 'min_z': float("inf"), 'max_z': float("-inf") + 'min_z': float("inf"), 'max_z': float("-inf"), + 'signal_emitted_x': False, 'signal_emitted_y': False, + 'signal_emitted_z': False, 'calib_completed': False } self.stages[sn]['min_x'] = min(self.stages[sn]['min_x'], self.stage.stage_x) @@ -618,37 +622,46 @@ def complete_calibration(self, filtered_df): self._get_transM(filtered_df, save_to_csv=True, file_name=self.file_name) print("\n\n=========================================================") - print("Before BA") self._print_formatted_transM() print("=========================================================") self._update_info_ui(disp_avg_error=True, save_to_csv=True, \ file_name = f"transM_{self.stage.sn}.csv") # Init point_mesh - point_mesh = PointMesh(self.model, self.file_name, self.stage.sn) - point_mesh.set_transM(self.transM_LR, self.scale) # Set transM + self.stages[self.stage.sn]['calib_completed'] = True + if hasattr(self, 'point_mesh_not_clibed'): + del self.point_mesh_not_clibed + self.point_mesh[self.stage.sn] = PointMesh(self.model, self.file_name, self.stage.sn) + self.point_mesh[self.stage.sn].set_transM(self.transM_LR, self.scale) # Set transM if self.model.bundle_adjustment: ret = self.run_bundle_adjustment(self.file_name) if ret: print("\n=========================================================") - print("After BA") + print("** After Bundle Adjustment **") self._print_formatted_transM() print("=========================================================") self._update_info_ui(disp_avg_error=True, save_to_csv=True, \ file_name = f"transM_BA_{self.stage.sn}.csv") - point_mesh.set_transM_BA(self.transM_LR, self.scale) # set TransM_BA + self.point_mesh[self.stage.sn].set_transM_BA(self.transM_LR, self.scale) # set TransM_BA else: return - # Emit the signal to indicate that calibration is complete + # Emit the signal to indicate that calibration is complete self.calib_complete.emit(self.stage.sn, self.transM_LR, self.scale) logger.debug( f"complete probe calibration {self.stage.sn}, {self.transM_LR}, {self.scale}" ) - # Draw trajectory - point_mesh.show() + def view_3d_trajectory(self, sn): + if not self.stages.get(sn, {}).get('calib_completed', False): + print("View - Calibration is not complete.") + if sn == self.stage.sn: + self.point_mesh_not_clibed = PointMesh(self.model, self.csv_file, self.stage.sn) + self.point_mesh_not_clibed.set_transM(self.transM_LR, self.scale) + self.point_mesh_not_clibed.show() + else: + self.point_mesh[sn].show() def run_bundle_adjustment(self, file_path): bal_problem = BALProblem(self.model, file_path) @@ -661,19 +674,6 @@ def run_bundle_adjustment(self, file_path): if self.transM_LR is None: return False - """ - # Save local_pts and optimzied global pts in file_path - # Save local_pts and optimized global pts in file_path - df = pd.DataFrame({ - 'local_x': local_pts[:, 0], - 'local_y': local_pts[:, 1], - 'local_z': local_pts[:, 2], - 'global_x': opt_global_pts[:, 0], - 'global_y': opt_global_pts[:, 1], - 'opt_global_z': opt_global_pts[:, 2] - }) - df.to_csv(file_path, index=False)""" - logger.debug(f"Number of observations: {len(bal_problem.observations)}") logger.debug(f"Number of 3d points: {len(bal_problem.points)}") for i in range(len(bal_problem.list_cameras)): diff --git a/parallax/stage_widget.py b/parallax/stage_widget.py index 3ffa416..3222eed 100644 --- a/parallax/stage_widget.py +++ b/parallax/stage_widget.py @@ -9,8 +9,6 @@ import logging import os import math -import time - import numpy as np from PyQt5.QtCore import QTimer, Qt from PyQt5.QtWidgets import (QLabel, QMessageBox, QPushButton, QSizePolicy, @@ -84,6 +82,12 @@ def __init__(self, model, ui_dir, screen_widgets): QLabel, "probeCalibrationLabel" ) self.probeCalibrationLabel.setTextInteractionFlags(Qt.TextSelectableByMouse) + self.viewTrajectory_btn = self.probe_calib_widget.findChild( + QPushButton, "viewTrajectory_btn" + ) + self.viewTrajectory_btn.clicked.connect( + self.view_trajectory_button_handler + ) # Reticle Widget self.reticle_detection_status = ( @@ -127,6 +131,7 @@ def __init__(self, model, ui_dir, screen_widgets): self.calib_x.hide() self.calib_y.hide() self.calib_z.hide() + self.viewTrajectory_btn.hide() self.probeCalibration.calib_complete_x.connect(self.calib_x_complete) self.probeCalibration.calib_complete_y.connect(self.calib_y_complete) self.probeCalibration.calib_complete_z.connect(self.calib_z_complete) @@ -737,6 +742,7 @@ def probe_detect_default_status_ui(self, sn = None): } """) self.hide_x_y_z() + self.hide_trajectory_btn() self.probeCalibrationLabel.setText("") self.probe_calibration_btn.setChecked(False) @@ -837,6 +843,8 @@ def probe_detect_accepted_status(self, stage_sn, transformation_matrix, scale, s self.probe_calibration_btn.setChecked(True) self.hide_x_y_z() + if not self.viewTrajectory_btn.isVisible(): + self.viewTrajectory_btn.show() if self.filter == "probe_detection": for screen in self.screen_widgets: camera_name = screen.get_camera_name() @@ -888,6 +896,10 @@ def hide_x_y_z(self): self.set_default_x_y_z_style() + def hide_trajectory_btn(self): + if self.viewTrajectory_btn.isVisible(): + self.viewTrajectory_btn.hide() + def calib_x_complete(self, switch_probe = False): """ Updates the UI to indicate that the calibration for the X-axis is complete. @@ -990,7 +1002,8 @@ def update_probe_calib_status(self, moving_stage_id, transM, scale, L2_err, dist if self.moving_stage_id == self.selected_stage_id: # If moving stage is the selected stage, update the probe calibration status on UI self.display_probe_calib_status(transM, scale, L2_err, dist_traveled) - #self.display_probe_calib_status(transM, L2_err, dist_traveled) + if not self.viewTrajectory_btn.isVisible(): + self.viewTrajectory_btn.show() else: # If moving stage is not the selected stage, save the calibration info content = ( @@ -1062,4 +1075,7 @@ def update_stages(self, prev_stage_id, curr_stage_id): if self.transM is not None: self.display_probe_calib_status(self.transM, self.scale, self.L2_err, self.dist_travled) - self.probe_detection_status = probe_detection_status \ No newline at end of file + self.probe_detection_status = probe_detection_status + + def view_trajectory_button_handler(self): + self.probeCalibration.view_3d_trajectory(self.selected_stage_id) \ No newline at end of file diff --git a/ui/probe_calib.ui b/ui/probe_calib.ui index 09c211a..53b875f 100644 --- a/ui/probe_calib.ui +++ b/ui/probe_calib.ui @@ -18,29 +18,38 @@ 10 10 - 286 - 705 + 281 + 715 1 - - - - Qt::Horizontal + + + + false - + - 40 - 20 + 0 + 30 - + + + 30 + 30 + + + + Y + + - - + + Qt::Horizontal @@ -52,35 +61,29 @@ - - - - - 0 - 0 - + + + + false - 200 - 40 + 0 + 30 - 300 - 40 + 30 + 30 - Probe Calibration - - - true + Z - + false @@ -102,63 +105,103 @@ - - - + + + + Qt::Horizontal + + - 0 - 600 + 40 + 20 - - - - - Qt::AlignLeading|Qt::AlignLeft|Qt::AlignTop - - + - - - - false + + + + + 0 + 0 + - 0 + 40 30 - 30 + 40 30 + + +QPushButton{ + color: white; + background-color: black; +} + +QPushButton:pressed { + background-color: rgb(224, 0, 0); +} + +QPushButton:hover { + background-color: rgb(100, 30, 30); +} + - Y + 3D - - - - - + false + + + + 0 - 30 + 600 + + + + + + + Qt::AlignLeading|Qt::AlignLeft|Qt::AlignTop + + + + + + + + 0 + 0 + + + + + 200 + 40 - 30 - 30 + 300 + 40 - Z + Probe Calibration + + + true From abee06003758de0b597f31eedaa96d0492330ede Mon Sep 17 00:00:00 2001 From: hannalee2 Date: Tue, 6 Aug 2024 15:24:59 -0700 Subject: [PATCH 05/18] If BA not finished, show only local pts and global pts --- parallax/point_mesh.py | 31 ++++++++++++------------------- parallax/probe_calibration.py | 11 +++++------ 2 files changed, 17 insertions(+), 25 deletions(-) diff --git a/parallax/point_mesh.py b/parallax/point_mesh.py index 66bf061..d00e356 100644 --- a/parallax/point_mesh.py +++ b/parallax/point_mesh.py @@ -12,11 +12,12 @@ csv_file = os.path.join(debug_dir, "points.csv") class PointMesh(QWidget): - def __init__(self, model, file_path, sn): + def __init__(self, model, file_path, sn, calib_completed=False): super().__init__() self.model = model self.file_path = file_path self.sn = sn + self.calib_completed = calib_completed self.R, self.R_BA = {}, {} self.T, self.T_BA = {}, {} @@ -28,12 +29,9 @@ def __init__(self, model, file_path, sn): self.figure = plt.figure(figsize=(15, 10)) self.ax = self.figure.add_subplot(111, projection='3d') - - """ - def initialize(self): - if not hasattr(self, 'ax'): - # Initialize self.ax if not already - print("self.ax initialized")""" + self.canvas = FigureCanvas(self.figure) + self.canvas.setParent(self) + #self.resizeEvent = self._on_resize def show(self): self._parse_csv() @@ -61,7 +59,7 @@ def _parse_csv(self): self.global_pts = self.df[['global_x', 'global_y', 'global_z']].values self.points_dict['global_pts'] = self.global_pts - if self.model.bundle_adjustment: + if self.model.bundle_adjustment and self.calib_completed: self.m_global_pts = self.df[['m_global_x', 'm_global_y', 'm_global_z']].values self.points_dict['m_global_pts'] = self.m_global_pts @@ -101,7 +99,7 @@ def _init_ui(self): self.buttons[key] = button # Set initial state of buttons - if self.model.bundle_adjustment: + if self.model.bundle_adjustment and self.calib_completed: keys_to_check = ['local_pts_BA', 'opt_global_pts'] else: keys_to_check = ['local_pts', 'global_pts'] @@ -113,16 +111,11 @@ def _init_ui(self): # Update the legend self._update_legend() - """ - # Add zoom in and zoom out buttons - self.zoom_in_button = QPushButton('Zoom In') - self.zoom_in_button.clicked.connect(self._zoom_in) - self.ui.verticalLayout1.addWidget(self.zoom_in_button) - - self.zoom_out_button = QPushButton('Zoom Out') - self.zoom_out_button.clicked.connect(self._zoom_out) - self.ui.verticalLayout1.addWidget(self.zoom_out_button) - """ + def _on_resize(self, event): + new_size = event.size() + self.canvas.resize(new_size.width(), new_size.height()) + self.figure.tight_layout() # Adjust the layout to fit into the new size + self.canvas.draw() # Redraw the canvas def _get_button_name(self, key): if key == 'local_pts': diff --git a/parallax/probe_calibration.py b/parallax/probe_calibration.py index e520224..3b87468 100644 --- a/parallax/probe_calibration.py +++ b/parallax/probe_calibration.py @@ -630,8 +630,8 @@ def complete_calibration(self, filtered_df): # Init point_mesh self.stages[self.stage.sn]['calib_completed'] = True if hasattr(self, 'point_mesh_not_clibed'): - del self.point_mesh_not_clibed - self.point_mesh[self.stage.sn] = PointMesh(self.model, self.file_name, self.stage.sn) + del self.point_mesh_not_calibrated + self.point_mesh[self.stage.sn] = PointMesh(self.model, self.file_name, self.stage.sn, calib_completed=True) self.point_mesh[self.stage.sn].set_transM(self.transM_LR, self.scale) # Set transM if self.model.bundle_adjustment: @@ -655,11 +655,10 @@ def complete_calibration(self, filtered_df): def view_3d_trajectory(self, sn): if not self.stages.get(sn, {}).get('calib_completed', False): - print("View - Calibration is not complete.") if sn == self.stage.sn: - self.point_mesh_not_clibed = PointMesh(self.model, self.csv_file, self.stage.sn) - self.point_mesh_not_clibed.set_transM(self.transM_LR, self.scale) - self.point_mesh_not_clibed.show() + self.point_mesh_not_calibrated = PointMesh(self.model, self.csv_file, self.stage.sn) + self.point_mesh_not_calibrated.set_transM(self.transM_LR, self.scale) + self.point_mesh_not_calibrated.show() else: self.point_mesh[sn].show() From b17fc7bc4325c671c77c313e1a475e05bab5e66a Mon Sep 17 00:00:00 2001 From: hannalee2 Date: Wed, 7 Aug 2024 10:57:59 -0700 Subject: [PATCH 06/18] Expand the ui to fit into the pop-up window. --- parallax/point_mesh.py | 10 ++++++++-- ui/point_mesh.ui | 30 +++++++++++++++++++++--------- 2 files changed, 29 insertions(+), 11 deletions(-) diff --git a/parallax/point_mesh.py b/parallax/point_mesh.py index d00e356..9693328 100644 --- a/parallax/point_mesh.py +++ b/parallax/point_mesh.py @@ -5,6 +5,7 @@ from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FigureCanvas from PyQt5.QtWidgets import QWidget, QPushButton from PyQt5.uic import loadUi +from PyQt5.QtCore import Qt package_dir = os.path.dirname(os.path.abspath(__file__)) debug_dir = os.path.join(os.path.dirname(package_dir), "debug") @@ -14,6 +15,8 @@ class PointMesh(QWidget): def __init__(self, model, file_path, sn, calib_completed=False): super().__init__() + self.setWindowFlags(Qt.Window | Qt.WindowMinimizeButtonHint | \ + Qt.WindowMaximizeButtonHint | Qt.WindowCloseButtonHint) self.model = model self.file_path = file_path self.sn = sn @@ -31,7 +34,7 @@ def __init__(self, model, file_path, sn, calib_completed=False): self.ax = self.figure.add_subplot(111, projection='3d') self.canvas = FigureCanvas(self.figure) self.canvas.setParent(self) - #self.resizeEvent = self._on_resize + self.resizeEvent = self._on_resize def show(self): self._parse_csv() @@ -109,7 +112,7 @@ def _init_ui(self): self._draw_specific_points(key) # Update the legend - self._update_legend() + self._update_legend() def _on_resize(self, event): new_size = event.size() @@ -117,6 +120,9 @@ def _on_resize(self, event): self.figure.tight_layout() # Adjust the layout to fit into the new size self.canvas.draw() # Redraw the canvas + # Resize horizontal layout + self.ui.horizontalLayoutWidget.resize(new_size.width(), new_size.height()) + def _get_button_name(self, key): if key == 'local_pts': return 'stage' diff --git a/ui/point_mesh.ui b/ui/point_mesh.ui index 4f63616..bf15a35 100644 --- a/ui/point_mesh.ui +++ b/ui/point_mesh.ui @@ -6,20 +6,20 @@ 0 0 - 1273 - 800 + 1691 + 1040 - 1200 - 800 + 400 + 300 - 4000 - 3000 + 7680 + 4320 @@ -51,13 +51,25 @@ QPushButton:checked { - 20 + 10 10 - 1571 - 981 + 1661 + 1001 + + 5 + + + 5 + + + 20 + + + 20 + From 6e9e7ab3a7d22f74268c0c773b512ee0fe3e85d9 Mon Sep 17 00:00:00 2001 From: hannalee2 Date: Thu, 8 Aug 2024 09:53:25 -0700 Subject: [PATCH 07/18] Drawing plot using plotly lib --- parallax/point_mesh.py | 166 +++++++++++----------------------- parallax/probe_calibration.py | 3 +- ui/point_mesh.ui | 4 +- 3 files changed, 58 insertions(+), 115 deletions(-) diff --git a/parallax/point_mesh.py b/parallax/point_mesh.py index 9693328..6241e0a 100644 --- a/parallax/point_mesh.py +++ b/parallax/point_mesh.py @@ -1,11 +1,10 @@ import os -import mplcursors import pandas as pd -import matplotlib.pyplot as plt -from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FigureCanvas +import plotly.graph_objs as go from PyQt5.QtWidgets import QWidget, QPushButton from PyQt5.uic import loadUi from PyQt5.QtCore import Qt +from PyQt5.QtWebEngineWidgets import QWebEngineView package_dir = os.path.dirname(os.path.abspath(__file__)) debug_dir = os.path.join(os.path.dirname(package_dir), "debug") @@ -25,22 +24,23 @@ def __init__(self, model, file_path, sn, calib_completed=False): self.R, self.R_BA = {}, {} self.T, self.T_BA = {}, {} self.S, self.S_BA = {}, {} - self.points_dict = {} + self.traces = {} # Plotly trace objects self.colors = {} - self.cursors = {} # Dictionary to keep track of mplcursors for each key - - self.figure = plt.figure(figsize=(15, 10)) - self.ax = self.figure.add_subplot(111, projection='3d') - self.canvas = FigureCanvas(self.figure) - self.canvas.setParent(self) self.resizeEvent = self._on_resize + self._init_ui() def show(self): self._parse_csv() - self._init_ui() + self._init_buttons() + self._update_canvas() super().show() # Show the widget + def _init_ui(self): + self.ui = loadUi(os.path.join(ui_dir, "point_mesh.ui"), self) + self.web_view = QWebEngineView(self) + self.ui.verticalLayout1.addWidget(self.web_view) + def set_transM(self, transM, scale): self.R[self.sn] = transM[:3, :3] self.T[self.sn] = transM[:3, 3] @@ -53,7 +53,7 @@ def set_transM_BA(self, transM, scale): def _parse_csv(self): self.df = pd.read_csv(self.file_path) - self.df = self.df[self.df["sn"] == self.sn] # filter by sn + self.df = self.df[self.df["sn"] == self.sn] # filter by sn self.local_pts_org = self.df[['local_x', 'local_y', 'local_z']].values self.local_pts = self._local_to_global(self.local_pts_org, self.R[self.sn], self.T[self.sn], self.S[self.sn]) @@ -73,7 +73,7 @@ def _parse_csv(self): self.points_dict['local_pts_BA'] = self.local_pts_BA # Assign unique colors to each key - color_list = ['r', 'g', 'b', 'c', 'm', 'y', 'k'] + color_list = ['red', 'blue', 'green', 'cyan', 'magenta'] for i, key in enumerate(self.points_dict.keys()): self.colors[key] = color_list[i % len(color_list)] @@ -83,25 +83,17 @@ def _local_to_global(self, local_pts, R, t, scale=None): global_coords_exp = R @ local_pts.T + t.reshape(-1, 1) return global_coords_exp.T - def _init_ui(self): - self.ui = loadUi(os.path.join(ui_dir, "point_mesh.ui"), self) - self.canvas = FigureCanvas(self.figure) - - # Add the canvas to the first column of verticalLayout1 - self.ui.verticalLayout1.addWidget(self.canvas) - - # Add buttons to the existing verticalLayout2 + def _init_buttons(self): self.buttons = {} for key in self.points_dict.keys(): button_name = self._get_button_name(key) button = QPushButton(f'{button_name}') - button.setCheckable(True) # Make the button checkable + button.setCheckable(True) button.setMaximumWidth(200) button.clicked.connect(lambda checked, key=key: self._update_plot(key, checked)) self.ui.verticalLayout2.addWidget(button) self.buttons[key] = button - # Set initial state of buttons if self.model.bundle_adjustment and self.calib_completed: keys_to_check = ['local_pts_BA', 'opt_global_pts'] else: @@ -111,18 +103,6 @@ def _init_ui(self): self.buttons[key].setChecked(True) self._draw_specific_points(key) - # Update the legend - self._update_legend() - - def _on_resize(self, event): - new_size = event.size() - self.canvas.resize(new_size.width(), new_size.height()) - self.figure.tight_layout() # Adjust the layout to fit into the new size - self.canvas.draw() # Redraw the canvas - - # Resize horizontal layout - self.ui.horizontalLayoutWidget.resize(new_size.width(), new_size.height()) - def _get_button_name(self, key): if key == 'local_pts': return 'stage' @@ -142,87 +122,49 @@ def _update_plot(self, key, checked): self._draw_specific_points(key) else: self._remove_points_from_plot(key) - self._update_legend() - self.canvas.draw() + self._update_canvas() def _remove_points_from_plot(self, key): - # Remove the points and lines corresponding to the given key - label_name = self._get_button_name(key) - artists_to_remove = [artist for artist in self.ax.lines + self.ax.collections if artist.get_label() == label_name] - for artist in artists_to_remove: - artist.remove() - - self._remove_points_info(key) - - def _remove_points_info(self, key='all'): - # Remove the associated cursor if it exists - if key == 'all': - # Remove all cursors - for key in list(self.cursors.keys()): - cursor = self.cursors[key] - cursor.remove() - del self.cursors[key] # Clear the dictionary after removing all cursors - - else: - if key in self.cursors: - cursor = self.cursors[key] - cursor.remove() - del self.cursors[key] + if key in self.points_dict: + del self.traces[key] # Remove from self.traces + self._update_canvas() def _draw_specific_points(self, key): pts = self.points_dict[key] - scatter = self._add_points_to_plot(self.ax, pts, key, s=0.5) - self._add_lines_to_plot(self.ax, pts, key, linewidth=0.5) - - # Add mplcursors for hover functionality - cursor = mplcursors.cursor(scatter, hover=True) - - def on_add(sel): - # Compute the annotation text - text = f'({int(pts[sel.index, 0])}, {int(pts[sel.index, 1])}, {int(pts[sel.index, 2])})' - sel.annotation.set_text(text) - # Set the background color to match the line color - sel.annotation.get_bbox_patch().set_facecolor(self.colors[key]) - sel.annotation.get_bbox_patch().set_alpha(1.0) # Set transparency - cursor.connect("add", on_add) - - # Save the cursor to remove later if necessary - self.cursors[key] = cursor - - def _add_points_to_plot(self, ax, pts, name, s=1): - label_name = self._get_button_name(name) - scatter = ax.scatter(pts[:, 0], pts[:, 1], pts[:, 2], s=s, c=self.colors[name], label=label_name) - return scatter - - def _add_lines_to_plot(self, ax, pts, name, linewidth=0.5): - ax.plot(pts[:, 0], pts[:, 1], pts[:, 2], color=self.colors[name], linewidth=linewidth, label=self._get_button_name(name)) - - def _update_legend(self): - handles, labels = self.ax.get_legend_handles_labels() - by_label = dict(zip(labels, handles)) - self.ax.legend(by_label.values(), by_label.keys(), loc='upper left') - - def _zoom_in(self): - self._zoom(0.9) - #self._remove_points_info(key='all') # Remove all cursors - - def _zoom_out(self): - self._zoom(1.1) - #self._remove_points_info(key='all') # Remove all cursors - - def _zoom(self, scale_factor): - xlim = self.ax.get_xlim() - ylim = self.ax.get_ylim() - zlim = self.ax.get_zlim() - - self.ax.set_xlim([x * scale_factor for x in xlim]) - self.ax.set_ylim([y * scale_factor for y in ylim]) - self.ax.set_zlim([z * scale_factor for z in zlim]) - - self.canvas.draw() + scatter = go.Scatter3d( + x=pts[:, 0], y=pts[:, 1], z=pts[:, 2], + mode='markers+lines', + marker=dict(size=2, color=self.colors[key]), + name=self._get_button_name(key), + hoverinfo='x+y+z' + ) + self.traces[key] = scatter # Store the trace in self.traces + + def _update_canvas(self): + data = list(self.traces.values()) + layout = go.Layout( + scene=dict( + xaxis_title='X', + yaxis_title='Y', + zaxis_title='Z' + ), + margin=dict(l=0, r=0, b=0, t=0) + ) + fig = go.Figure(data=data, layout=layout) + html_content = fig.to_html(include_plotlyjs='cdn') + self.web_view.setHtml(html_content) def wheelEvent(self, event): - if event.angleDelta().y() > 0: - self._zoom_in() - else: - self._zoom_out() \ No newline at end of file + # Get the mouse position + mouse_position = event.pos() + # Apply zoom based on the scroll direction + scale_factor = 0.9 if event.angleDelta().y() > 0 else 1.1 + self._zoom(scale_factor, mouse_position.x(), mouse_position.y()) + + def _on_resize(self, event): + new_size = event.size() + self.web_view.resize(new_size.width(), new_size.height()) + self._update_canvas() + + # Resize horizontal layout + self.ui.horizontalLayoutWidget.resize(new_size.width(), new_size.height()) \ No newline at end of file diff --git a/parallax/probe_calibration.py b/parallax/probe_calibration.py index 3b87468..0e8f0d9 100644 --- a/parallax/probe_calibration.py +++ b/parallax/probe_calibration.py @@ -268,7 +268,8 @@ def _get_transM(self, df, remove_noise=True, save_to_csv=False, file_name=None): local_points, global_points = self._get_local_global_points(df) if remove_noise: - if self._is_criteria_met_points_min_max() and len(local_points) > 10 \ + #if self._is_criteria_met_points_min_max() and len(local_points) > 10 \ + if self._is_criteria_met_points_min_max() \ and self.R is not None and self.origin is not None: local_points, global_points, valid_indices = self._remove_outliers(local_points, global_points) diff --git a/ui/point_mesh.ui b/ui/point_mesh.ui index bf15a35..720ee16 100644 --- a/ui/point_mesh.ui +++ b/ui/point_mesh.ui @@ -59,10 +59,10 @@ QPushButton:checked { - 5 + 0 - 5 + 0 20 From c4ed5c189a4c24e1ef5716507690e550f67f3023 Mon Sep 17 00:00:00 2001 From: hannalee2 Date: Thu, 8 Aug 2024 10:41:15 -0700 Subject: [PATCH 08/18] Display rounded pts information on plot --- parallax/point_mesh.py | 6 +++++- parallax/probe_calibration.py | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/parallax/point_mesh.py b/parallax/point_mesh.py index 6241e0a..42f590e 100644 --- a/parallax/point_mesh.py +++ b/parallax/point_mesh.py @@ -131,8 +131,12 @@ def _remove_points_from_plot(self, key): def _draw_specific_points(self, key): pts = self.points_dict[key] + x_rounded = [round(x, 0) for x in pts[:, 0]] + y_rounded = [round(y, 0) for y in pts[:, 1]] + z_rounded = [round(z, 0) for z in pts[:, 2]] + scatter = go.Scatter3d( - x=pts[:, 0], y=pts[:, 1], z=pts[:, 2], + x=x_rounded, y=y_rounded, z=z_rounded, mode='markers+lines', marker=dict(size=2, color=self.colors[key]), name=self._get_button_name(key), diff --git a/parallax/probe_calibration.py b/parallax/probe_calibration.py index 0e8f0d9..c474096 100644 --- a/parallax/probe_calibration.py +++ b/parallax/probe_calibration.py @@ -58,7 +58,7 @@ def __init__(self, model, stage_listener): self.df = None self.inliers = [] self.stage = None - + """ self.threshold_min_max = 250 self.threshold_min_max_z = 200 self.LR_err_L2_threshold = 200 @@ -84,7 +84,7 @@ def __init__(self, model, stage_listener): [0.0, 0.0, 0.0, 0.0], ] ) - """ + self.model_LR, self.transM_LR, self.transM_LR_prev = None, None, None self.origin, self.R, self.scale = None, None, np.array([1, 1, 1]) From 1dddac99fc9679346f1384f85daed46a751afbf8 Mon Sep 17 00:00:00 2001 From: hannalee2 Date: Thu, 8 Aug 2024 10:42:16 -0700 Subject: [PATCH 09/18] Updated dependency (added plotly, PyQtWebEngine) --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 689174a..a5186ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,8 @@ dependencies = [ "scikit-image", "requests", "scikit-learn", - "mplcursors" + "plotly", + "PyQtWebEngine" ] [tool.setuptools] From 9596899214f2d99a2f063ef604dcf4cb589ca1de Mon Sep 17 00:00:00 2001 From: hannalee2 Date: Thu, 8 Aug 2024 11:22:42 -0700 Subject: [PATCH 10/18] Cloase the point mesh QWidget App when mainWindlow is closed --- parallax/main_window_wip.py | 5 +++++ parallax/model.py | 11 +++++++++++ parallax/point_mesh.py | 11 ++++++++++- 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/parallax/main_window_wip.py b/parallax/main_window_wip.py index 6f1d9d5..2e52d17 100644 --- a/parallax/main_window_wip.py +++ b/parallax/main_window_wip.py @@ -772,3 +772,8 @@ def save_user_configs(self): width = self.width() height = self.height() self.user_setting.save_user_configs(nColumn, directory, width, height) + + + def closeEvent(self, event): + self.model.close_all_point_meshes() + event.accept() \ No newline at end of file diff --git a/parallax/model.py b/parallax/model.py index 9d9fdd4..13aab75 100755 --- a/parallax/model.py +++ b/parallax/model.py @@ -26,6 +26,9 @@ def __init__(self, version="V1", bundle_adjustment=False): self.nMockCameras = 0 self.focos = [] + # point mesh + self.point_mesh_instances = [] + # stage self.nStages = 0 self.stages = {} @@ -199,3 +202,11 @@ def save_all_camera_frames(self): filename = 'camera%d_%s.png' % (i, camera.get_last_capture_time()) camera.save_last_image(filename) self.msg_log.post("Saved camera frame: %s" % filename) + + def add_point_mesh_instance(self, instance): + self.point_mesh_instances.append(instance) + + def close_all_point_meshes(self): + for instance in self.point_mesh_instances: + instance.close() + self.point_mesh_instances.clear() \ No newline at end of file diff --git a/parallax/point_mesh.py b/parallax/point_mesh.py index 42f590e..342c99e 100644 --- a/parallax/point_mesh.py +++ b/parallax/point_mesh.py @@ -30,6 +30,9 @@ def __init__(self, model, file_path, sn, calib_completed=False): self.resizeEvent = self._on_resize self._init_ui() + # Register this instance with the model + self.model.add_point_mesh_instance(self) + def show(self): self._parse_csv() self._init_buttons() @@ -171,4 +174,10 @@ def _on_resize(self, event): self._update_canvas() # Resize horizontal layout - self.ui.horizontalLayoutWidget.resize(new_size.width(), new_size.height()) \ No newline at end of file + self.ui.horizontalLayoutWidget.resize(new_size.width(), new_size.height()) + + def closeEvent(self, event): + if self in self.model.point_mesh_instances: + self.model.point_mesh_instances.remove(self) + self.web_view.close() + event.accept() \ No newline at end of file From 9ffb98890dcf42137dda4fae6fea382d0f326e88 Mon Sep 17 00:00:00 2001 From: hannalee2 Date: Thu, 8 Aug 2024 11:31:18 -0700 Subject: [PATCH 11/18] Display only the most recent PointMesh instance for a specific sn if user click 'plot 3d' button multiple times. --- parallax/model.py | 9 ++++++--- parallax/point_mesh.py | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/parallax/model.py b/parallax/model.py index 13aab75..f3e74db 100755 --- a/parallax/model.py +++ b/parallax/model.py @@ -27,7 +27,7 @@ def __init__(self, version="V1", bundle_adjustment=False): self.focos = [] # point mesh - self.point_mesh_instances = [] + self.point_mesh_instances = {} # stage self.nStages = 0 @@ -204,9 +204,12 @@ def save_all_camera_frames(self): self.msg_log.post("Saved camera frame: %s" % filename) def add_point_mesh_instance(self, instance): - self.point_mesh_instances.append(instance) + sn = instance.sn + if sn in self.point_mesh_instances: + self.point_mesh_instances[sn].close() + self.point_mesh_instances[sn] = instance def close_all_point_meshes(self): - for instance in self.point_mesh_instances: + for instance in self.point_mesh_instances.values(): instance.close() self.point_mesh_instances.clear() \ No newline at end of file diff --git a/parallax/point_mesh.py b/parallax/point_mesh.py index 342c99e..4b74378 100644 --- a/parallax/point_mesh.py +++ b/parallax/point_mesh.py @@ -178,6 +178,6 @@ def _on_resize(self, event): def closeEvent(self, event): if self in self.model.point_mesh_instances: - self.model.point_mesh_instances.remove(self) + del self.model.point_mesh_instances[self.sn] self.web_view.close() event.accept() \ No newline at end of file From 121a68b0677cff3a0545b6ea9ef64fcad75fa642 Mon Sep 17 00:00:00 2001 From: hannalee2 Date: Fri, 9 Aug 2024 10:45:34 -0700 Subject: [PATCH 12/18] 3D visualization is done. If probe calib is done, reuses the instances to shorten the drawing time. Only one plot shows for each calib. --- parallax/main_window_wip.py | 1 - parallax/model.py | 3 +-- parallax/point_mesh.py | 37 +++++++++++++++++------------------ parallax/probe_calibration.py | 37 +++++++++++++++++------------------ 4 files changed, 37 insertions(+), 41 deletions(-) diff --git a/parallax/main_window_wip.py b/parallax/main_window_wip.py index 2e52d17..a5016e0 100644 --- a/parallax/main_window_wip.py +++ b/parallax/main_window_wip.py @@ -773,7 +773,6 @@ def save_user_configs(self): height = self.height() self.user_setting.save_user_configs(nColumn, directory, width, height) - def closeEvent(self, event): self.model.close_all_point_meshes() event.accept() \ No newline at end of file diff --git a/parallax/model.py b/parallax/model.py index f3e74db..9c6deaf 100755 --- a/parallax/model.py +++ b/parallax/model.py @@ -3,7 +3,6 @@ """ from PyQt5.QtCore import QObject, pyqtSignal - from .camera import MockCamera, PySpinCamera, close_cameras, list_cameras from .stage_listener import Stage, StageInfo @@ -205,7 +204,7 @@ def save_all_camera_frames(self): def add_point_mesh_instance(self, instance): sn = instance.sn - if sn in self.point_mesh_instances: + if sn in self.point_mesh_instances.keys(): self.point_mesh_instances[sn].close() self.point_mesh_instances[sn] = instance diff --git a/parallax/point_mesh.py b/parallax/point_mesh.py index 4b74378..d9c006b 100644 --- a/parallax/point_mesh.py +++ b/parallax/point_mesh.py @@ -12,7 +12,7 @@ csv_file = os.path.join(debug_dir, "points.csv") class PointMesh(QWidget): - def __init__(self, model, file_path, sn, calib_completed=False): + def __init__(self, model, file_path, sn, transM, scale, transM_BA=None, scale_BA=None, calib_completed=False): super().__init__() self.setWindowFlags(Qt.Window | Qt.WindowMinimizeButtonHint | \ Qt.WindowMaximizeButtonHint | Qt.WindowCloseButtonHint) @@ -20,6 +20,7 @@ def __init__(self, model, file_path, sn, calib_completed=False): self.file_path = file_path self.sn = sn self.calib_completed = calib_completed + self.web_view = None self.R, self.R_BA = {}, {} self.T, self.T_BA = {}, {} @@ -28,23 +29,30 @@ def __init__(self, model, file_path, sn, calib_completed=False): self.traces = {} # Plotly trace objects self.colors = {} self.resizeEvent = self._on_resize - self._init_ui() - + # Register this instance with the model self.model.add_point_mesh_instance(self) - def show(self): + self.ui = loadUi(os.path.join(ui_dir, "point_mesh.ui"), self) + self._set_transM(transM, scale) + if transM_BA is not None and scale_BA is not None and \ + self.model.bundle_adjustment and self.calib_completed: + self.set_transM_BA(transM_BA, scale_BA) self._parse_csv() self._init_buttons() + + def show(self): + self._init_ui() self._update_canvas() super().show() # Show the widget def _init_ui(self): - self.ui = loadUi(os.path.join(ui_dir, "point_mesh.ui"), self) + if self.web_view is not None: + self.web_view.close() self.web_view = QWebEngineView(self) self.ui.verticalLayout1.addWidget(self.web_view) - def set_transM(self, transM, scale): + def _set_transM(self, transM, scale): self.R[self.sn] = transM[:3, :3] self.T[self.sn] = transM[:3, 3] self.S[self.sn] = scale[:3] @@ -88,6 +96,7 @@ def _local_to_global(self, local_pts, R, t, scale=None): def _init_buttons(self): self.buttons = {} + for key in self.points_dict.keys(): button_name = self._get_button_name(key) button = QPushButton(f'{button_name}') @@ -161,13 +170,6 @@ def _update_canvas(self): html_content = fig.to_html(include_plotlyjs='cdn') self.web_view.setHtml(html_content) - def wheelEvent(self, event): - # Get the mouse position - mouse_position = event.pos() - # Apply zoom based on the scroll direction - scale_factor = 0.9 if event.angleDelta().y() > 0 else 1.1 - self._zoom(scale_factor, mouse_position.x(), mouse_position.y()) - def _on_resize(self, event): new_size = event.size() self.web_view.resize(new_size.width(), new_size.height()) @@ -175,9 +177,6 @@ def _on_resize(self, event): # Resize horizontal layout self.ui.horizontalLayoutWidget.resize(new_size.width(), new_size.height()) - - def closeEvent(self, event): - if self in self.model.point_mesh_instances: - del self.model.point_mesh_instances[self.sn] - self.web_view.close() - event.accept() \ No newline at end of file + + + diff --git a/parallax/probe_calibration.py b/parallax/probe_calibration.py index c474096..65ac153 100644 --- a/parallax/probe_calibration.py +++ b/parallax/probe_calibration.py @@ -58,9 +58,9 @@ def __init__(self, model, stage_listener): self.df = None self.inliers = [] self.stage = None - """ + self.threshold_min_max = 250 - self.threshold_min_max_z = 200 + self.threshold_min_max_z = 0 self.LR_err_L2_threshold = 200 self.threshold_avg_error = 500 self.threshold_matrix = np.array( @@ -84,7 +84,7 @@ def __init__(self, model, stage_listener): [0.0, 0.0, 0.0, 0.0], ] ) - + """ self.model_LR, self.transM_LR, self.transM_LR_prev = None, None, None self.origin, self.R, self.scale = None, None, np.array([1, 1, 1]) @@ -598,11 +598,6 @@ def update(self, stage, debug_info=None): # update points in the file self.stage = stage self._update_local_global_point(debug_info) # Do no update if it is duplicates - - # TODO - # get whole list of local and global points in pd format - #local_points, global_points = self._get_local_global_points() - #self.transM_LR = self._get_transM_LR_orthogonal(local_points, global_points) #remove outliers filtered_df = self._filter_df_by_sn(self.stage.sn) self.transM_LR = self._get_transM(filtered_df) @@ -628,14 +623,8 @@ def complete_calibration(self, filtered_df): self._update_info_ui(disp_avg_error=True, save_to_csv=True, \ file_name = f"transM_{self.stage.sn}.csv") - # Init point_mesh - self.stages[self.stage.sn]['calib_completed'] = True - if hasattr(self, 'point_mesh_not_clibed'): - del self.point_mesh_not_calibrated - self.point_mesh[self.stage.sn] = PointMesh(self.model, self.file_name, self.stage.sn, calib_completed=True) - self.point_mesh[self.stage.sn].set_transM(self.transM_LR, self.scale) # Set transM - if self.model.bundle_adjustment: + self.old_transM, self.old_scale = self.transM_LR, self.scale ret = self.run_bundle_adjustment(self.file_name) if ret: print("\n=========================================================") @@ -644,23 +633,33 @@ def complete_calibration(self, filtered_df): print("=========================================================") self._update_info_ui(disp_avg_error=True, save_to_csv=True, \ file_name = f"transM_BA_{self.stage.sn}.csv") - self.point_mesh[self.stage.sn].set_transM_BA(self.transM_LR, self.scale) # set TransM_BA else: return - + # Emit the signal to indicate that calibration is complete self.calib_complete.emit(self.stage.sn, self.transM_LR, self.scale) logger.debug( f"complete probe calibration {self.stage.sn}, {self.transM_LR}, {self.scale}" ) + # Init PointMesh + if not self.model.bundle_adjustment: + self.point_mesh[self.stage.sn] = PointMesh(self.model, self.file_name, self.stage.sn, \ + self.transM_LR, self.scale, calib_completed=True) + else: + self.point_mesh[self.stage.sn] = PointMesh(self.model, self.file_name, self.stage.sn, \ + self.old_transM, self.old_scale, \ + self.transM_LR, self.scale, calib_completed=True) + self.stages[self.stage.sn]['calib_completed'] = True + def view_3d_trajectory(self, sn): if not self.stages.get(sn, {}).get('calib_completed', False): if sn == self.stage.sn: - self.point_mesh_not_calibrated = PointMesh(self.model, self.csv_file, self.stage.sn) - self.point_mesh_not_calibrated.set_transM(self.transM_LR, self.scale) + self.point_mesh_not_calibrated = PointMesh(self.model, self.csv_file, self.stage.sn, \ + self.transM_LR, self.scale) self.point_mesh_not_calibrated.show() else: + # If calib is completed, show the PointMesh instance. self.point_mesh[sn].show() def run_bundle_adjustment(self, file_path): From a97b8d53bef7fc0097da79ac51e4fc3dba1b6adf Mon Sep 17 00:00:00 2001 From: hannalee2 Date: Fri, 9 Aug 2024 10:46:56 -0700 Subject: [PATCH 13/18] Revert back to calib finish criteria --- parallax/probe_calibration.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/parallax/probe_calibration.py b/parallax/probe_calibration.py index 65ac153..ee67f52 100644 --- a/parallax/probe_calibration.py +++ b/parallax/probe_calibration.py @@ -58,7 +58,7 @@ def __init__(self, model, stage_listener): self.df = None self.inliers = [] self.stage = None - + """ self.threshold_min_max = 250 self.threshold_min_max_z = 0 self.LR_err_L2_threshold = 200 @@ -84,8 +84,7 @@ def __init__(self, model, stage_listener): [0.0, 0.0, 0.0, 0.0], ] ) - """ - + self.model_LR, self.transM_LR, self.transM_LR_prev = None, None, None self.origin, self.R, self.scale = None, None, np.array([1, 1, 1]) self.avg_err = None From 2f1a1e3b4303d4e2c1eace6ac352de9e1d7722cc Mon Sep 17 00:00:00 2001 From: hannalee2 Date: Fri, 9 Aug 2024 10:47:09 -0700 Subject: [PATCH 14/18] Update the release version --- parallax/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parallax/__init__.py b/parallax/__init__.py index 67bd916..02208d9 100644 --- a/parallax/__init__.py +++ b/parallax/__init__.py @@ -4,7 +4,7 @@ import os -__version__ = "0.37.20" +__version__ = "0.37.21" # allow multiple OpenMP instances os.environ["KMP_DUPLICATE_LIB_OK"] = "True" From 8949bf761a6591f7cd0c3d85fd9b70d8e598f522 Mon Sep 17 00:00:00 2001 From: hannalee2 Date: Fri, 9 Aug 2024 11:11:13 -0700 Subject: [PATCH 15/18] Reduce the init size of plot pop-up window to fit into lab monitor resolution --- ui/point_mesh.ui | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui/point_mesh.ui b/ui/point_mesh.ui index 720ee16..4299064 100644 --- a/ui/point_mesh.ui +++ b/ui/point_mesh.ui @@ -6,8 +6,8 @@ 0 0 - 1691 - 1040 + 1400 + 850 From b5f057ddbb8c3012654b31a79033e41feb4aed65 Mon Sep 17 00:00:00 2001 From: hannalee2 Date: Fri, 9 Aug 2024 11:20:28 -0700 Subject: [PATCH 16/18] Add the window title --- parallax/point_mesh.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/parallax/point_mesh.py b/parallax/point_mesh.py index d9c006b..ecf0f75 100644 --- a/parallax/point_mesh.py +++ b/parallax/point_mesh.py @@ -14,8 +14,6 @@ class PointMesh(QWidget): def __init__(self, model, file_path, sn, transM, scale, transM_BA=None, scale_BA=None, calib_completed=False): super().__init__() - self.setWindowFlags(Qt.Window | Qt.WindowMinimizeButtonHint | \ - Qt.WindowMaximizeButtonHint | Qt.WindowCloseButtonHint) self.model = model self.file_path = file_path self.sn = sn @@ -34,6 +32,10 @@ def __init__(self, model, file_path, sn, transM, scale, transM_BA=None, scale_BA self.model.add_point_mesh_instance(self) self.ui = loadUi(os.path.join(ui_dir, "point_mesh.ui"), self) + self.setWindowTitle(f"{self.sn} - Trajectory 3D View ") + self.setWindowFlags(Qt.Window | Qt.WindowMinimizeButtonHint | \ + Qt.WindowMaximizeButtonHint | Qt.WindowCloseButtonHint) + self._set_transM(transM, scale) if transM_BA is not None and scale_BA is not None and \ self.model.bundle_adjustment and self.calib_completed: From 3d5a29d41bb8c396931d37ddeec57d3fede1d473 Mon Sep 17 00:00:00 2001 From: hannalee2 Date: Fri, 9 Aug 2024 13:25:10 -0700 Subject: [PATCH 17/18] Update the debug level --- parallax/axis_filter.py | 2 +- parallax/bundle_adjustment.py | 2 +- parallax/probe_calibration.py | 2 +- parallax/probe_detect_manager.py | 19 ++++++++++--------- parallax/screen_widget.py | 2 +- parallax/stage_widget.py | 2 +- 6 files changed, 15 insertions(+), 14 deletions(-) diff --git a/parallax/axis_filter.py b/parallax/axis_filter.py index ffcbd64..b55ca27 100644 --- a/parallax/axis_filter.py +++ b/parallax/axis_filter.py @@ -14,7 +14,7 @@ # Set logger name logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) +logger.setLevel(logging.WARNING) class AxisFilter(QObject): """Class representing no filter.""" diff --git a/parallax/bundle_adjustment.py b/parallax/bundle_adjustment.py index 526fe47..340c244 100644 --- a/parallax/bundle_adjustment.py +++ b/parallax/bundle_adjustment.py @@ -6,7 +6,7 @@ # Set logger name logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) +logger.setLevel(logging.WARNING) class BALProblem: def __init__(self, model, file_path): diff --git a/parallax/probe_calibration.py b/parallax/probe_calibration.py index ee67f52..d6b0cb5 100644 --- a/parallax/probe_calibration.py +++ b/parallax/probe_calibration.py @@ -17,7 +17,7 @@ # Set logger name logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) +logger.setLevel(logging.WARNING) # Set the logging level for PyQt5.uic.uiparser/properties to WARNING, to ignore DEBUG messages logging.getLogger("PyQt5.uic.uiparser").setLevel(logging.WARNING) logging.getLogger("PyQt5.uic.properties").setLevel(logging.WARNING) diff --git a/parallax/probe_detect_manager.py b/parallax/probe_detect_manager.py index 5e83036..4ef2388 100644 --- a/parallax/probe_detect_manager.py +++ b/parallax/probe_detect_manager.py @@ -19,7 +19,7 @@ # Set logger name logger = logging.getLogger(__name__) -logger.setLevel(logging.WARNING) +logger.setLevel(logging.DEBUG) # Set the logging level for PyQt5.uic.uiparser/properties to WARNING, to ignore DEBUG messages logging.getLogger("PyQt5.uic.uiparser").setLevel(logging.WARNING) logging.getLogger("PyQt5.uic.properties").setLevel(logging.WARNING) @@ -43,7 +43,7 @@ def __init__(self, name, model): """Initialize Worker object""" QObject.__init__(self) self.model = model - self.name = name + self.name = name # Camera serial number self.running = False self.is_detection_on = False self.is_calib = False @@ -138,9 +138,8 @@ def process(self, frame, timestamp): self.currBgCmpProcess.update_reticle_zone(self.reticle_zone) if self.prev_img is not None: - if ( - self.probeDetect.angle is None - ): # Detecting probe for the first time + if self.probeDetect.angle is None: + # Detecting probe for the first time ret = self.currPrevCmpProcess.first_cmp( self.curr_img, self.prev_img, mask, gray_img ) @@ -163,6 +162,7 @@ def process(self, frame, timestamp): ret = self.currBgCmpProcess.update_cmp( self.curr_img, mask, gray_img ) + logger.debug(f"cam:{self.name}, ret: {ret}, stopped: {self.is_calib}") if ret: # Found if self.is_calib: # If calibaration is enable, use data for calibration self.found_coords.emit( @@ -181,7 +181,7 @@ def process(self, frame, timestamp): (255, 0, 0), -1, ) - else: # Otherwise, just draw a tip on the frame + else: # Otherwise, just draw a tip on the frame (yellow) cv2.circle( frame, self.probeDetect.probe_tip_org, @@ -190,9 +190,10 @@ def process(self, frame, timestamp): -1, ) self.prev_img = self.curr_img - logger.debug(f"{self.name} Found") - else: - logger.debug(f"{self.name} Not found") + #logger.debug(f"{self.name} Found") + else: + #logger.debug(f"{self.name} Not found") + pass else: self.prev_img = self.curr_img diff --git a/parallax/screen_widget.py b/parallax/screen_widget.py index 5077bc4..82f224b 100755 --- a/parallax/screen_widget.py +++ b/parallax/screen_widget.py @@ -19,7 +19,7 @@ # Set logger name logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) +logger.setLevel(logging.WARNING) # Set the logging level for PyQt5.uic.uiparser/properties to WARNING, to ignore DEBUG messages logging.getLogger("PyQt5.uic.uiparser").setLevel(logging.WARNING) logging.getLogger("PyQt5.uic.properties").setLevel(logging.WARNING) diff --git a/parallax/stage_widget.py b/parallax/stage_widget.py index 3222eed..8a4b574 100644 --- a/parallax/stage_widget.py +++ b/parallax/stage_widget.py @@ -21,7 +21,7 @@ from .stage_ui import StageUI logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) +logger.setLevel(logging.WARNING) class StageWidget(QWidget): """Widget for stage control and calibration.""" From be8d7af7ed0e5786d3b4b28f21fb4d825dba3bc9 Mon Sep 17 00:00:00 2001 From: hannalee2 Date: Fri, 9 Aug 2024 13:25:45 -0700 Subject: [PATCH 18/18] Update the debug level --- parallax/probe_detect_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parallax/probe_detect_manager.py b/parallax/probe_detect_manager.py index 4ef2388..003282c 100644 --- a/parallax/probe_detect_manager.py +++ b/parallax/probe_detect_manager.py @@ -19,7 +19,7 @@ # Set logger name logger = logging.getLogger(__name__) -logger.setLevel(logging.DEBUG) +logger.setLevel(logging.WARNING) # Set the logging level for PyQt5.uic.uiparser/properties to WARNING, to ignore DEBUG messages logging.getLogger("PyQt5.uic.uiparser").setLevel(logging.WARNING) logging.getLogger("PyQt5.uic.properties").setLevel(logging.WARNING)