test

.gitignore

@@ -0,0 +1,32 @@
+datasets/gt_3Dmodels/
+datasets/imgs_datasets/
+datasets/__pycache__/
+detectors/line2d/__pycache__/
+detectors/line2d/DeepLSD/__pycache__/
+detectors/line2d/LSD/__pycache__/
+detectors/point2d/__pycache__/
+detectors/point2d/SuperPoint/__pycache__/
+detectors/point2d/SuperPoint/weights/
+util/__pycache__/
+visualization/
+visualization_all/
+logs/
+experiments/
+
+*.npy
+*.png
+*.jpg
+*.pyc
+*.tar
+*.obj
+*.pth
+*.gif
+*.mp4
+*.h5
+*.swp
+*.zip
+*.tar.gz
+*.th
+*.so
+
+.vscode

README.md

@@ -4,8 +4,15 @@
 
 > Representing 3D sparse map points and lines for camera relocalization                                                                                                                                                
 > [Bach-Thuan Bui](https://thuanbb.github.io/), [Huy-Hoang Bui](https://github.com/AustrianOakvn), [Dinh-Tuan Tran](https://sites.google.com/view/tuantd), [Joo-Ho Lee](https://research-db.ritsumei.ac.jp/rithp/k03/resid/S000220;jsessionid=8CC0520A8C7C1F3D502596F0A07D64B0?lang=en)                   
-> 2024 IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS), 
+> 2024 IEEE/RSJ International Conference on Intelligent Robots and Systems (IROS) 
 
 ![demo_vid](assets/demo.gif)
 
-### Code is coming soon!
+## Installation
+```
+git submodule update --init --recursive
+
+# Refer to https://pytorch.org/get-started/previous-versions/ to install pytorch compatible with your CUDA
+python -m pip install torch==1.12.0 torchvision==0.13.0 
+python -m pip install -r requirements.txt
+```

cfgs/7scenes.yaml

@@ -0,0 +1,82 @@
+line2d:
+  max_num_2d_segs: 3000
+  do_merge_lines: False # Not implemented
+  visualize: False
+  save_l3dpp: False
+  detector:
+    name: "deeplsd" # ["lsd", "sold2", "deeplsd"] - for loading labeled 3D model
+    name_test_model: "deeplsd" # ["lsd", "sold2", "deeplsd"] - for testing (not depending on pre-3D model)
+  preprocessing:
+    grayscale: True
+
+point2d:
+  detector: 
+    name: "superpoint" # ["superpoint", "sift"]
+    configs:
+      force_num_keypoints: False
+      nms_radius: 3
+      max_keypoints: 2048
+    preprocessing:
+      grayscale: True
+      resize_max: 1024
+      resize_force: True
+      interpolation: 'cv2_area'  # pil_linear is more accurate but slower
+  matcher: "NN-superpoint" # ["superglue", "gluestick"] # not implemented (for unlabeled learning)
+
+regressor: 
+  name: pl2map # ["pl2map", "d2s"]
+  use_line: True
+  use_point: True
+  n_line_keypoints: 10 # number of keypoints used to represent a line
+
+train: # train configs
+  batch_size: 1
+  num_iters: 2500000 # number training iterations
+  loader_shuffle: True
+  loader_num_workers: 8
+  log_interval: 500 # log every n batches (visdom graph)
+  use_depth: False # use SfM corrected by depth or not
+  loss:
+    reprojection:
+      apply: True
+      type: "dyntanh" # ["l1", "l1+sqrt", "l1+log", "tanh", "dyntanh"]
+      soft_clamp: 50
+      soft_clamp_min: 1
+      circle_schedule: True # 'circle'(weight increasing) or 'linear' (weight decreasing)
+  augmentation:
+    apply: True
+    on_rate: 0.5 # probability of applying augmentation
+    brightness: 0.02
+    contrast: 0.02
+    homography:
+      apply: False  # if apply, augmented poses will be incorrect
+      perspective: True
+      scaling: True 
+      rotation: True
+      translation: True
+      n_scales: 5
+      n_angles: 25
+      scaling_amplitude: 0.1
+      perspective_amplitude_x: 0.1
+      perspective_amplitude_y: 0.1
+      patch_ratio: 0.8 # ratio of the patch to the image
+      max_angle: 45 # in degrees
+      allow_artifacts: False
+    dsacstar: # apply DSAC*-like augmentation
+      apply: True # homography augmentation must be disabled
+      aug_rotation: 30 # in degrees
+      aug_scale_min: 0.666666666 # 2/3
+      aug_scale_max: 1.5  # 3/2
+
+
+optimizer:
+  method: adam
+  base_lr: 0.0003 # base/start learning rate
+  weight_decay: 0.0
+  lr_decay: 0.5 # decay rate
+  num_lr_decay_step: 7 # decay every n epochs
+
+localization:
+  ransac: 
+    max_reproj_error: 12.0 
+    max_epipolar_error: 10.0

cfgs/Cambridge.yaml

@@ -0,0 +1,96 @@
+line2d:
+  max_num_2d_segs: 3000
+  do_merge_lines: False # Not implemented
+  visualize: False
+  save_l3dpp: False
+  detector:
+    name: "deeplsd" # ["lsd", "sold2", "deeplsd"] - for loading labeled 3D model
+    name_test_model: "deeplsd" # ["lsd", "sold2", "deeplsd"] - for testing (not depending on pre-3D model)
+  preprocessing:
+    grayscale: True
+
+point2d:
+  detector: 
+    name: "superpoint" # ["superpoint", "sift"]
+    configs:
+      force_num_keypoints: False
+      nms_radius: 3
+      max_keypoints: 2048
+    preprocessing:
+      grayscale: True
+      resize_max: 1024
+      resize_force: True
+      interpolation: 'cv2_area'  # pil_linear is more accurate but slower
+  matcher: "NN-superpoint" # ["superglue", "gluestick"] # not implemented (for unlabeled learning)
+
+regressor: 
+  name: pl2map # ["pl2map", "d2s"]
+  use_line: True
+  use_point: True
+  n_line_keypoints: 10 # number of keypoints used to represent a line
+
+train: # train configs
+  batch_size: 1
+  num_iters: 1500000 # number training iterations
+  loader_shuffle: True
+  loader_num_workers: 8
+  log_interval: 500 # log every n batches (visdom graph)
+  loss:
+    reprojection:
+      apply: True
+      start_apply: 0.8 # start applying reprojection loss
+      type: "dyntanh" # ["l1", "l1+sqrt", "l1+log", "tanh", "dyntanh"]
+      soft_clamp: 100
+      soft_clamp_min: 1
+      circle_schedule: True # 'circle'(weight increasing) or 'linear' (weight decreasing)
+  augmentation:
+    apply: True
+    on_rate: 0.5 # probability of applying augmentation
+    brightness: 0.0
+    contrast: 0.0
+    homography:
+      apply: False  # if apply, augmented poses will be incorrect
+      perspective: True
+      scaling: True 
+      rotation: True
+      translation: True
+      n_scales: 5
+      n_angles: 25
+      scaling_amplitude: 0.1
+      perspective_amplitude_x: 0.1
+      perspective_amplitude_y: 0.1
+      patch_ratio: 0.8 # ratio of the patch to the image
+      max_angle: 45 # in degrees
+      allow_artifacts: False
+    dsacstar: # apply DSAC*-like augmentation
+      apply: True # homography augmentation must be disabled
+      aug_rotation: 30 # in degrees
+      aug_scale_min: 0.666666666 # 2/3
+      aug_scale_max: 1.5  # 3/2
+
+
+optimizer:
+  method: adam
+  base_lr: 0.0005 # base/start learning rate
+  weight_decay: 0.0
+  lr_decay: 0.5 # decay rate
+  num_lr_decay_step: 7 # decay every n epochs, 7 
+
+localization:
+  2d_matcher: "sold2" 
+  epipolar_filter: False
+  IoU_threshold: 0.2
+  reprojection_filter: null 
+  ransac: 
+    method: "hybrid"   # [null, "hybrid"] null for pose refinement, hybrid for pose estimation
+    thres: 10 # ??
+    thres_point: 10 # ??
+    thres_line: 10 # ??
+    weight_point: 1.0   # data type weights for scoring
+    weight_line: 1.0    # data type weights for scoring
+    final_least_squares: True
+    min_num_iterations: 100
+  optimize:
+    loss_func: "HuberLoss"
+    loss_func_args: [1.0]
+  line_cost_func: "PerpendicularDist"

cfgs/default.yaml

@@ -0,0 +1,93 @@
+line2d:
+  max_num_2d_segs: 3000
+  do_merge_lines: False # Not implemented
+  visualize: False
+  save_l3dpp: False
+  detector:
+    name: "deeplsd" # ["lsd", "sold2", "deeplsd"] - for loading labeled 3D model
+    name_test_model: "deeplsd" # ["lsd", "sold2", "deeplsd"] - for testing (not depending on pre-3D model)
+  preprocessing:
+    grayscale: True
+
+point2d:
+  detector: 
+    name: "superpoint" # ["superpoint", "sift"]
+    configs:
+      force_num_keypoints: False
+      nms_radius: 3
+      max_keypoints: 2048
+    preprocessing:
+      grayscale: True
+      resize_max: 1024
+      resize_force: True
+      interpolation: 'cv2_area'  # pil_linear is more accurate but slower
+  matcher: "NN-superpoint" # ["superglue", "gluestick"] # not implemented (for unlabeled learning)
+
+regressor: 
+  name: pl2map # ["pl2map", "d2s"]
+  use_line: True
+  use_point: True
+  n_line_keypoints: 10 # number of keypoints used to represent a line
+
+train: # train configs
+  batch_size: 1
+  num_iters: 2500000 # number training iterations
+  loader_shuffle: True
+  loader_num_workers: 8
+  log_interval: 500 # log every n batches (visdom graph)
+  use_depth: False # use SfM corrected by depth or not
+  loss:
+    reprojection:
+      apply: False
+      type: "dyntanh" # ["l1", "l1+sqrt", "l1+log", "tanh", "dyntanh"]
+      soft_clamp: 50
+      soft_clamp_min: 1
+      circle_schedule: True # 'circle'(weight increasing) or 'linear' (weight decreasing)
+  augmentation:
+    apply: False
+    on_rate: 0.5 # probability of applying augmentation
+    brightness: 0.1
+    contrast: 0.1
+    homography:
+      apply: False  # if apply, augmented poses will be incorrect
+      perspective: True
+      scaling: True 
+      rotation: True
+      translation: True
+      n_scales: 5
+      n_angles: 25
+      scaling_amplitude: 0.1
+      perspective_amplitude_x: 0.1
+      perspective_amplitude_y: 0.1
+      patch_ratio: 0.8 # ratio of the patch to the image
+      max_angle: 45 # in degrees
+      allow_artifacts: False
+    dsacstar: # apply DSAC*-like augmentation
+      apply: False # homography augmentation must be disabled
+      aug_rotation: 30 # in degrees
+      aug_scale_min: 0.666666666 # 2/3
+      aug_scale_max: 1.5  # 3/2
+
+localization:
+  2d_matcher: "sold2" # ["epipolar", "sold2", "superglue_endpoints"] Other configs for superglue_endpoints are the same as in "line2d" section
+  epipolar_filter: False # No use for epipolar matcher
+  IoU_threshold: 0.2
+  reprojection_filter: null  # [null, "Perpendicular", "Midpoint", "Midpoint_Perpendicular"]
+  ransac: 
+    method: "hybrid"    # [null, "ransac", "solver", "hybrid"]
+    thres: 10.0         # Only for normal & solver
+    thres_point: 10.0
+    thres_line: 10.0
+    weight_point: 1.0   # data type weights for scoring
+    weight_line: 1.0    # data type weights for scoring
+    final_least_squares: True
+    min_num_iterations: 100
+    solver_flags: [True, True, True, True]
+  optimize:
+    loss_func: "TrivialLoss"
+    loss_func_args: []
+  line_cost_func: "PerpendicularDist"
+  line_weight: 1.0        # weight for optimization (cost function)
+  hloc:
+    skip_exists: False
+  skip_exists: False