diff --git a/cuda_11.3_env b/cuda_11.3_env new file mode 100644 index 00000000..803c4b8a --- /dev/null +++ b/cuda_11.3_env @@ -0,0 +1,10 @@ +CUDAVER=cuda-11.3 +CUDA_ABS_PATH=/usr/local/$CUDAVER +export PATH=$CUDA_ABS_PATH/bin:$PATH +export LD_LIBRARY_PATH=$CUDA_ABS_PATH/lib:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=$CUDA_ABS_PATH/lib64:$LD_LIBRARY_PATH +export CUDADIR=$CUDA_ABS_PATH +export CUDA_PATH=$CUDA_ABS_PATH +export CUDA_ROOT=$CUDA_ABS_PATH +export CUDA_HOME=$CUDA_ABS_PATH +export CUDA_HOST_COMPILER=/usr/bin/gcc diff --git a/docs/GETTING_STARTED.md b/docs/GETTING_STARTED.md index 060456c8..1f849e24 100644 --- a/docs/GETTING_STARTED.md +++ b/docs/GETTING_STARTED.md @@ -9,20 +9,28 @@ We describe how to prepare the other datasets in the [preparing external dataset ### Campus Object Dataset (CODa) -Please download the official CODa using the [dataset development kit](https://github.com/ut-amrl/coda-devkit) -and link it in the data directory in this repo as follows. You may need to symlink the `3d_raw` directory -to the `3d_comp` directory if you downloaded CODa by split rather than by sequence. - +Please download the official CODa using the [dataset development kit](https://github.com/ut-amrl/coda-devkit) with the command: +```bashrc +python scripts/download_split.py -d data/coda -t split --split full ``` +Link it in the data directory `coda_original_format` in this repo as follows. You may need to symlink the `3d_raw` directory to the `3d_comp` directory. + +Next, download the `2d_bbox` folder from the [url](https://web.corral.tacc.utexas.edu/texasrobotics/web_CODa/CODa_models/). Unzip and arrange as follows: + +```bash coda-models ├── data -│ ├── coda_original_format +│ └── coda_original_format +│ ├── 2d_bbox │ ├── 2d_rect -| ├── 3d_raw +│ ├── 3d_bbox +│ ├── 3d_comp +│ ├── 3d_raw # soft-link to 3d_comp +│ ├── 3d_semantic │ ├── calibrations │ ├── metadata │ ├── poses -| ├── timestamps +│ └── timestamps ├── pcdet ├── tools ``` @@ -32,33 +40,64 @@ coda-models python tools/create_data.py coda ``` +You should now see the following file structure: + +```bash +coda-models +├── data +│ ├── coda_original_format +│ │ ├── 2d_bbox +│ │ ├── 2d_rect +│ │ ├── 3d_bbox +│ │ ├── 3d_comp +│ │ ├── 3d_raw # soft-link to 3d_comp +│ │ ├── 3d_semantic +│ │ ├── calibrations +│ │ ├── metadata +│ │ ├── poses +│ │ └── timestamps +│ │ +│ └── coda128_allclass_full +│ ├── ImageSets +│ ├── testing +│ └── training +├── pcdet +├── tools +``` + + * Generate the data infos by running the following command: ```python python -m pcdet.datasets.coda.coda_dataset create_coda_infos tools/cfgs/dataset_configs/da_coda_oracle_dataset_full.yaml ``` -You should now see the following file structure. +You should now see the following file structure: -``` +```bash coda-models ├── data │ ├── coda_original_format -│ ├── 2d_rect -| ├── 3d_raw -│ ├── calibrations -│ ├── metadata -│ ├── poses -| ├── timestamps -│ ├── coda128_allclass_full +│ │ ├── 2d_bbox +│ │ ├── 2d_rect +│ │ ├── 3d_bbox +│ │ ├── 3d_comp +│ │ ├── 3d_raw # soft-link to 3d_comp +│ │ ├── 3d_semantic +│ │ ├── calibrations +│ │ ├── metadata +│ │ ├── poses +│ │ └── timestamps +│ │ +│ └── coda128_allclass_full │ ├── gt_database -| ├── ImageSets +│ ├── ImageSets │ ├── testing │ ├── training │ ├── coda_dbinfos_train.pkl -| ├── coda_infos_test.pkl -| ├── coda_infos_train.pkl -| ├── coda_infos_trainval.pkl -| ├── coda_infos_val.pkl +│ ├── coda_infos_test.pkl +│ ├── coda_infos_train.pkl +│ ├── coda_infos_trainval.pkl +│ └── coda_infos_val.pkl ├── pcdet ├── tools ``` diff --git a/docs/INSTALL.md b/docs/INSTALL.md index d66f63b7..b4345e17 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -45,7 +45,19 @@ pip install spconv-cu113 pip install -r requirements.txt ``` -c. Install this repository +c. Link to cuda_11.3 on your machine and use correct GCC/G++ + +```bashrc +source cuda_11.3_env + +sudo apt-get install gcc-9 g++-9 -y +sudo ln -sfn /usr/bin/gcc-9 /usr/bin/gcc +sudo ln -sfn /usr/bin/gcc-9 /usr/bin/cc +sudo ln -sfn /usr/bin/g++-9 /usr/bin/g++ +sudo ln -sfn /usr/bin/g++-9 /usr/bin/c++ +``` + +d. Install this repository ``` python setup.py develop diff --git a/environment.yml b/environment.yml index b60d611c..9f86ec8f 100644 --- a/environment.yml +++ b/environment.yml @@ -196,7 +196,6 @@ dependencies: - stack-data==0.6.2 - tensorboardx==2.6.2.2 - threadpoolctl==3.2.0 - - tifffile==2023.8.30 - tqdm==4.66.1 - traitlets==5.9.0 - tzdata==2023.3 diff --git a/tools/create_data.py b/tools/create_data.py index 62224bf9..201085a6 100644 --- a/tools/create_data.py +++ b/tools/create_data.py @@ -1,4 +1,6 @@ # Copyright (c) OpenMMLab. All rights reserved. +import os, sys +sys.path.append(os.getcwd()) import argparse from os import path as osp @@ -35,8 +37,8 @@ def coda_data_prep(root_path, split=split, channels=channels ) + print("CODa Split= {} Length = {}".format(split, len(converter))) converter.convert() - print("length dataset ", len(converter)) parser = argparse.ArgumentParser(description='Data converter arg parser') parser.add_argument('dataset', metavar='coda', help='name of the dataset') diff --git a/tools/data_converter/coda_converter.py b/tools/data_converter/coda_converter.py index 8e19bcbf..cec58c25 100644 --- a/tools/data_converter/coda_converter.py +++ b/tools/data_converter/coda_converter.py @@ -210,6 +210,15 @@ def __init__(self, 'cam1': 1, 'os1': 2 } + # CODA Camera to KITTI Camera + # 0 is 2 + # 1 is 3 + # 2 is assumed 4 + self.coda_to_kitti_id = { + 0: 2, + 1: 3, + 2: 4 + } # Used to downsample lidar vertical channels self.channels = channels @@ -232,7 +241,7 @@ def process_metadata(self): label_list = meta_json["ObjectTracking"][self.split] self.bbox_label_files.extend(label_list) - lidar_list = [label_path.replace('3d_label', '3d_raw').replace('.json', '.bin') + lidar_list = [label_path.replace('3d_label', '3d_comp').replace('.json', '.bin') for label_path in label_list] self.lidar_files.extend(lidar_list) @@ -285,10 +294,10 @@ def get_filename_info(filename): @staticmethod def set_filename_by_prefix(modality, sensor_name, trajectory, frame): if "2d_rect"==modality: - filetype = "jpg" # change to jpg later + filetype = "png" # change to jpg later elif "2d_bbox"==modality: filetype = "txt" - elif "3d_raw"==modality: + elif "3d_comp"==modality or "3d_raw"==modality: filetype = "bin" elif "3d_bbox"==modality: filetype = "json" @@ -381,7 +390,7 @@ def save_image(self, traj, src_img_path, cam_id, frame_idx, file_idx): frame_idx (int): Current frame index. """ assert isfile(src_img_path), "Image file does not exist: %s" % src_img_path - kitti_img_path = f'{self.image_save_dir}{str(cam_id)}/' + \ + kitti_img_path = f'{self.image_save_dir}{str(self.coda_to_kitti_id[cam_id])}/' + \ f'{str(traj).zfill(2)}{str(frame_idx).zfill(5)}.jpg' shutil.copyfile(src_img_path, kitti_img_path) @@ -424,10 +433,18 @@ def save_calib(self, traj, frame_idx, file_idx): for cam_id in self.cam_ids: calib_context += 'P' + str(cam_id) + ': ' + \ ' '.join(camera_calibs[cam_id]) + '\n' + # Also map cam_id to KITTI camera ids + for cam_id in self.cam_ids: + calib_context += 'P' + str(self.coda_to_kitti_id[cam_id]) + ': ' + \ + ' '.join(camera_calibs[cam_id]) + '\n' calib_context += 'R0_rect' + ': ' + ' '.join(R0_rect) + '\n' for cam_id in self.cam_ids: - calib_context += 'Tr_velo_to_cam_' + str(cam_id) + ': ' + \ - ' '.join(Tr_os1_to_cams[cam_id]) + '\n' + if cam_id == 0: + calib_context += 'Tr_velo_to_cam' + ': ' + \ + ' '.join(Tr_os1_to_cams[cam_id]) + '\n' + else: + calib_context += 'Tr_velo_to_cam_' + str(cam_id) + ': ' + \ + ' '.join(Tr_os1_to_cams[cam_id]) + '\n' with open( f'{self.calib_save_dir}/' + @@ -442,7 +459,7 @@ def save_lidar(self, traj, frame_idx, file_idx, channels=128): traj (int): Current trajectory index. frame_idx (int): Current frame index. """ - bin_file = self.set_filename_by_prefix("3d_raw", "os1", traj, frame_idx) + bin_file = self.set_filename_by_prefix("3d_comp", "os1", traj, frame_idx) bin_path = join(self.load_dir, "3d_raw", "os1", traj, bin_file) assert isfile(bin_path), "Bin file for traj %s frame %s does not exist: %s" % (traj, frame_idx, bin_path) point_cloud = np.fromfile(bin_path, dtype=np.float32).reshape(-1, 4) @@ -554,7 +571,7 @@ def save_label(self, traj, cam_id, frame_idx, file_idx): line_all = line[:-1] + '\n' fp_label = open( - f'{self.label_save_dir}{cam_id}/' + + f'{self.label_save_dir}{self.coda_to_kitti_id[cam_id]}/' + f'{str(traj).zfill(2)}{str(frame_idx).zfill(5)}.txt', 'a') fp_label.write(line) fp_label.close() diff --git a/tools/generate_coda_image_depth_map.py b/tools/generate_coda_image_depth_map.py new file mode 100644 index 00000000..523db7b6 --- /dev/null +++ b/tools/generate_coda_image_depth_map.py @@ -0,0 +1,151 @@ +""" + Sample Run: + python test/.py +""" +import os, sys +sys.path.append(os.getcwd()) + +import os.path as osp +import glob +import numpy as np +np.set_printoptions (precision= 2, suppress= True) + +import imageio +import cv2 +from collections import Counter +from matplotlib import pyplot as plt + + +def get_calib_from_file(calib_file): + with open(calib_file) as f: + lines = f.readlines() + + obj = lines[2].strip().split(' ')[1:] + P2 = np.array(obj, dtype=np.float32) + obj = lines[3].strip().split(' ')[1:] + P3 = np.array(obj, dtype=np.float32) + obj = lines[4].strip().split(' ')[1:] + R0 = np.array(obj, dtype=np.float32) + obj = lines[5].strip().split(' ')[1:] + Tr_velo_to_cam = np.array(obj, dtype=np.float32) + + return {'P2': P2.reshape(3, 4), + 'P3': P3.reshape(3, 4), + 'R0': R0.reshape(3, 3), + 'Tr_velo2cam': Tr_velo_to_cam.reshape(3, 4)} + + +def load_velodyne_points(filename): + """Load 3D point cloud from KITTI file format + (adapted from https://github.com/hunse/kitti) + """ + points = np.fromfile(filename, dtype=np.float32).reshape(-1, 4) + points[:, 3] = 1.0 # homogeneous + return points + +def sub2ind(matrixSize, rowSub, colSub): + """Convert row, col matrix subscripts to linear indices + """ + m, n = matrixSize + return rowSub * (n-1) + colSub - 1 + +def generate_depth_map(calib_filename, velo_filename, cam=2, vel_depth=False): + """Generate a depth map from velodyne data + https://github.com/nianticlabs/monodepth2/blob/master/kitti_utils.py + """ + # # load calibration files + # cam2cam = read_calib_file(os.path.join(calib_dir, 'calib_cam_to_cam.txt')) + # velo2cam = read_calib_file(os.path.join(calib_dir, 'calib_velo_to_cam.txt')) + # velo2cam = np.hstack((velo2cam['R'].reshape(3, 3), velo2cam['T'][..., np.newaxis])) + # velo2cam = np.vstack((velo2cam, np.array([0, 0, 0, 1.0]))) + # + # # get image shape + # im_shape = cam2cam["S_rect_02"][::-1].astype(np.int32) + # + # # compute projection matrix velodyne->image plane + # R_cam2rect = np.eye(4) + # R_cam2rect[:3, :3] = cam2cam['R_rect_00'].reshape(3, 3) + # P_rect = cam2cam['P_rect_0'+str(cam)].reshape(3, 4) + # P_velo2im = np.dot(np.dot(P_rect, R_cam2rect), velo2cam) + + # width, height from file command in linux + # Here we write as height, width + im_shape = [1024, 1224] + + calib = get_calib_from_file(calib_filename) + P2 = np.eye(4) + P2[:3, :] = calib['P2'] + R0 = np.eye(4) + R0[:3, :3] = calib['R0'] + Tr_velo2cam = np.eye(4) + Tr_velo2cam[:3, :] = calib['Tr_velo2cam'] + + # x = P2 * R0_rect * Tr_velo_to_cam * y + # https://github.com/abhi1kumar/groomed_nms/blob/main/data/kitti_split1/devkit/readme.txt#L96 + P_velo2im = P2 @ R0 @ Tr_velo2cam + + # load velodyne points and remove all behind image plane (approximation) + # each row of the velodyne data is forward, left, up, reflectance + velo = load_velodyne_points(velo_filename) + velo = velo[velo[:, 0] >= 0, :] + + # project the points to the camera + velo_pts_im = np.dot(P_velo2im, velo.T).T + velo_pts_im[:, :2] = velo_pts_im[:, :2] / velo_pts_im[:, 2][..., np.newaxis] + + if vel_depth: + velo_pts_im[:, 2] = velo[:, 0] + + # check if in bounds + # use minus 1 to get the exact same value as KITTI matlab code + velo_pts_im[:, 0] = np.round(velo_pts_im[:, 0]) - 1 + velo_pts_im[:, 1] = np.round(velo_pts_im[:, 1]) - 1 + val_inds = (velo_pts_im[:, 0] >= 0) & (velo_pts_im[:, 1] >= 0) + val_inds = val_inds & (velo_pts_im[:, 0] < im_shape[1]) & (velo_pts_im[:, 1] < im_shape[0]) + velo_pts_im = velo_pts_im[val_inds, :] + + # project to image + depth = np.zeros((im_shape[:2])) + depth[velo_pts_im[:, 1].astype(np.int64), velo_pts_im[:, 0].astype(np.int64)] = velo_pts_im[:, 2] + + # find the duplicate points and choose the closest depth + inds = sub2ind(depth.shape, velo_pts_im[:, 1], velo_pts_im[:, 0]) + dupe_inds = [item for item, count in Counter(inds).items() if count > 1] + for dd in dupe_inds: + pts = np.where(inds == dd)[0] + x_loc = int(velo_pts_im[pts[0], 0]) + y_loc = int(velo_pts_im[pts[0], 1]) + depth[y_loc, x_loc] = velo_pts_im[pts, 2].min() + depth[depth < 0] = 0 + + return depth + +if __name__ == '__main__': + base_folder = "data/coda/" + vmin= 0 + vmax= 50 + cmap= 'magma_r' + + for split in ["testing", "training"]: + calib_folder = osp.join(base_folder, split, "calib") + print(calib_folder) + calib_files = sorted(glob.glob(calib_folder + "/*.txt")) + + num_files = len(calib_files) + output_folder = calib_folder.replace("calib", "depth_image") + os.makedirs(output_folder, exist_ok= True) + + print("{} files found".format(num_files)) + + for i, calib_file in enumerate(calib_files): + velo_file = calib_file.replace("calib", "velodyne").replace(".txt", ".bin") + gt_depth = generate_depth_map(calib_file, velo_file) # h x w + + output_file = osp.join(output_folder, osp.basename(calib_file).replace(".txt", ".png")) + cv2.imwrite(output_file, gt_depth) + + # plt.imshow(gt_depth, vmin= vmin, vmax= vmax, cmap= cmap) + # plt.show() + + if (i + 1) % 500 == 0 or (i + 1) == num_files: + print("{} images done.".format(i+1)) \ No newline at end of file