init

.gitignore

@@ -0,0 +1,12 @@
+/.vscode
+/visualized_data/*
+/libcity/tmp/*
+/libcity/__pycache__
+/libcity/*/__pycache__
+/libcity/data/dataset/__pycache__
+/.idea
+/libcity/log/*
+/libcity/model/*/__pycache__
+/libcity/data/dataset/*/__pycache__
+/libcity/cache/model_cache/*
+/libcity/cache/evaluate_cache/*

CHIBike.json

@@ -0,0 +1,26 @@
+{
+    "dataset_class": "PDFormerGridDataset",
+    "input_window": 6,
+    "output_window": 1,
+    "train_rate": 0.7,
+    "eval_rate": 0.1,
+    "batch_size": 16,
+    "add_time_in_day": true,
+    "add_day_in_week": true,
+    "use_row_column": false,
+    "lr_warmup_epoch": 15,
+    "max_epoch": 200,
+    "embed_dim": 32,
+    "far_mask_delta": 3,
+    "geo_num_heads": 2,
+    "sem_num_heads": 2,
+    "t_num_heads": 4,
+    "cluster_method": "kshape",
+    "cand_key_days": 14,
+    "seed": 1,
+    "type_ln": "pre",
+    "set_loss": "huber",
+    "huber_delta": 2,
+    "mask_val": 5,
+    "mode": "average"
+}

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 aptx1231
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

NYCTaxi.json

@@ -0,0 +1,24 @@
+{
+    "dataset_class": "PDFormerGridDataset",
+    "input_window": 6,
+    "output_window": 1,
+    "train_rate": 0.7,
+    "eval_rate": 0.1,
+    "batch_size": 16,
+    "add_time_in_day": true,
+    "add_day_in_week": true,
+    "use_row_column": false,
+    "max_epoch": 200,
+    "far_mask_delta": 3,
+    "geo_num_heads": 2,
+    "sem_num_heads": 2,
+    "t_num_heads": 4,
+    "cluster_method": "kshape",
+    "cand_key_days": 14,
+    "seed": 2,
+    "type_ln": "pre",
+    "set_loss": "huber",
+    "huber_delta": 2,
+    "mask_val": 10,
+    "mode": "average"
+}

PeMS04.json

@@ -0,0 +1,24 @@
+{
+    "dataset_class": "PDFormerDataset",
+    "input_window": 12,
+    "output_window": 12,
+    "train_rate": 0.6,
+    "eval_rate": 0.2,
+    "batch_size": 16,
+    "add_time_in_day": true,
+    "add_day_in_week": true,
+    "step_size": 1274,
+    "max_epoch": 200,
+    "bidir": true,
+    "far_mask_delta": 7,
+    "geo_num_heads": 4,
+    "sem_num_heads": 2,
+    "t_num_heads": 2,
+    "cluster_method": "kshape",
+    "cand_key_days": 14,
+    "seed": 1,
+    "type_ln": "pre",
+    "set_loss": "huber",
+    "huber_delta": 2,
+    "mode": "average"
+}

PeMS07.json

@@ -0,0 +1,25 @@
+{
+    "dataset_class": "PDFormerDataset",
+    "input_window": 12,
+    "output_window": 12,
+    "train_rate": 0.6,
+    "eval_rate": 0.2,
+    "batch_size": 8,
+    "grad_accmu_steps": 2,
+    "add_time_in_day": true,
+    "add_day_in_week": true,
+    "step_size": 4232,
+    "max_epoch": 200,
+    "bidir": true,
+    "far_mask_delta": 7,
+    "geo_num_heads": 4,
+    "sem_num_heads": 2,
+    "t_num_heads": 2,
+    "cluster_method": "kshape",
+    "cand_key_days": 14,
+    "seed": 1,
+    "type_ln": "pre",
+    "set_loss": "huber",
+    "huber_delta": 2,
+    "mode": "average"
+}

PeMS08.json

@@ -0,0 +1,24 @@
+{
+    "dataset_class": "PDFormerDataset",
+    "input_window": 12,
+    "output_window": 12,
+    "train_rate": 0.6,
+    "eval_rate": 0.2,
+    "batch_size": 16,
+    "add_time_in_day": true,
+    "add_day_in_week": true,
+    "step_size": 2776,
+    "max_epoch": 200,
+    "bidir": true,
+    "far_mask_delta": 7,
+    "geo_num_heads": 4,
+    "sem_num_heads": 2,
+    "t_num_heads": 2,
+    "cluster_method": "kshape",
+    "cand_key_days": 21,
+    "seed": 1,
+    "type_ln": "pre",
+    "set_loss": "huber",
+    "huber_delta": 2,
+    "mode": "average"
+}

T-Drive.json

@@ -0,0 +1,25 @@
+{
+    "dataset_class": "PDFormerGridDataset",
+    "input_window": 6,
+    "output_window": 1,
+    "train_rate": 0.7,
+    "eval_rate": 0.1,
+    "batch_size": 16,
+    "add_time_in_day": true,
+    "add_day_in_week": true,
+    "use_row_column": false,
+    "far_mask_delta": 3,
+    "geo_num_heads": 2,
+    "sem_num_heads": 2,
+    "t_num_heads": 4,
+    "cluster_method": "kshape",
+    "cand_key_days": 14,
+    "seed": 42,
+    "max_epoch": 200,
+    "type_ln": "pre",
+    "drop_path": 0,
+    "set_loss": "huber",
+    "huber_delta": 2,
+    "mask_val": 10,
+    "mode": "average"
+}

libcity/config/__init__.py

@@ -0,0 +1,5 @@
+from libcity.config.config_parser import ConfigParser
+
+__all__ = [
+    'ConfigParser'
+]

libcity/config/config_parser.py

@@ -0,0 +1,134 @@
+import os
+import json
+import torch
+
+
+class ConfigParser(object):
+
+    def __init__(self, task, model, dataset, config_file=None,
+                 saved_model=True, train=True, other_args=None, hyper_config_dict=None, initial_ckpt=None):
+        self.config = {}
+        self._parse_external_config(task, model, dataset, saved_model, train, other_args, hyper_config_dict, initial_ckpt)
+        self._parse_config_file(config_file)
+        self._load_default_config()
+        self._init_device()
+
+    def _parse_external_config(self, task, model, dataset,
+                               saved_model=True, train=True, other_args=None, hyper_config_dict=None, initial_ckpt=None):
+        if task is None:
+            raise ValueError('the parameter task should not be None!')
+        if model is None:
+            raise ValueError('the parameter model should not be None!')
+        if dataset is None:
+            raise ValueError('the parameter dataset should not be None!')
+        self.config['task'] = task
+        self.config['model'] = model
+        self.config['dataset'] = dataset
+        self.config['saved_model'] = saved_model
+        self.config['train'] = False if task == 'map_matching' else train
+        if other_args is not None:
+            for key in other_args:
+                self.config[key] = other_args[key]
+        if hyper_config_dict is not None:
+            for key in hyper_config_dict:
+                self.config[key] = hyper_config_dict[key]
+        self.config['initial_ckpt'] = initial_ckpt
+
+    def _parse_config_file(self, config_file):
+        if config_file is not None:
+            if os.path.exists('./{}.json'.format(config_file)):
+                with open('./{}.json'.format(config_file), 'r') as f:
+                    x = json.load(f)
+                    for key in x:
+                        if key not in self.config:
+                            self.config[key] = x[key]
+            else:
+                raise FileNotFoundError(
+                    'Config file {}.json is not found. Please ensure \
+                    the config file is in the root dir and is a JSON \
+                    file.'.format(config_file))
+
+    def _load_default_config(self):
+        with open('./libcity/config/task_config.json', 'r') as f:
+            task_config = json.load(f)
+            if self.config['task'] not in task_config:
+                raise ValueError(
+                    'task {} is not supported.'.format(self.config['task']))
+            task_config = task_config[self.config['task']]
+            if self.config['model'] not in task_config['allowed_model']:
+                raise ValueError('task {} do not support model {}'.format(
+                    self.config['task'], self.config['model']))
+            model = self.config['model']
+            if 'dataset_class' not in self.config:
+                self.config['dataset_class'] = task_config[model]['dataset_class']
+            if self.config['task'] == 'traj_loc_pred' and 'traj_encoder' not in self.config:
+                self.config['traj_encoder'] = task_config[model]['traj_encoder']
+            if 'executor' not in self.config:
+                self.config['executor'] = task_config[model]['executor']
+            if 'evaluator' not in self.config:
+                self.config['evaluator'] = task_config[model]['evaluator']
+            if self.config['model'].upper() in ['LSTM', 'GRU', 'RNN']:
+                self.config['rnn_type'] = self.config['model']
+                self.config['model'] = 'RNN'
+        default_file_list = []
+        default_file_list.append('model/{}/{}.json'.format(self.config['task'], self.config['model']))
+        default_file_list.append('data/{}.json'.format(self.config['dataset_class']))
+        default_file_list.append('executor/{}.json'.format(self.config['executor']))
+        default_file_list.append('evaluator/{}.json'.format(self.config['evaluator']))
+        for file_name in default_file_list:
+            with open('./libcity/config/{}'.format(file_name), 'r') as f:
+                x = json.load(f)
+                for key in x:
+                    if key not in self.config:
+                        self.config[key] = x[key]
+        with open('./raw_data/{}/config.json'.format(self.config['dataset']), 'r') as f:
+            x = json.load(f)
+            for key in x:
+                if key == 'info':
+                    for ik in x[key]:
+                        if ik not in self.config:
+                            self.config[ik] = x[key][ik]
+                else:
+                    if key not in self.config:
+                        self.config[key] = x[key]
+
+    def _init_device(self):
+        use_gpu = self.config.get('gpu', True)
+        distributed = False
+        if 'WORLD_SIZE' in os.environ:
+            distributed = int(os.environ['WORLD_SIZE']) > 1
+        self.config['distributed'] = distributed
+        if use_gpu and distributed:
+            local_rank = self.config['local_rank']
+            assert local_rank >= 0
+            torch.cuda.set_device(local_rank)
+            torch.distributed.init_process_group(backend='nccl', init_method='env://')
+            rank = torch.distributed.get_rank()
+            self.config["rank"] = rank
+            assert rank >= 0
+            self.config['world_size'] = torch.distributed.get_world_size()
+            self.config['device'] = torch.device(
+                "cuda:%d" % local_rank if torch.cuda.is_available() else "cpu")
+        else:
+            if use_gpu:
+                torch.cuda.set_device(0)
+            self.config['device'] = torch.device(
+                "cuda:0" if torch.cuda.is_available() and use_gpu else "cpu")
+
+    def get(self, key, default=None):
+        return self.config.get(key, default)
+
+    def __getitem__(self, key):
+        if key in self.config:
+            return self.config[key]
+        else:
+            raise KeyError('{} is not in the config'.format(key))
+
+    def __setitem__(self, key, value):
+        self.config[key] = value
+
+    def __contains__(self, key):
+        return key in self.config
+
+    def __iter__(self):
+        return self.config.__iter__()

libcity/config/data/PDFormerDataset.json

@@ -0,0 +1,17 @@
+{
+    "batch_size": 64,
+    "cache_dataset": true,
+    "num_workers": 0,
+    "pad_with_last_sample": true,
+    "train_rate": 0.7,
+    "eval_rate": 0.1,
+    "scaler": "none",
+    "load_external": false,
+    "normal_external": false,
+    "ext_scaler": "none",
+    "input_window": 12,
+    "output_window": 12,
+    "add_time_in_day": false,
+    "add_day_in_week": false,
+    "lape_dim": 8
+}

libcity/config/data/PDFormerGridDataset.json

@@ -0,0 +1,19 @@
+{
+    "batch_size": 64,
+    "cache_dataset": true,
+    "num_workers": 0,
+    "pad_with_last_sample": true,
+    "train_rate": 0.7,
+    "eval_rate": 0.1,
+    "scaler": "none",
+    "load_external": false,
+    "normal_external": false,
+    "ext_scaler": "none",
+    "input_window": 6,
+    "output_window": 1,
+    "output_dim": 2,
+    "add_time_in_day": false,
+    "add_day_in_week": false,
+    "use_row_column": false,
+    "lape_dim": 8
+}

libcity/config/data/TrafficStateDataset.json

@@ -0,0 +1,16 @@
+{
+  "batch_size": 64,
+  "cache_dataset": true,
+  "num_workers": 0,
+  "pad_with_last_sample": true,
+  "train_rate": 0.7,
+  "eval_rate": 0.1,
+  "scaler": "none",
+  "load_external": false,
+  "normal_external": false,
+  "ext_scaler": "none",
+  "input_window": 12,
+  "output_window": 12,
+  "add_time_in_day": false,
+  "add_day_in_week": false
+}