update sema and configs

.gitignore

@@ -136,3 +136,4 @@ configs/selfsup_IP89
 *.toml
 openmixup/models/classifiers/backup
 work_dirs_EXP
+configs/regression_IP89

configs/selfsup/_base_/datasets/cifar10/simsiam_sz224_bs64.py

@@ -39,11 +39,14 @@
 # dataset summary
 data = dict(
     imgs_per_gpu=64,  # V100: 64 x 8gpus x 8 accumulates = bs4096
-    workers_per_gpu=6,  # according to total cpus cores, usually 4 workers per 32~128 imgs
+    workers_per_gpu=4,  # according to total cpus cores, usually 4 workers per 32~128 imgs
     train=dict(
         type=dataset_type,
         data_source=dict(split='train', return_label=False, **data_source_cfg),
         num_views=[1, 1],
         pipelines=[train_pipeline1, train_pipeline2],
         prefetch=prefetch,
     ))
+
+# checkpoint
+checkpoint_config = dict(interval=10, max_keep_ckpts=1)

configs/selfsup/_base_/datasets/cifar100/deepcluster_sz224_bs64.py

@@ -1,7 +1,7 @@
 _base_ = '../cifar10/deepcluster_sz224_bs64.py'
 
 # dataset settings
-img_norm_cfg = dict(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
+img_norm_cfg = dict(mean=[0.4914, 0.4822, 0.4465], std=[0.2023, 0.1994, 0.201])
 data_source_cfg = dict(type='CIFAR100', root='data/cifar100/')
 
 # dataset summary

configs/selfsup/_base_/datasets/cifar100/metrics_knn_svm_umap.py

@@ -4,7 +4,7 @@
     dict(type='Resize', size=256),
     dict(type='CenterCrop', size=224),
     dict(type='ToTensor'),
-    dict(type='Normalize', mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+    dict(type='Normalize', mean=[0.4914, 0.4822, 0.4465], std=[0.2023, 0.1994, 0.201]),
 ]
 val_data = dict(
     train=dict(

configs/selfsup/_base_/datasets/cifar100/metrics_umap.py

@@ -4,7 +4,7 @@
     dict(type='Resize', size=256),
     dict(type='CenterCrop', size=224),
     dict(type='ToTensor'),
-    dict(type='Normalize', mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+    dict(type='Normalize', mean=[0.4914, 0.4822, 0.4465], std=[0.2023, 0.1994, 0.201]),
 ]
 val_data = dict(
     train=dict(

configs/selfsup/_base_/datasets/stl10/byol_sz96_bs256.py

@@ -0,0 +1,59 @@
+# dataset settings
+data_source_cfg = dict(type='ImageNet', return_label=False)
+# ImageNet dataset
+data_train_list = 'data/meta/STL10/train_10w_unlabeled.txt'
+data_train_root = 'data/stl10/train/'
+data_test_list = 'data/meta/STL10/test_8k_unlabeled.txt'
+data_test_root = 'data/stl10/test/'
+
+dataset_type = 'MultiViewDataset'
+img_norm_cfg = dict(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
+train_pipeline1 = [
+    dict(type='RandomResizedCrop', size=96, interpolation=3),  # bicubic
+    dict(type='RandomHorizontalFlip'),
+    dict(type='RandomAppliedTrans',
+        transforms=[dict(
+            type='ColorJitter',
+            brightness=0.4, contrast=0.4, saturation=0.2, hue=0.1)
+        ],
+        p=0.8),
+    dict(type='RandomGrayscale', p=0.2),
+    dict(type='GaussianBlur', sigma_min=0.1, sigma_max=2.0, p=1.),
+    dict(type='Solarization', p=0.),
+]
+train_pipeline2 = [
+    dict(type='RandomResizedCrop', size=96, interpolation=3),  # bicubic
+    dict(type='RandomHorizontalFlip'),
+    dict(type='RandomAppliedTrans',
+        transforms=[
+            dict(type='ColorJitter',
+                brightness=0.4, contrast=0.4, saturation=0.2, hue=0.1)
+        ],
+        p=0.8),
+    dict(type='RandomGrayscale', p=0.2),
+    dict(type='GaussianBlur', sigma_min=0.1, sigma_max=2.0, p=0.1),
+    dict(type='Solarization', p=0.2),
+]
+
+# prefetch
+prefetch = True
+if not prefetch:
+    train_pipeline1.extend([dict(type='ToTensor'), dict(type='Normalize', **img_norm_cfg)])
+    train_pipeline2.extend([dict(type='ToTensor'), dict(type='Normalize', **img_norm_cfg)])
+
+# dataset summary
+data = dict(
+    imgs_per_gpu=256,  # V100: 256 x 8gpus x 2 accumulates = bs4096
+    workers_per_gpu=8,  # according to total cpus cores, usually 4 workers per 32~128 imgs
+    train=dict(
+        type=dataset_type,
+        data_source=dict(
+            list_file=data_train_list, root=data_train_root,
+            **data_source_cfg),
+        num_views=[1, 1],
+        pipelines=[train_pipeline1, train_pipeline2],
+        prefetch=prefetch,
+    ))
+
+# checkpoint
+checkpoint_config = dict(interval=50, max_keep_ckpts=1)

configs/selfsup/_base_/datasets/stl10/mocov1_sz96_bs64.py

@@ -0,0 +1,39 @@
+# dataset settings
+data_source_cfg = dict(type='ImageNet', return_label=False)
+# ImageNet dataset
+data_train_list = 'data/meta/STL10/train_10w_unlabeled.txt'
+data_train_root = 'data/stl10/train/'
+data_test_list = 'data/meta/STL10/test_8k_unlabeled.txt'
+data_test_root = 'data/stl10/test/'
+
+dataset_type = 'MultiViewDataset'
+img_norm_cfg = dict(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
+train_pipeline = [
+    dict(type='RandomResizedCrop', size=96, scale=(0.2, 1.)),
+    dict(type='RandomHorizontalFlip'),
+    dict(type='ColorJitter',
+        brightness=0.4, contrast=0.4, saturation=0.4, hue=0.4),
+]
+
+# prefetch
+prefetch = True
+if not prefetch:
+    train_pipeline.extend([dict(type='ToTensor'), dict(type='Normalize', **img_norm_cfg)])
+
+# dataset summary
+data = dict(
+    imgs_per_gpu=64,
+    workers_per_gpu=4,
+    drop_last=True,
+    train=dict(
+        type=dataset_type,
+        data_source=dict(
+            list_file=data_train_list, root=data_train_root,
+            **data_source_cfg),
+        num_views=[2],
+        pipelines=[train_pipeline],
+        prefetch=prefetch,
+    ))
+
+# checkpoint
+checkpoint_config = dict(interval=10, max_keep_ckpts=1)

configs/selfsup/_base_/datasets/stl10/mocov2_sz96_bs64.py

@@ -0,0 +1,45 @@
+# dataset settings
+data_source_cfg = dict(type='ImageNet', return_label=False)
+# ImageNet dataset
+data_train_list = 'data/meta/STL10/train_10w_unlabeled.txt'
+data_train_root = 'data/stl10/train/'
+data_test_list = 'data/meta/STL10/test_8k_unlabeled.txt'
+data_test_root = 'data/stl10/test/'
+
+dataset_type = 'MultiViewDataset'
+img_norm_cfg = dict(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
+train_pipeline = [
+    dict(type='RandomResizedCrop', size=96, scale=(0.2, 1.)),
+    dict(type='RandomHorizontalFlip'),
+    dict(type='RandomAppliedTrans',
+        transforms=[dict(
+            type='ColorJitter',
+            brightness=0.4, contrast=0.4, saturation=0.4, hue=0.1)
+        ],
+        p=0.8),
+    dict(type='RandomGrayscale', p=0.2),
+    dict(type='GaussianBlur', sigma_min=0.1, sigma_max=2.0, p=0.5),
+]
+
+# prefetch
+prefetch = True
+if not prefetch:
+    train_pipeline.extend([dict(type='ToTensor'), dict(type='Normalize', **img_norm_cfg)])
+
+# dataset summary
+data = dict(
+    imgs_per_gpu=64,
+    workers_per_gpu=4,
+    drop_last=True,
+    train=dict(
+        type=dataset_type,
+        data_source=dict(
+            list_file=data_train_list, root=data_train_root,
+            **data_source_cfg),
+        num_views=[2],
+        pipelines=[train_pipeline],
+        prefetch=prefetch,
+    ))
+
+# checkpoint
+checkpoint_config = dict(interval=10, max_keep_ckpts=1)

configs/selfsup/_base_/datasets/stl10/simclr_sz96_bs256.py

@@ -0,0 +1,44 @@
+# dataset settings
+data_source_cfg = dict(type='ImageNet', return_label=False)
+# ImageNet dataset
+data_train_list = 'data/meta/STL10/train_10w_unlabeled.txt'
+data_train_root = 'data/stl10/train/'
+data_test_list = 'data/meta/STL10/test_8k_unlabeled.txt'
+data_test_root = 'data/stl10/test/'
+
+dataset_type = 'MultiViewDataset'
+img_norm_cfg = dict(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
+train_pipeline = [
+    dict(type='RandomResizedCrop', size=96),
+    dict(type='RandomHorizontalFlip'),
+    dict(type='RandomAppliedTrans',
+        transforms=[dict(
+            type='ColorJitter',
+            brightness=0.8, contrast=0.8, saturation=0.8, hue=0.2)
+        ],
+        p=0.8),
+    dict(type='RandomGrayscale', p=0.2),
+    dict(type='GaussianBlur', sigma_min=0.1, sigma_max=2.0, p=0.5),
+]
+
+# prefetch
+prefetch = True
+if not prefetch:
+    train_pipeline.extend([dict(type='ToTensor'), dict(type='Normalize', **img_norm_cfg)])
+
+# dataset summary
+data = dict(
+    imgs_per_gpu=256,  # V100: 256 x 8gpus x 2 accumulates = bs4096
+    workers_per_gpu=8,  # according to total cpus cores, usually 4 workers per 32~128 imgs
+    train=dict(
+        type=dataset_type,
+        data_source=dict(
+            list_file=data_train_list, root=data_train_root,
+            **data_source_cfg),
+        num_views=[2],
+        pipelines=[train_pipeline],
+        prefetch=prefetch,
+    ))
+
+# checkpoint
+checkpoint_config = dict(interval=10, max_keep_ckpts=1)

configs/selfsup/byol/cifar10/r18_8xb256_accu2_cos_lr4_8_knn_svm_umap_fp16_ep1000.py

@@ -6,7 +6,7 @@
     dict(type='Resize', size=256),
     dict(type='CenterCrop', size=224),
     dict(type='ToTensor'),
-    dict(type='Normalize', mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+    dict(type='Normalize', mean=[0.4914, 0.4822, 0.4465], std=[0.2023, 0.1994, 0.201]),
 ]
 val_data = dict(
     train=dict(

configs/selfsup/byol/cifar100/r18_8xb256_accu2_cos_lr4_8_knn_svm_umap_fp16_ep1000.py

@@ -6,7 +6,7 @@
     dict(type='Resize', size=256),
     dict(type='CenterCrop', size=224),
     dict(type='ToTensor'),
-    dict(type='Normalize', mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+    dict(type='Normalize', mean=[0.4914, 0.4822, 0.4465], std=[0.2023, 0.1994, 0.201]),
 ]
 val_data = dict(
     train=dict(

configs/selfsup/byol/stl10/r50_8xb256_accu2_cos_lr4_8_fp16_ep1000.py

@@ -0,0 +1,42 @@
+_base_ = [
+    '../../_base_/models/byol/r50.py',
+    '../../_base_/datasets/stl10/byol_sz96_bs256.py',
+    '../../_base_/default_runtime.py',
+]
+
+# dataset
+data = dict(imgs_per_gpu=256, workers_per_gpu=8)
+
+# interval for accumulate gradient
+update_interval = 2  # total: 8 x bs256 x 2 accumulates = bs4096
+
+# optimizer
+optimizer = dict(
+    type='LARS',
+    lr=4.8,  # lr=4.8 / bs4096 for longer training
+    momentum=0.9, weight_decay=1e-6,
+    paramwise_options={
+        '(bn|ln|gn)(\d+)?.(weight|bias)': dict(weight_decay=0., lars_exclude=True),
+        'bias': dict(weight_decay=0., lars_exclude=True),
+    })
+
+# fp16
+use_fp16 = True
+fp16 = dict(type='mmcv', loss_scale='dynamic')
+# optimizer args
+optimizer_config = dict(update_interval=update_interval, grad_clip=None)
+
+# lr scheduler
+lr_config = dict(
+    policy='CosineAnnealing',
+    by_epoch=False, min_lr=0.,
+    warmup='linear',
+    warmup_iters=10, warmup_by_epoch=True,
+    warmup_ratio=1e-5,
+)
+
+# log, 50k / 4096
+log_config = dict(interval=49)
+
+# runtime settings
+runner = dict(type='EpochBasedRunner', max_epochs=1000)

configs/selfsup/deepcluster/cifar10/r18_8xb64_step_knn_svm_umap_ep1000.py

@@ -1,7 +1,7 @@
 _base_ = 'r18_8xb64_step_ep1000.py'
 
 # dataset settings for SSL metrics
-val_data_source_cfg = dict(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
+val_data_source_cfg = dict(mean=[0.4914, 0.4822, 0.4465], std=[0.2023, 0.1994, 0.201])
 data_source_cfg = dict(type='CIFAR10', root='data/cifar10/')
 
 val_test_pipeline = [

configs/selfsup/deepcluster/cifar100/r18_8xb64_step_knn_svm_umap_ep1000.py

@@ -1,7 +1,7 @@
 _base_ = 'r18_8xb64_step_ep1000.py'
 
 # dataset settings for SSL metrics
-val_data_source_cfg = dict(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
+val_data_source_cfg = dict(mean=[0.4914, 0.4822, 0.4465], std=[0.2023, 0.1994, 0.201])
 data_source_cfg = dict(type='CIFAR100', root='data/cifar100/')
 
 val_test_pipeline = [

configs/selfsup/mocov2/cifar10/r18_4xb64_cos_umap_knn_ep1000.py

@@ -0,0 +1,46 @@
+_base_ = 'r18_4xb64_cos_ep1000.py'
+
+# dataset settings for SSL metrics
+val_data_source_cfg = dict(type='CIFAR10', root='data/cifar10/')
+test_pipeline = [
+    dict(type='Resize', size=256),
+    dict(type='CenterCrop', size=224),
+    dict(type='ToTensor'),
+    dict(type='Normalize', mean=[0.4914, 0.4822, 0.4465], std=[0.2023, 0.1994, 0.201]),
+]
+val_data = dict(
+    train=dict(
+        type='ClassificationDataset',
+        data_source=dict(split='train', **val_data_source_cfg),
+        pipeline=test_pipeline,
+        prefetch=False,
+    ),
+    val=dict(
+        type='ClassificationDataset',
+        data_source=dict(split='test', **val_data_source_cfg),
+        pipeline=test_pipeline,
+        prefetch=False,
+    ))
+
+# interval for accumulate gradient
+update_interval = 1
+
+# additional hooks
+custom_hooks = [
+    dict(type='SSLMetricHook',
+        val_dataset=val_data['val'],
+        train_dataset=val_data['train'],  # remove it if metric_mode is None
+        forward_mode='vis',
+        metric_mode=['knn', 'svm',],  # linear metric (take a bit long time on imagenet)
+        metric_args=dict(
+            knn=200, temperature=0.07, chunk_size=256,
+            dataset='onehot', costs_list="0.01,0.1,1.0,10.0,100.0", default_cost=None, num_workers=8,),
+        visual_mode='umap',  # 'tsne' or 'umap'
+        visual_args=dict(n_epochs=400, plot_backend='seaborn'),
+        save_val=False,  # whether to save results
+        initial=True,
+        interval=50,
+        imgs_per_gpu=256,
+        workers_per_gpu=4,
+        eval_param=dict(topk=(1, 5))),
+]