kd_config.json

{
    "name": "koelectra_kd_model",
    "n_gpu": 1,

    "model": {
        "type": "BeepKcElectraHateModel",
        "args": {
            "name": "monologg/koelectra-small-v3-discriminator",
            "num_classes": 3
        }
    },
    "teacher_model": {
        "type": "BeepKcElectraHateModel",
        "args": {
            "name": "beomi/beep-KcELECTRA-base-hate",
            "num_classes": 3
        }
    },
    "tokenizer": {
        "student": {
            "type": "monologg/koelectra-small-v3-discriminator"
        },
        "teacher": {
            "type": "tokenizer/"
        }
    },
    "data_loader": {
        "type": "MnistDataLoader",
        "args":{
            "data_dir": "AI-it/korean-hate-speech",
            "batch_size": 64,
            "max_length": 64,
            "shuffle": true,
            "validation_split": 0.1,
            "num_workers": 2
        },
        "data_files": {
            "train": "train_hate.csv",
            "valid": "dev_hate.csv"
        },
        "test_data_file": {
            "test": "test_hate_no_label.csv"
        }
    },
    "optimizer": {
        "type": "AdamW",
        "args":{
            "lr": 5e-5,
            "eps": 1e-8
        },
        "weight_decay": 0.0
    },
    "loss": "knowledge_distillation_loss",
    "metrics": [
        "macro_f1"
    ],
    "lr_scheduler": {
        "type": "CosineAnnealingLR",
        "args": {
            "T_max": 300,
            "eta_min": 1e-5
        }
    },
    "trainer": {
        "epochs": 2,

        "save": {
            "dir": "saved/",
            "steps": 300,
            "limits": 3
        },
        "verbosity": 2,
        
        "monitor": "max val/macro_f1",
        "early_stop": 5
    }
}