diff --git a/scripts/training/build_dataset.py b/scripts/training/build_dataset.py new file mode 100644 index 0000000..772531c --- /dev/null +++ b/scripts/training/build_dataset.py @@ -0,0 +1,84 @@ +import logging +import os +from typing import Union, List +import datasets +import torch +from datasets import load_dataset, concatenate_datasets +import transformers + + +IGNORE_INDEX = -100 + +logger = logging.getLogger('__name__') + +PROMPT_TEMPLATE = ( + "[INST] {instruction} [/INST]" + ) + +def build_instruction_dataset(data_path: Union[List[str],str], + tokenizer: transformers.PreTrainedTokenizer, + max_seq_length: int, data_cache_dir = None, + preprocessing_num_workers = None, + ): + + def tokenization(examples): + sources = [] + targets = [] + prompt = PROMPT_TEMPLATE + for instruction, input_text, output in zip(examples['instruction'],examples['input'],examples['output']): + if input_text is not None and input_text !="": + instruction = instruction+'\n' + input_text + source = prompt.format_map({'instruction':instruction}) + target = f"{output}{tokenizer.eos_token}" + + sources.append(source) + targets.append(target) + + tokenized_sources = tokenizer(sources,return_attention_mask=False) + tokenized_targets = tokenizer(targets,return_attention_mask=False,add_special_tokens=False) + + all_input_ids = [] + all_labels = [] + for s,t in zip(tokenized_sources['input_ids'],tokenized_targets['input_ids']): + if len(s) >= max_seq_length: + continue + input_ids = torch.LongTensor(s + t)[:max_seq_length] + labels = torch.LongTensor([IGNORE_INDEX] * len(s) + t)[:max_seq_length] + all_input_ids.append(input_ids) + all_labels.append(labels) + + results = {'input_ids':all_input_ids, 'labels': all_labels} + return results + + + logging.warning("building dataset...") + all_datasets = [] + + if not isinstance(data_path,(list,tuple)): + data_path = [data_path] + for file in data_path: + + if data_cache_dir is None: + data_cache_dir = str(os.path.dirname(file)) + cache_path = os.path.join(data_cache_dir,os.path.basename(file).split('.')[0]+f"_{max_seq_length}") + os.makedirs(cache_path, exist_ok=True) + try: + processed_dataset = datasets.load_from_disk(cache_path) + logger.info(f'training datasets-{file} has been loaded from disk') + except Exception: + raw_dataset = load_dataset("json", data_files=file, cache_dir=cache_path) + tokenization_func = tokenization + tokenized_dataset = raw_dataset.map( + tokenization_func, + batched=True, + num_proc=preprocessing_num_workers, + remove_columns=["instruction","input","output"], + keep_in_memory=False, + desc="preprocessing on dataset", + ) + processed_dataset = tokenized_dataset + processed_dataset.save_to_disk(cache_path) + processed_dataset.set_format('torch') + all_datasets.append(processed_dataset['train']) + all_datasets = concatenate_datasets(all_datasets) + return all_datasets diff --git a/scripts/training/ds_zero2_no_offload.json b/scripts/training/ds_zero2_no_offload.json new file mode 100644 index 0000000..117a911 --- /dev/null +++ b/scripts/training/ds_zero2_no_offload.json @@ -0,0 +1,27 @@ +{ + "fp16": { + "enabled": "auto", + "loss_scale": 0, + "loss_scale_window": 100, + "initial_scale_power": 16, + "hysteresis": 2, + "min_loss_scale": 1e-10 + }, + + "zero_optimization": { + "stage": 2, + "allgather_partitions": true, + "allgather_bucket_size": 1e8, + "overlap_comm": true, + "reduce_scatter": true, + "reduce_bucket_size": 1e8, + "contiguous_gradients": true + }, + + "gradient_accumulation_steps": "auto", + "gradient_clipping": "auto", + "steps_per_print": 2000, + "train_batch_size": "auto", + "train_micro_batch_size_per_gpu": "auto", + "wall_clock_breakdown": false +} diff --git a/scripts/training/run_clm_pt_with_peft.py b/scripts/training/run_clm_pt_with_peft.py new file mode 100644 index 0000000..b66d78d --- /dev/null +++ b/scripts/training/run_clm_pt_with_peft.py @@ -0,0 +1,600 @@ +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2020 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Fine-tuning the library models for causal language modeling (GPT, GPT-2, CTRL, ...) on a text file or a dataset. + +Here is the full list of checkpoints on the hub that can be fine-tuned by this script: +https://huggingface.co/models?filter=text-generation +""" +# You can also adapt this script on your own causal language modeling task. Pointers for this are left as comments. + +import logging +import numpy as np +import math +import os +import sys +from dataclasses import dataclass, field +from itertools import chain +from typing import Optional, List, Dict, Any, Mapping +from pathlib import Path +import datasets +import torch +from datasets import load_dataset, concatenate_datasets + +import transformers +from transformers import ( + CONFIG_MAPPING, + MODEL_FOR_CAUSAL_LM_MAPPING, + AutoConfig, + AutoModelForCausalLM, + MixtralForCausalLM, + LlamaTokenizer, + AutoTokenizer, + HfArgumentParser, + Trainer, + TrainingArguments, + set_seed, + BitsAndBytesConfig +) +from transformers.testing_utils import CaptureLogger +from transformers.trainer_utils import get_last_checkpoint +from transformers.utils import send_example_telemetry +from transformers.utils.versions import require_version +from peft import LoraConfig, TaskType, get_peft_model, PeftModel, prepare_model_for_kbit_training + + +def fault_tolerance_data_collator(features: List) -> Dict[str, Any]: + if not isinstance(features[0], Mapping): + features = [vars(f) for f in features] + first = features[0] + batch = {} + + # Special handling for labels. + # Ensure that tensor is created with the correct type + # (it should be automatically the case, but let's make sure of it.) + if "label" in first and first["label"] is not None: + label = first["label"].item() if isinstance(first["label"], torch.Tensor) else first["label"] + dtype = torch.long if isinstance(label, int) else torch.float + batch["labels"] = torch.tensor([f["label"] for f in features], dtype=dtype) + elif "label_ids" in first and first["label_ids"] is not None: + if isinstance(first["label_ids"], torch.Tensor): + batch["labels"] = torch.stack([f["label_ids"] for f in features]) + else: + dtype = torch.long if isinstance(first["label_ids"][0], int) else torch.float + batch["labels"] = torch.tensor([f["label_ids"] for f in features], dtype=dtype) + + # Handling of all other possible keys. + # Again, we will use the first element to figure out which key/values are not None for this model. + + try: + for k, v in first.items(): + if k not in ("label", "label_ids") and v is not None and not isinstance(v, str): + if isinstance(v, torch.Tensor): + batch[k] = torch.stack([f[k] for f in features]) + elif isinstance(v, np.ndarray): + batch[k] = torch.tensor(np.stack([f[k] for f in features])) + else: + batch[k] = torch.tensor([f[k] for f in features]) + except ValueError: # quick fix by simply take the first example + for k, v in first.items(): + if k not in ("label", "label_ids") and v is not None and not isinstance(v, str): + if isinstance(v, torch.Tensor): + batch[k] = torch.stack([features[0][k]] * len(features)) + elif isinstance(v, np.ndarray): + batch[k] = torch.tensor(np.stack([features[0][k]] * len(features))) + else: + batch[k] = torch.tensor([features[0][k]] * len(features)) + + return batch + + +MODEL_CONFIG_CLASSES = list(MODEL_FOR_CAUSAL_LM_MAPPING.keys()) +MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES) + + +@dataclass +class ModelArguments: + """ + Arguments pertaining to which model/config/tokenizer we are going to fine-tune, or train from scratch. + """ + + model_name_or_path: Optional[str] = field( + default=None, + metadata={ + "help": ( + "The model checkpoint for weights initialization.Don't set if you want to train a model from scratch." + ) + }, + ) + tokenizer_name_or_path: Optional[str] = field( + default=None, + metadata={ + "help": ( + "The tokenizer for weights initialization.Don't set if you want to train a model from scratch." + ) + }, + ) + model_type: Optional[str] = field( + default=None, + metadata={"help": "If training from scratch, pass a model type from the list: " + ", ".join(MODEL_TYPES)}, + ) + config_overrides: Optional[str] = field( + default=None, + metadata={ + "help": ( + "Override some existing default config settings when a model is trained from scratch. Example: " + "n_embd=10,resid_pdrop=0.2,scale_attn_weights=false,summary_type=cls_index" + ) + }, + ) + config_name: Optional[str] = field( + default=None, metadata={"help": "Pretrained config name or path if not the same as model_name"} + ) + tokenizer_name: Optional[str] = field( + default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"} + ) + cache_dir: Optional[str] = field( + default=None, + metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"}, + ) + use_fast_tokenizer: bool = field( + default=False, + metadata={"help": "Whether to use one of the fast tokenizer (backed by the tokenizers library) or not."}, + ) + model_revision: str = field( + default="main", + metadata={"help": "The specific model version to use (can be a branch name, tag name or commit id)."}, + ) + use_auth_token: bool = field( + default=False, + metadata={ + "help": ( + "Will use the token generated when running `huggingface-cli login` (necessary to use this script " + "with private models)." + ) + }, + ) + torch_dtype: Optional[str] = field( + default=None, + metadata={ + "help": ( + "Override the default `torch.dtype` and load the model under this dtype. If `auto` is passed, the " + "dtype will be automatically derived from the model's weights." + ), + "choices": ["auto", "bfloat16", "float16", "float32"], + }, + ) + + def __post_init__(self): + if self.config_overrides is not None and (self.config_name is not None or self.model_name_or_path is not None): + raise ValueError( + "--config_overrides can't be used in combination with --config_name or --model_name_or_path" + ) + + +@dataclass +class DataTrainingArguments: + """ + Arguments pertaining to what data we are going to input our model for training and eval. + """ + + dataset_dir: Optional[str] = field( + default=None, metadata={"help": "The name of the dataset to use (via the datasets library)."} + ) + dataset_config_name: Optional[str] = field( + default=None, metadata={"help": "The configuration name of the dataset to use (via the datasets library)."} + ) + train_file: Optional[str] = field(default=None, metadata={"help": "The input training data file (a text file)."}) + validation_file: Optional[str] = field( + default=None, + metadata={"help": "An optional input evaluation data file to evaluate the perplexity on (a text file)."}, + ) + max_train_samples: Optional[int] = field( + default=None, + metadata={ + "help": ( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ) + }, + ) + max_eval_samples: Optional[int] = field( + default=None, + metadata={ + "help": ( + "For debugging purposes or quicker training, truncate the number of evaluation examples to this " + "value if set." + ) + }, + ) + streaming: bool = field(default=False, metadata={"help": "Enable streaming mode"}) + block_size: Optional[int] = field( + default=None, + metadata={ + "help": ( + "Optional input sequence length after tokenization. " + "The training dataset will be truncated in block of this size for training. " + "Default to the model max input length for single sentence inputs (take into account special tokens)." + ) + }, + ) + overwrite_cache: bool = field( + default=False, metadata={"help": "Overwrite the cached training and evaluation sets"} + ) + validation_split_percentage: Optional[float] = field( + default=0.05, + metadata={ + "help": "The percentage of the train set used as validation set in case there's no validation split" + }, + ) + preprocessing_num_workers: Optional[int] = field( + default=None, + metadata={"help": "The number of processes to use for the preprocessing."}, + ) + keep_linebreaks: bool = field( + default=True, metadata={"help": "Whether to keep line breaks when using TXT files or not."} + ) + data_cache_dir: Optional[str] = field(default="./", metadata={"help": "The datasets processed stored"}) + + def __post_init__(self): + if self.streaming: + require_version("datasets>=2.0.0", "The streaming feature requires `datasets>=2.0.0`") + + +@dataclass +class MyTrainingArguments(TrainingArguments): + trainable : Optional[str] = field(default="q_proj,v_proj") + lora_rank : Optional[int] = field(default=8) + lora_dropout : Optional[float] = field(default=0.1) + lora_alpha : Optional[float] = field(default=32.) + modules_to_save : Optional[str] = field(default=None) + debug_mode : Optional[bool] = field(default=False) + peft_path : Optional[str] = field(default=None) + use_flash_attention_2 : Optional[bool] = field(default=False) + double_quant: Optional[bool] = field(default=True) + quant_type: Optional[str] = field(default="nf4") + load_in_kbits: Optional[int] = field(default=16) + output_router_logits: Optional[bool] = field(default=False) + + +logger = logging.getLogger(__name__) + + +def main(): + parser = HfArgumentParser((ModelArguments, DataTrainingArguments, MyTrainingArguments)) + if len(sys.argv) == 2 and sys.argv[1].endswith(".json"): + # If we pass only one argument to the script and it's the path to a json file, + # let's parse it to get our arguments. + model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1])) + else: + model_args, data_args, training_args = parser.parse_args_into_dataclasses() + + # Sending telemetry. Tracking the example usage helps us better allocate resources to maintain them. The + # information sent is the one passed as arguments along with your Python/PyTorch versions. + send_example_telemetry("run_clm", model_args, data_args) + + # Setup logging + logging.basicConfig(format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, # if training_args.local_rank in [-1, 0] else logging.WARN, + handlers=[logging.StreamHandler(sys.stdout)],) + + if training_args.should_log: + # The default of training_args.log_level is passive, so we set log level at info here to have that default. + transformers.utils.logging.set_verbosity_info() + + log_level = training_args.get_process_log_level() + logger.setLevel(log_level) + datasets.utils.logging.set_verbosity(log_level) + transformers.utils.logging.set_verbosity(log_level) + transformers.utils.logging.enable_default_handler() + transformers.utils.logging.enable_explicit_format() + # transformers.tokenization_utils.logging.set_verbosity_warning() + + # Log on each process the small summary: + logger.warning( + f"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}" + + f"distributed training: {bool(training_args.local_rank != -1)}, 16-bits training: {training_args.fp16}" + ) + + # Detecting last checkpoint. + last_checkpoint = None + if os.path.isdir(training_args.output_dir) and training_args.do_train and not training_args.overwrite_output_dir: + last_checkpoint = get_last_checkpoint(training_args.output_dir) + if last_checkpoint is None and len(os.listdir(training_args.output_dir)) > 0: + raise ValueError( + f"Output directory ({training_args.output_dir}) already exists and is not empty. " + "Use --overwrite_output_dir to overcome." + ) + elif last_checkpoint is not None and training_args.resume_from_checkpoint is None: + logger.info( + f"Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change " + "the `--output_dir` or add `--overwrite_output_dir` to train from scratch." + ) + + # Set seed before initializing model. + set_seed(training_args.seed) + + config_kwargs = { + "cache_dir": model_args.cache_dir, + "revision": model_args.model_revision, + "use_auth_token": True if model_args.use_auth_token else None, + "output_router_logits": True if training_args.output_router_logits else False + } + if model_args.config_name: + config = AutoConfig.from_pretrained(model_args.config_name, **config_kwargs) + elif model_args.model_name_or_path: + config = AutoConfig.from_pretrained(model_args.model_name_or_path, **config_kwargs) + else: + config = CONFIG_MAPPING[model_args.model_type]() + logger.warning("You are instantiating a new config instance from scratch.") + if model_args.config_overrides is not None: + logger.info(f"Overriding config: {model_args.config_overrides}") + config.update_from_string(model_args.config_overrides) + logger.info(f"New config: {config}") + + tokenizer_kwargs = { + "cache_dir": model_args.cache_dir, + "use_fast": model_args.use_fast_tokenizer, + "revision": model_args.model_revision, + "use_auth_token": True if model_args.use_auth_token else None, + } + if model_args.tokenizer_name: + tokenizer = AutoTokenizer.from_pretrained(model_args.tokenizer_name, **tokenizer_kwargs) + elif model_args.tokenizer_name_or_path: + tokenizer = LlamaTokenizer.from_pretrained(model_args.tokenizer_name_or_path, **tokenizer_kwargs) + else: + raise ValueError( + "You are instantiating a new tokenizer from scratch. This is not supported by this script." + "You can do it from another script, save it, and load it from here, using --tokenizer_name." + ) + tokenizer.add_eos_token = True + + # Preprocessing the datasets. + # First we tokenize all the texts. + # since this will be pickled to avoid _LazyModule error in Hasher force logger loading before tokenize_function + tok_logger = transformers.utils.logging.get_logger("transformers.tokenization_utils_base") + + def tokenize_function(examples): + with CaptureLogger(tok_logger) as cl: + output = tokenizer(examples["text"]) + # clm input could be much much longer than block_size + if "Token indices sequence length is longer than the" in cl.out: + tok_logger.warning( + "^^^^^^^^^^^^^^^^ Please ignore the warning above - this long input will be chunked into smaller bits" + " before being passed to the model." + ) + return output + if data_args.block_size is None: + block_size = tokenizer.model_max_length + if block_size > 1024: + logger.warning( + "The chosen tokenizer supports a `model_max_length` that is longer than the default `block_size` value" + " of 1024. If you would like to use a longer `block_size` up to `tokenizer.model_max_length` you can" + " override this default with `--block_size xxx`." + ) + block_size = 1024 + else: + if data_args.block_size > tokenizer.model_max_length: + logger.warning( + f"The block_size passed ({data_args.block_size}) is larger than the maximum length for the model" + f"({tokenizer.model_max_length}). Using block_size={tokenizer.model_max_length}." + ) + block_size = min(data_args.block_size, tokenizer.model_max_length) + + # Main data processing function that will concatenate all texts from our dataset and generate chunks of block_size. + def group_texts(examples): + # Concatenate all texts. + concatenated_examples = {k: list(chain(*examples[k])) for k in examples.keys()} + total_length = len(concatenated_examples[list(examples.keys())[0]]) + # We drop the small remainder, we could add padding if the model supported it instead of this drop, you can + # customize this part to your needs. + if total_length >= block_size: + total_length = (total_length // block_size) * block_size + # Split by chunks of max_len. + result = { + k: [t[i : i + block_size] for i in range(0, total_length, block_size)] + for k, t in concatenated_examples.items() + } + result["labels"] = result["input_ids"].copy() + return result + with training_args.main_process_first(desc="dataset map tokenization and grouping"): + lm_datasets = [] + path = Path(data_args.dataset_dir) + files = [file.name for file in path.glob("*.txt")] + if training_args.debug_mode is True: + files = [files[0]] + for idx, file in enumerate(files): + data_file = os.path.join(path, file) + filename = ''.join(file.split(".")[:-1]) + cache_path = os.path.join(data_args.data_cache_dir, filename+f"_{block_size}") + os.makedirs(cache_path, exist_ok=True) + try: + processed_dataset = datasets.load_from_disk(cache_path, keep_in_memory=False) + logger.info(f'training datasets-{filename} has been loaded from disk') + except Exception: + cache_dir = os.path.join(data_args.data_cache_dir, filename+f"_text_{block_size}") + os.makedirs(cache_dir, exist_ok=True) + raw_dataset = load_dataset("text", data_files=data_file, cache_dir=cache_dir, keep_in_memory=False) + logger.info(f"{file} has been loaded") + tokenized_dataset = raw_dataset.map( + tokenize_function, + batched=True, + num_proc=data_args.preprocessing_num_workers, + remove_columns="text", + load_from_cache_file=True, + keep_in_memory=False, + cache_file_names = {k: os.path.join(cache_dir, 'tokenized.arrow') for k in raw_dataset}, + desc="Running tokenizer on dataset", + ) + grouped_datasets = tokenized_dataset.map( + group_texts, + batched=True, + num_proc=data_args.preprocessing_num_workers, + load_from_cache_file=True, + keep_in_memory=False, + cache_file_names = {k: os.path.join(cache_dir, 'grouped.arrow') for k in tokenized_dataset}, + desc=f"Grouping texts in chunks of {block_size}", + ) + processed_dataset = grouped_datasets + processed_dataset.save_to_disk(cache_path) + if idx == 0: + lm_datasets = processed_dataset['train'] + else: + lm_datasets = concatenate_datasets([lm_datasets, processed_dataset["train"]]) + lm_datasets = lm_datasets.train_test_split(test_size = data_args.validation_split_percentage) + + if training_args.do_train: + train_dataset = lm_datasets['train'] + if data_args.max_train_samples is not None: + max_train_samples = min(len(train_dataset), data_args.max_train_samples) + train_dataset = train_dataset.select(range(max_train_samples)) + logger.info(f"Num train_samples {len(train_dataset)}") + logger.info("Training example:") + logger.info(tokenizer.decode(train_dataset[0]['input_ids'])) + if training_args.do_eval: + eval_dataset = lm_datasets["test"] + if data_args.max_eval_samples is not None: + max_eval_samples = min(len(eval_dataset), data_args.max_eval_samples) + eval_dataset = eval_dataset.select(range(max_eval_samples)) + logger.info(f"Num eval_samples {len(eval_dataset)}") + logger.info("Evaluation example:") + logger.info(tokenizer.decode(eval_dataset[0]['input_ids'])) + compute_dtype = (torch.float16 if training_args.fp16 else (torch.bfloat16 if training_args.bf16 else torch.float32)) + if training_args.load_in_kbits in [4, 8]: + if training_args.modules_to_save is not None: + load_in_8bit_skip_modules = training_args.modules_to_save.split(',') + else: + load_in_8bit_skip_modules = None + quantization_config = BitsAndBytesConfig( + load_in_4bit=training_args.load_in_kbits == 4, + load_in_8bit=training_args.load_in_kbits == 8, + llm_int8_threshold=6.0, + load_in_8bit_skip_modules=load_in_8bit_skip_modules, + bnb_4bit_compute_dtype=compute_dtype, + bnb_4bit_use_double_quant=training_args.double_quant, + bnb_4bit_quant_type=training_args.quant_type # {'fp4', 'nf4'} + ) + else: + quantization_config = None + if quantization_config is not None: + logger.info(f"quantization_config:{quantization_config.to_dict()}") + if model_args.model_name_or_path: + torch_dtype = ( + model_args.torch_dtype + if model_args.torch_dtype in ["auto", None] + else getattr(torch, model_args.torch_dtype) + ) + device_map = {"":int(os.environ.get("LOCAL_RANK") or 0)} + model = MixtralForCausalLM.from_pretrained( + model_args.model_name_or_path, + from_tf=bool(".ckpt" in model_args.model_name_or_path), + config=config, + cache_dir=model_args.cache_dir, + revision=model_args.model_revision, + use_auth_token=True if model_args.use_auth_token else None, + torch_dtype=torch_dtype, + low_cpu_mem_usage=True, + device_map=device_map, + quantization_config=quantization_config, + attn_implementation="flash_attention_2" if training_args.use_flash_attention_2 else "sdpa" + ) + else: + model = AutoModelForCausalLM.from_config(config) + n_params = sum({p.data_ptr(): p.numel() for p in model.parameters()}.values()) + logger.info(f"Training new model from scratch - Total size={n_params/2**20:.2f}M params") + if training_args.load_in_kbits in [4, 8]: + model = prepare_model_for_kbit_training(model, use_gradient_checkpointing=training_args.gradient_checkpointing) + model.config.use_cache = False + model_vocab_size = model.get_output_embeddings().weight.size(0) + tokenizer_vocab_size = len(tokenizer) + logger.info(f"Model vocab size: {model_vocab_size}") + logger.info(f"Tokenizer vocab size: {tokenizer_vocab_size}") + if model_vocab_size != tokenizer_vocab_size: + logger.info(f"Resize model vocab size to {tokenizer_vocab_size}") + model.resize_token_embeddings(tokenizer_vocab_size) + if training_args.peft_path is not None: + logger.info("Peft from pre-trained model") + model = PeftModel.from_pretrained(model, training_args.peft_path, device_map=device_map, is_trainable=True) + else: + logger.info("Init new peft model") + target_modules = training_args.trainable.split(',') + modules_to_save = training_args.modules_to_save + if modules_to_save is not None: + modules_to_save = modules_to_save.split(',') + lora_rank = training_args.lora_rank + lora_dropout = training_args.lora_dropout + lora_alpha = training_args.lora_alpha + logger.info(f"target_modules: {target_modules}") + logger.info(f"lora_rank: {lora_rank}") + peft_config = LoraConfig( + task_type=TaskType.CAUSAL_LM, + target_modules=target_modules, + inference_mode=False, + r=lora_rank, lora_alpha=lora_alpha, + lora_dropout=lora_dropout, + modules_to_save=modules_to_save) + model = get_peft_model(model, peft_config) + model.print_trainable_parameters() + + # Initialize our Trainer + trainer = Trainer( + model=model, + args=training_args, + train_dataset=train_dataset if training_args.do_train else None, + eval_dataset=eval_dataset if training_args.do_eval else None, + tokenizer=tokenizer, + data_collator=fault_tolerance_data_collator + ) + # Training + if training_args.do_train: + checkpoint = None + if training_args.resume_from_checkpoint is not None: + checkpoint = training_args.resume_from_checkpoint + elif last_checkpoint is not None: + checkpoint = last_checkpoint + train_result = trainer.train(resume_from_checkpoint=checkpoint) + trainer.save_model() + metrics = train_result.metrics + + max_train_samples = ( + data_args.max_train_samples if data_args.max_train_samples is not None else len(train_dataset) + ) + metrics["train_samples"] = min(max_train_samples, len(train_dataset)) + + trainer.log_metrics("train", metrics) + trainer.save_metrics("train", metrics) + trainer.save_state() + + # Evaluation + if training_args.do_eval: + logger.info("*** Evaluate ***") + + metrics = trainer.evaluate() + + max_eval_samples = data_args.max_eval_samples if data_args.max_eval_samples is not None else len(eval_dataset) + metrics["eval_samples"] = min(max_eval_samples, len(eval_dataset)) + try: + perplexity = math.exp(metrics["eval_loss"]) + except OverflowError: + perplexity = float("inf") + metrics["perplexity"] = perplexity + + trainer.log_metrics("eval", metrics) + trainer.save_metrics("eval", metrics) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/training/run_clm_sft_with_peft.py b/scripts/training/run_clm_sft_with_peft.py new file mode 100644 index 0000000..1503ac3 --- /dev/null +++ b/scripts/training/run_clm_sft_with_peft.py @@ -0,0 +1,424 @@ +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2020 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Fine-tuning the library models for causal language modeling (GPT, GPT-2, CTRL, ...) on a text file or a dataset. + +Here is the full list of checkpoints on the hub that can be fine-tuned by this script: +https://huggingface.co/models?filter=text-generation +""" +# You can also adapt this script on your own causal language modeling task. Pointers for this are left as comments. + +import logging +import math +import os +import sys +from dataclasses import dataclass, field +from typing import Optional +from pathlib import Path +import datasets +import torch +from build_dataset import build_instruction_dataset +import transformers +from transformers import ( + CONFIG_MAPPING, + AutoConfig, + BitsAndBytesConfig, + MixtralForCausalLM, + LlamaTokenizer, + AutoTokenizer, + HfArgumentParser, + Trainer, + TrainingArguments, + set_seed, + DataCollatorForSeq2Seq +) +from transformers.trainer_utils import get_last_checkpoint +from transformers.utils import send_example_telemetry +from transformers.utils.versions import require_version +from peft import LoraConfig, TaskType, get_peft_model, PeftModel, prepare_model_for_kbit_training + +require_version("datasets>=1.8.0", "To fix: pip install -r examples/pytorch/language-modeling/requirements.txt") + + +@dataclass +class ModelArguments: + """ + Arguments pertaining to which model/config/tokenizer we are going to fine-tune, or train from scratch. + """ + + model_name_or_path: Optional[str] = field( + default=None, + metadata={ + "help": ( + "The model checkpoint for weights initialization.Don't set if you want to train a model from scratch." + ) + }, + ) + tokenizer_name_or_path: Optional[str] = field( + default=None, + metadata={ + "help": ( + "The tokenizer for weights initialization.Don't set if you want to train a model from scratch." + ) + }, + ) + + config_overrides: Optional[str] = field( + default=None, + metadata={ + "help": ( + "Override some existing default config settings when a model is trained from scratch. Example: " + "n_embd=10,resid_pdrop=0.2,scale_attn_weights=false,summary_type=cls_index" + ) + }, + ) + config_name: Optional[str] = field( + default=None, metadata={"help": "Pretrained config name or path if not the same as model_name"} + ) + tokenizer_name: Optional[str] = field( + default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"} + ) + cache_dir: Optional[str] = field( + default=None, + metadata={"help": "Where do you want to store the pretrained models downloaded from huggingface.co"}, + ) + use_fast_tokenizer: bool = field( + default=False, + metadata={"help": "Whether to use one of the fast tokenizer (backed by the tokenizers library) or not."}, + ) + model_revision: str = field( + default="main", + metadata={"help": "The specific model version to use (can be a branch name, tag name or commit id)."}, + ) + use_auth_token: bool = field( + default=False, + metadata={ + "help": ( + "Will use the token generated when running `huggingface-cli login` (necessary to use this script " + "with private models)." + ) + }, + ) + torch_dtype: Optional[str] = field( + default=None, + metadata={ + "help": ( + "Override the default `torch.dtype` and load the model under this dtype. If `auto` is passed, the " + "dtype will be automatically derived from the model's weights." + ), + "choices": ["auto", "bfloat16", "float16", "float32"], + }, + ) + + def __post_init__(self): + if self.config_overrides is not None and (self.config_name is not None or self.model_name_or_path is not None): + raise ValueError( + "--config_overrides can't be used in combination with --config_name or --model_name_or_path" + ) + + +@dataclass +class DataTrainingArguments: + """ + Arguments pertaining to what data we are going to input our model for training and eval. + """ + + dataset_dir: Optional[str] = field( + default=None, metadata={"help": "The name of the dataset to use (via the datasets library)."} + ) + + train_file: Optional[str] = field(default=None, metadata={"help": "The input training data file (a text file)."}) + validation_file: Optional[str] = field( + default=None, + metadata={"help": "An optional input evaluation data file to evaluate the perplexity on (a text file)."}, + ) + + overwrite_cache: bool = field( + default=False, metadata={"help": "Overwrite the cached training and evaluation sets"} + ) + validation_split_percentage: Optional[float] = field( + default=0.05, + metadata={ + "help": "The percentage of the train set used as validation set in case there's no validation split" + }, + ) + preprocessing_num_workers: Optional[int] = field( + default=None, + metadata={"help": "The number of processes to use for the preprocessing."}, + ) + keep_linebreaks: bool = field( + default=True, metadata={"help": "Whether to keep line breaks when using TXT files or not."} + ) + data_cache_dir: Optional[str] = field(default=None, metadata={"help": "The datasets processed stored"}) + + max_seq_length: Optional[int] = field(default=1024) + + +@dataclass +class MyTrainingArguments(TrainingArguments): + trainable : Optional[str] = field(default="q_proj,v_proj") + lora_rank : Optional[int] = field(default=8) + lora_dropout : Optional[float] = field(default=0.1) + lora_alpha : Optional[float] = field(default=32.) + modules_to_save : Optional[str] = field(default=None) + peft_path : Optional[str] = field(default=None) + use_flash_attention_2 : Optional[bool] = field(default=False) + double_quant: Optional[bool] = field(default=True) + quant_type: Optional[str] = field(default="nf4") + load_in_kbits: Optional[int] = field(default=16) + output_router_logits: Optional[bool] = field(default=False) + + +logger = logging.getLogger(__name__) + + +def main(): + + parser = HfArgumentParser((ModelArguments, DataTrainingArguments, MyTrainingArguments)) + if len(sys.argv) == 2 and sys.argv[1].endswith(".json"): + # If we pass only one argument to the script and it's the path to a json file, + # let's parse it to get our arguments. + model_args, data_args, training_args = parser.parse_json_file(json_file=os.path.abspath(sys.argv[1])) + else: + model_args, data_args, training_args = parser.parse_args_into_dataclasses() + + send_example_telemetry("run_clm", model_args, data_args) + + # Setup logging + logging.basicConfig(format="%(asctime)s - %(levelname)s - %(name)s - %(message)s",datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, # if training_args.local_rank in [-1, 0] else logging.WARN, + handlers=[logging.StreamHandler(sys.stdout)],) + + + if training_args.should_log: + # The default of training_args.log_level is passive, so we set log level at info here to have that default. + transformers.utils.logging.set_verbosity_info() + + log_level = training_args.get_process_log_level() + logger.setLevel(log_level) + datasets.utils.logging.set_verbosity(log_level) + transformers.utils.logging.set_verbosity(log_level) + transformers.utils.logging.enable_default_handler() + transformers.utils.logging.enable_explicit_format() + # transformers.tokenization_utils.logging.set_verbosity_warning() + + # Log on each process the small summary: + logger.warning( + f"Process rank: {training_args.local_rank}, device: {training_args.device}, n_gpu: {training_args.n_gpu}" + + f"distributed training: {bool(training_args.local_rank != -1)}, 16-bits training: {training_args.fp16 or training_args.bf16}" + ) + + # Detecting last checkpoint. + last_checkpoint = None + if os.path.isdir(training_args.output_dir) and training_args.do_train and not training_args.overwrite_output_dir: + last_checkpoint = get_last_checkpoint(training_args.output_dir) + if last_checkpoint is None and len(os.listdir(training_args.output_dir)) > 0: + raise ValueError( + f"Output directory ({training_args.output_dir}) already exists and is not empty. " + "Use --overwrite_output_dir to overcome." + ) + elif last_checkpoint is not None and training_args.resume_from_checkpoint is None: + logger.info( + f"Checkpoint detected, resuming training at {last_checkpoint}. To avoid this behavior, change " + "the `--output_dir` or add `--overwrite_output_dir` to train from scratch." + ) + + # Set seed before initializing model. + set_seed(training_args.seed) + + config_kwargs = { + "cache_dir": model_args.cache_dir, + "revision": model_args.model_revision, + "use_auth_token": True if model_args.use_auth_token else None, + "output_router_logits": True if training_args.output_router_logits else False + } + if model_args.config_name: + config = AutoConfig.from_pretrained(model_args.config_name, **config_kwargs) + elif model_args.model_name_or_path: + config = AutoConfig.from_pretrained(model_args.model_name_or_path, **config_kwargs) + else: + config = CONFIG_MAPPING[model_args.model_type]() + logger.warning("You are instantiating a new config instance from scratch.") + if model_args.config_overrides is not None: + logger.info(f"Overriding config: {model_args.config_overrides}") + config.update_from_string(model_args.config_overrides) + logger.info(f"New config: {config}") + + tokenizer_kwargs = { + "cache_dir": model_args.cache_dir, + "use_fast": model_args.use_fast_tokenizer, + "revision": model_args.model_revision, + "use_auth_token": True if model_args.use_auth_token else None, + } + if model_args.tokenizer_name: + tokenizer = AutoTokenizer.from_pretrained(model_args.tokenizer_name, **tokenizer_kwargs) + elif model_args.tokenizer_name_or_path: + tokenizer = LlamaTokenizer.from_pretrained(model_args.tokenizer_name_or_path, **tokenizer_kwargs) + else: + raise ValueError( + "You are instantiating a new tokenizer from scratch. This is not supported by this script." + "You can do it from another script, save it, and load it from here, using --tokenizer_name." + ) + if tokenizer.pad_token_id is None: + tokenizer.pad_token = tokenizer.eos_token + data_collator = DataCollatorForSeq2Seq(tokenizer=tokenizer) + eval_dataset=None + train_dataset = None + + if training_args.do_train: + with training_args.main_process_first(desc="loading and tokenization"): + path = Path(data_args.dataset_dir) + files = [os.path.join(path,file.name) for file in path.glob("*.json")] + logger.info(f"Training files: {' '.join(files)}") + train_dataset = build_instruction_dataset( + data_path=files, + tokenizer=tokenizer, + max_seq_length=data_args.max_seq_length, + data_cache_dir = None, + preprocessing_num_workers = data_args.preprocessing_num_workers) + logger.info(f"Num train_samples {len(train_dataset)}") + logger.info("Training example:") + logger.info(tokenizer.decode(train_dataset[0]['input_ids'])) + if training_args.do_eval: + with training_args.main_process_first(desc="loading and tokenization"): + files = [data_args.validation_file] + logger.info(f"Evaluation files: {' '.join(files)}") + eval_dataset = build_instruction_dataset( + data_path=files, + tokenizer=tokenizer, + max_seq_length=data_args.max_seq_length, + data_cache_dir=None, + preprocessing_num_workers = data_args.preprocessing_num_workers) + logger.info(f"Num eval_samples {len(eval_dataset)}") + logger.info("Evaluation example:") + logger.info(tokenizer.decode(eval_dataset[0]['input_ids'])) + + torch_dtype = ( + model_args.torch_dtype + if model_args.torch_dtype in ["auto", None] + else getattr(torch, model_args.torch_dtype) + ) + compute_dtype = (torch.float16 if training_args.fp16 else (torch.bfloat16 if training_args.bf16 else torch.float32)) + if training_args.load_in_kbits in [4, 8]: + if training_args.modules_to_save is not None: + load_in_8bit_skip_modules = training_args.modules_to_save.split(',') + else: + load_in_8bit_skip_modules = None + quantization_config = BitsAndBytesConfig( + load_in_4bit=training_args.load_in_kbits == 4, + load_in_8bit=training_args.load_in_kbits == 8, + llm_int8_threshold=6.0, + load_in_8bit_skip_modules=load_in_8bit_skip_modules, + bnb_4bit_compute_dtype=compute_dtype, + bnb_4bit_use_double_quant=training_args.double_quant, + bnb_4bit_quant_type=training_args.quant_type # {'fp4', 'nf4'} + ) + else: + quantization_config = None + if quantization_config is not None: + logger.info(f"quantization_config:{quantization_config.to_dict()}") + device_map = {"":int(os.environ.get("LOCAL_RANK") or 0)} + model = MixtralForCausalLM.from_pretrained( + model_args.model_name_or_path, + config=config, + cache_dir=model_args.cache_dir, + revision=model_args.model_revision, + use_auth_token=True if model_args.use_auth_token else None, + torch_dtype=torch_dtype, + low_cpu_mem_usage=True, + device_map=device_map, + quantization_config=quantization_config, + attn_implementation="flash_attention_2" if training_args.use_flash_attention_2 else "sdpa" + ) + if training_args.load_in_kbits in [4, 8]: + model = prepare_model_for_kbit_training(model, use_gradient_checkpointing=training_args.gradient_checkpointing) + model.config.use_cache = False + model_vocab_size = model.get_input_embeddings().weight.shape[0] + logger.info(f"Model vocab size: {model_vocab_size}") + logger.info(f"len(tokenizer):{len(tokenizer)}") + if model_vocab_size != len(tokenizer): + logger.info(f"Resize model vocab size to {len(tokenizer)}") + model.resize_token_embeddings(len(tokenizer)) + if training_args.peft_path is not None: + logger.info("Peft from pre-trained model") + model = PeftModel.from_pretrained(model, training_args.peft_path, device_map=device_map, is_trainable=True) + else: + logger.info("Init new peft model") + target_modules = training_args.trainable.split(',') + modules_to_save = training_args.modules_to_save + if modules_to_save is not None: + modules_to_save = modules_to_save.split(',') + lora_rank = training_args.lora_rank + lora_dropout = training_args.lora_dropout + lora_alpha = training_args.lora_alpha + logger.info(f"target_modules: {target_modules}") + logger.info(f"lora_rank: {lora_rank}") + peft_config = LoraConfig( + task_type=TaskType.CAUSAL_LM, + target_modules=target_modules, + inference_mode=False, + r=lora_rank, lora_alpha=lora_alpha, + lora_dropout=lora_dropout, + modules_to_save=modules_to_save) + model = get_peft_model(model, peft_config) + model.print_trainable_parameters() + + # Initialize our Trainer + trainer = Trainer( + model=model, + args=training_args, + train_dataset=train_dataset, + eval_dataset=eval_dataset, + tokenizer=tokenizer, + data_collator=data_collator, + ) + + # Training + if training_args.do_train: + checkpoint = None + if training_args.resume_from_checkpoint is not None: + checkpoint = training_args.resume_from_checkpoint + elif last_checkpoint is not None: + checkpoint = last_checkpoint + train_result = trainer.train(resume_from_checkpoint=checkpoint) + trainer.save_model() + + metrics = train_result.metrics + + metrics["train_samples"] = len(train_dataset) + + trainer.log_metrics("train", metrics) + trainer.save_metrics("train", metrics) + trainer.save_state() + + # Evaluation + if training_args.do_eval: + logger.info("*** Evaluate ***") + + metrics = trainer.evaluate() + metrics["eval_samples"] =len(eval_dataset) + try: + perplexity = math.exp(metrics["eval_loss"]) + except OverflowError: + perplexity = float("inf") + metrics["perplexity"] = perplexity + + trainer.log_metrics("eval", metrics) + trainer.save_metrics("eval", metrics) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/scripts/training/run_pt.sh b/scripts/training/run_pt.sh new file mode 100644 index 0000000..186205c --- /dev/null +++ b/scripts/training/run_pt.sh @@ -0,0 +1,58 @@ + +## 运行脚本前请仔细阅读wiki(https://github.com/ymcui/Chinese-Mixtral/wiki/pt_scripts_zh) +## Read the wiki(https://github.com/ymcui/Chinese-Mixtral/wiki/pt_scripts_en) carefully before running the script +lr=1e-4 +lora_rank=64 +lora_alpha=128 +lora_trainable="q_proj,v_proj,k_proj,o_proj,gate,w1,w2,w3" +modules_to_save="embed_tokens,lm_head" +lora_dropout=0.05 + +pretrained_model=path/to/hf/mixtral/dir +dataset_dir=path/to/pt/data/dir +data_cache=temp_data_cache_dir +per_device_train_batch_size=1 +gradient_accumulation_steps=8 +block_size=1024 +output_dir=output_dir + +deepspeed_config_file=ds_zero2_no_offload.json + +torchrun --nnodes 1 --nproc_per_node 1 run_clm_pt_with_peft.py \ + --deepspeed ${deepspeed_config_file} \ + --model_name_or_path ${pretrained_model} \ + --tokenizer_name_or_path ${pretrained_model} \ + --dataset_dir ${dataset_dir} \ + --data_cache_dir ${data_cache} \ + --validation_split_percentage 0.001 \ + --per_device_train_batch_size ${per_device_train_batch_size} \ + --do_train \ + --seed $RANDOM \ + --fp16 \ + --num_train_epochs 1 \ + --lr_scheduler_type cosine \ + --learning_rate ${lr} \ + --warmup_ratio 0.05 \ + --weight_decay 0.1 \ + --logging_strategy steps \ + --logging_steps 10 \ + --save_strategy steps \ + --save_total_limit 3 \ + --save_steps 200 \ + --gradient_accumulation_steps ${gradient_accumulation_steps} \ + --preprocessing_num_workers 8 \ + --block_size ${block_size} \ + --output_dir ${output_dir} \ + --overwrite_output_dir \ + --ddp_timeout 30000 \ + --logging_first_step True \ + --lora_rank ${lora_rank} \ + --lora_alpha ${lora_alpha} \ + --trainable ${lora_trainable} \ + --lora_dropout ${lora_dropout} \ + --modules_to_save ${modules_to_save} \ + --torch_dtype float16 \ + --load_in_kbits 4 \ + --gradient_checkpointing \ + --ddp_find_unused_parameters False \ + --output_router_logits diff --git a/scripts/training/run_sft.sh b/scripts/training/run_sft.sh new file mode 100644 index 0000000..eae77e0 --- /dev/null +++ b/scripts/training/run_sft.sh @@ -0,0 +1,62 @@ + +## 运行脚本前请仔细阅读wiki(https://github.com/ymcui/Chinese-Mixtral/wiki/sft_scripts_zh) +## Read the wiki(https://github.com/ymcui/Chinese-Mixtral/wiki/sft_scripts_en) carefully before running the script +lr=1e-4 +lora_rank=64 +lora_alpha=128 +lora_trainable="q_proj,v_proj,k_proj,o_proj,gate,w1,w2,w3" +modules_to_save="embed_tokens,lm_head" +lora_dropout=0.05 + +pretrained_model=path/to/hf/chinese-mixtral/dir/or/model_id +dataset_dir=path/to/sft/data/dir +per_device_train_batch_size=1 +per_device_eval_batch_size=1 +gradient_accumulation_steps=8 +max_seq_length=1024 +output_dir=output_dir +validation_file=validation_file_name + +deepspeed_config_file=ds_zero2_no_offload.json + +torchrun --nnodes 1 --nproc_per_node 1 run_clm_sft_with_peft.py \ + --deepspeed ${deepspeed_config_file} \ + --model_name_or_path ${pretrained_model} \ + --tokenizer_name_or_path ${pretrained_model} \ + --dataset_dir ${dataset_dir} \ + --per_device_train_batch_size ${per_device_train_batch_size} \ + --per_device_eval_batch_size ${per_device_eval_batch_size} \ + --do_train \ + --do_eval \ + --seed $RANDOM \ + --fp16 \ + --num_train_epochs 3 \ + --lr_scheduler_type cosine \ + --learning_rate ${lr} \ + --warmup_ratio 0.05 \ + --weight_decay 0.1 \ + --logging_strategy steps \ + --logging_steps 10 \ + --save_strategy steps \ + --save_total_limit 3 \ + --evaluation_strategy steps \ + --eval_steps 100 \ + --save_steps 200 \ + --gradient_accumulation_steps ${gradient_accumulation_steps} \ + --preprocessing_num_workers 8 \ + --max_seq_length ${max_seq_length} \ + --output_dir ${output_dir} \ + --overwrite_output_dir \ + --ddp_timeout 30000 \ + --logging_first_step True \ + --lora_rank ${lora_rank} \ + --lora_alpha ${lora_alpha} \ + --trainable ${lora_trainable} \ + --lora_dropout ${lora_dropout} \ + --modules_to_save ${modules_to_save} \ + --torch_dtype float16 \ + --validation_file ${validation_file} \ + --load_in_kbits 4 \ + --gradient_checkpointing \ + --ddp_find_unused_parameters False \ + --output_router_logits