From 4d87afd6480737b64b5974c9c40a5b1c5a4600b3 Mon Sep 17 00:00:00 2001 From: liu zhengxi <380185688@qq.com> Date: Sat, 19 Dec 2020 12:21:45 +0800 Subject: [PATCH] Fix hung (#5121) * fix hung * add shuffle batch * update * reader_seed to shuffle_seed * seed for shuffle batch --- .../transformer/configs/transformer.big.yaml | 6 ++++++ PaddleNLP/benchmark/transformer/reader.py | 12 +++++++++++- .../transformer/configs/transformer.base.yaml | 6 ++++++ .../transformer/configs/transformer.big.yaml | 6 ++++++ .../machine_translation/transformer/reader.py | 12 +++++++++++- 5 files changed, 40 insertions(+), 2 deletions(-) diff --git a/PaddleNLP/benchmark/transformer/configs/transformer.big.yaml b/PaddleNLP/benchmark/transformer/configs/transformer.big.yaml index 05a6520110..fa321f1605 100644 --- a/PaddleNLP/benchmark/transformer/configs/transformer.big.yaml +++ b/PaddleNLP/benchmark/transformer/configs/transformer.big.yaml @@ -27,6 +27,12 @@ pool_size: 200000 sort_type: "global" batch_size: 4096 infer_batch_size: 16 +shuffle_batch: True +# Data shuffle only works when sort_type is pool or none +shuffle: True +# shuffle_seed must be set when shuffle is True and using multi-cards to train. +# Otherwise, the number of batches cannot be guaranteed. +shuffle_seed: 128 # Hyparams for training: # The number of epoches for training diff --git a/PaddleNLP/benchmark/transformer/reader.py b/PaddleNLP/benchmark/transformer/reader.py index 38fcda422f..9e3f86ee0e 100644 --- a/PaddleNLP/benchmark/transformer/reader.py +++ b/PaddleNLP/benchmark/transformer/reader.py @@ -43,6 +43,12 @@ def create_data_loader(args): mode=m, transform_func=transform_func) for m in ["train", "dev"] ] + if args.shuffle or args.shuffle_batch: + if args.shuffle_seed == "None" or args.shuffle_seed is None: + shuffle_seed = 0 + else: + shuffle_seed = args.shuffle_seed + def _max_token_fn(current_idx, current_batch_size, tokens_sofar, data_source): return max(tokens_sofar, @@ -69,7 +75,8 @@ def _key(size_so_far, minibatch_len): key=trg_key, buffer_size=buffer_size).sort( key=src_key, buffer_size=buffer_size) else: - sampler = sampler.shuffle() + if args.shuffle: + sampler = sampler.shuffle(seed=shuffle_seed) if args.sort_type == SortType.POOL: buffer_size = args.pool_size sampler = sampler.sort(key=src_key, buffer_size=buffer_size) @@ -83,6 +90,9 @@ def _key(size_so_far, minibatch_len): if m == "train": batch_sampler = batch_sampler.shard() + if args.shuffle_batch: + batch_sampler.shuffle(seed=shuffle_seed) + data_loader = DataLoader( dataset=dataset, batch_sampler=batch_sampler, diff --git a/PaddleNLP/examples/machine_translation/transformer/configs/transformer.base.yaml b/PaddleNLP/examples/machine_translation/transformer/configs/transformer.base.yaml index 7ea9ebbe71..57070dc296 100644 --- a/PaddleNLP/examples/machine_translation/transformer/configs/transformer.base.yaml +++ b/PaddleNLP/examples/machine_translation/transformer/configs/transformer.base.yaml @@ -27,6 +27,12 @@ pool_size: 200000 sort_type: "global" batch_size: 4096 infer_batch_size: 8 +shuffle_batch: True +# Data shuffle only works when sort_type is pool or none +shuffle: True +# shuffle_seed must be set when shuffle is True and using multi-cards to train. +# Otherwise, the number of batches cannot be guaranteed. +shuffle_seed: 128 # Hyparams for training: # The number of epoches for training diff --git a/PaddleNLP/examples/machine_translation/transformer/configs/transformer.big.yaml b/PaddleNLP/examples/machine_translation/transformer/configs/transformer.big.yaml index d458f4c7eb..4cd3b1201f 100644 --- a/PaddleNLP/examples/machine_translation/transformer/configs/transformer.big.yaml +++ b/PaddleNLP/examples/machine_translation/transformer/configs/transformer.big.yaml @@ -27,6 +27,12 @@ pool_size: 200000 sort_type: "global" batch_size: 4096 infer_batch_size: 8 +shuffle_batch: True +# Data shuffle only works when sort_type is pool or none +shuffle: True +# shuffle_seed must be set when shuffle is True and using multi-cards to train. +# Otherwise, the number of batches cannot be guaranteed. +shuffle_seed: 128 # Hyparams for training: # The number of epoches for training diff --git a/PaddleNLP/examples/machine_translation/transformer/reader.py b/PaddleNLP/examples/machine_translation/transformer/reader.py index 38fcda422f..9e3f86ee0e 100644 --- a/PaddleNLP/examples/machine_translation/transformer/reader.py +++ b/PaddleNLP/examples/machine_translation/transformer/reader.py @@ -43,6 +43,12 @@ def create_data_loader(args): mode=m, transform_func=transform_func) for m in ["train", "dev"] ] + if args.shuffle or args.shuffle_batch: + if args.shuffle_seed == "None" or args.shuffle_seed is None: + shuffle_seed = 0 + else: + shuffle_seed = args.shuffle_seed + def _max_token_fn(current_idx, current_batch_size, tokens_sofar, data_source): return max(tokens_sofar, @@ -69,7 +75,8 @@ def _key(size_so_far, minibatch_len): key=trg_key, buffer_size=buffer_size).sort( key=src_key, buffer_size=buffer_size) else: - sampler = sampler.shuffle() + if args.shuffle: + sampler = sampler.shuffle(seed=shuffle_seed) if args.sort_type == SortType.POOL: buffer_size = args.pool_size sampler = sampler.sort(key=src_key, buffer_size=buffer_size) @@ -83,6 +90,9 @@ def _key(size_so_far, minibatch_len): if m == "train": batch_sampler = batch_sampler.shard() + if args.shuffle_batch: + batch_sampler.shuffle(seed=shuffle_seed) + data_loader = DataLoader( dataset=dataset, batch_sampler=batch_sampler,