Merge branch 'main' into moco

NVIDIA · Jan 2, 2025 · 77684d5 · 77684d5
2 parents ac26890 + d901ebd
commit 77684d5
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/3rdparty/Megatron-LM b/3rdparty/Megatron-LM
diff --git a/3rdparty/NeMo b/3rdparty/NeMo
+1 −1		.gitlab/stages/00.pre.yml
+1 −0		examples/gpt3/gpt_config.yaml
+55 −50		examples/inference/README.md
+5 −5		examples/inference/gpt/gpt_batch_inference.py
+3 −3		examples/inference/t5/simple_t5_batch_inference.py
+2 −2		examples/multimodal/README.md
+137 −66		examples/multimodal/dataset_helpers.py
+0 −0		examples/multimodal/evaluation/evaluate_ai2d.py
+0 −0		examples/multimodal/evaluation/evaluate_chartqa.py
+0 −0		examples/multimodal/evaluation/evaluate_coco.py
+0 −0		examples/multimodal/evaluation/evaluate_mathvista.py
+6 −0		examples/multimodal/evaluation/evaluate_mmmu.py
+0 −0		examples/multimodal/evaluation/evaluate_ocrbench.py
+0 −0		examples/multimodal/evaluation/evaluate_textvqa.py
+0 −0		examples/multimodal/evaluation/evaluate_vqav2.py
+0 −0		examples/multimodal/evaluation/evaluation_datasets.py
+9 −2		examples/multimodal/nvlm/README.md
+1 −1		examples/multimodal/nvlm/pretrain_qwen20_72b_internvit_6b.sh
+1 −1		examples/multimodal/nvlm/pretrain_yi_34b_internvit_6b.sh
+1 −1		examples/multimodal/nvlm/run_text_generation_qwen20_72b_internvit_6b.sh
+2 −2		examples/multimodal/nvlm/run_text_generation_yi_34b_internvit_6b.sh
+1 −1		examples/multimodal/nvlm/sft_34b_internvit.sh
+1 −1		examples/multimodal/nvlm/sft_qwen20_72b_internvit_6b.sh
+1 −6		examples/multimodal/pretrain_mistral_clip.sh
+1 −1		examples/multimodal/run_text_generation.py
+1 −6		examples/multimodal/sft_mistral_clip.sh
+4 −13		examples/multimodal/text_generation_mistral_clip.sh
+2 −2		examples/multimodal/train.py
+2 −1		megatron/core/dist_checkpointing/mapping.py
+0 −2		megatron/core/dist_checkpointing/serialization.py
+13 −14		megatron/core/dist_checkpointing/validation.py
+21 −6		megatron/core/distributed/distributed_data_parallel.py
+29 −18		megatron/core/extensions/transformer_engine.py
+4 −29		megatron/core/inference/common_inference_params.py
+15 −8		megatron/core/inference/engines/mcore_engine.py
+2 −2		megatron/core/inference/inference_request.py
+35 −0		megatron/core/inference/sampling_params.py
+3 −3		megatron/core/inference/scheduler.py
+4 −4		megatron/core/inference/text_generation_controllers/encoder_decoder_text_generation_controller.py
+3 −398		megatron/core/inference/text_generation_controllers/simple_text_generation_controller.py
+400 −0		megatron/core/inference/text_generation_controllers/text_generation_controller.py
+48 −24		megatron/core/models/bert/bert_layer_specs.py
+16 −13		megatron/core/models/common/embeddings/rope_utils.py
+5 −2		megatron/core/models/multimodal/llava_model.py
+49 −24		megatron/core/optimizer/__init__.py
+21 −9		megatron/core/optimizer/clip_grads.py
+193 −102		megatron/core/optimizer/distrib_optimizer.py
+143 −86		megatron/core/optimizer/optimizer.py
+65 −0		megatron/core/optimizer/optimizer_config.py
+10 −0		megatron/core/pipeline_parallel/schedules.py
+80 −76		megatron/core/rerun_state_machine.py
+670 −212		megatron/core/transformer/cuda_graphs.py
+4 −1		megatron/core/transformer/moe/README.md
+115 −4		megatron/core/transformer/moe/moe_utils.py
+55 −26		megatron/core/transformer/moe/router.py
+4 −1		megatron/core/transformer/transformer_block.py
+32 −6		megatron/core/transformer/transformer_config.py
+4 −5		megatron/inference/text_generation/forward_step.py
+39 −8		megatron/training/arguments.py
+27 −12		megatron/training/checkpointing.py
+37 −13		megatron/training/training.py
+49 −11		megatron/training/utils.py
+4 −10		pretrain_vlm.py
+1 −0		...sts/test_cases/gpt/gpt3_345m_nightly_dgx_a100_1N8G_mcore_tp2_pp2_ep2_te_4experts2parallel/model_config.yaml
+0 −1		...s/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp2_resume_torch_dist_rope_embeddings_dgx_a100_1N8G/model_config.yaml
+0 −1		tests/functional_tests/test_cases/gpt/gpt3_mr_mcore_te_tp1_pp2_rope_embeddings_dgx_a100_1N8G/model_config.yaml
+1 −0		tests/functional_tests/test_cases/gpt/gpt3_nightly_mcore_te_tp2_pp1_modelopt_distill_resume/model_config.yaml
+1 −0		...imodal-llava/multimodal_llava_mr_mcore_te_tp4_pp1_freeze_vit_freeze_lm_dgx_a100_1N8G/golden_values_dev.json
+1 −0		...imodal-llava/multimodal_llava_mr_mcore_te_tp4_pp1_freeze_vit_freeze_lm_dgx_a100_1N8G/golden_values_lts.json
+57 −0		.../multimodal-llava/multimodal_llava_mr_mcore_te_tp4_pp1_freeze_vit_freeze_lm_dgx_a100_1N8G/model_config.yaml
+1 −0		...ava/multimodal_llava_mr_mcore_te_tp4_pp1_freeze_vit_freeze_lm_dist_opt_dgx_a100_1N8G/golden_values_dev.json
+1 −0		...ava/multimodal_llava_mr_mcore_te_tp4_pp1_freeze_vit_freeze_lm_dist_opt_dgx_a100_1N8G/golden_values_lts.json
+58 −0		...al-llava/multimodal_llava_mr_mcore_te_tp4_pp1_freeze_vit_freeze_lm_dist_opt_dgx_a100_1N8G/model_config.yaml
+1 −0		...ional_tests/test_cases/t5/t5_220m_nightly_dgx_a100_1N8G_mcore_te_tp1_pp1_vp1_resume_torch/model_config.yaml
+1 −0		tests/functional_tests/test_cases/t5/t5_220m_nightly_dgx_a100_1N8G_mcore_te_tp2_pp1_vp1/model_config.yaml
+1 −0		..._tests/test_cases/t5/t5_220m_nightly_dgx_a100_1N8G_mcore_te_tp2_pp1_vp1_sequence_parallel/model_config.yaml
+2 −0		tests/test_utils/recipes/multimodal-llava.yaml
+68 −0		tests/unit_tests/dist_checkpointing/test_flattened_resharding.py
+33 −0		tests/unit_tests/dist_checkpointing/test_serialization.py
+6 −8		tests/unit_tests/inference/engines/test_mcore_engine.py
+3 −3		tests/unit_tests/inference/test_common_inference_params.py
+2 −2		tests/unit_tests/inference/test_scheduler.py
+2 −2		tests/unit_tests/inference/text_generation_controllers/test_encoder_decoder_text_generation_controller.py
+13 −13		tests/unit_tests/inference/text_generation_controllers/test_simple_text_generation_controller.py
+47 −0		tests/unit_tests/test_optimizer.py
+44 −0		tests/unit_tests/transformer/moe/test_aux_loss.py
+59 −0		tests/unit_tests/transformer/moe/test_routers.py