dense for gptq

AllentDan · Jan 14, 2025 · 9aaf0b9 · 9aaf0b9
1 parent 9c13834
commit 9aaf0b9
Show file tree

Hide file tree

Showing 2 changed files with 17 additions and 0 deletions.
diff --git a/lmdeploy/lite/apis/gptq.py b/lmdeploy/lite/apis/gptq.py
@@ -51,8 +51,11 @@ def auto_gptq(model: str,
     from auto_gptq.modeling._const import SUPPORTED_MODELS
 
     from ..modeling.internlm2_gptq import InternLM2GPTQForCausalLM
+    from ..modeling.internlm3_gptq import InternLM3GPTQForCausalLM
     SUPPORTED_MODELS.append('internlm2')
     GPTQ_CAUSAL_LM_MODEL_MAP.update(dict(internlm2=InternLM2GPTQForCausalLM))
+    SUPPORTED_MODELS.append('internlm3')
+    GPTQ_CAUSAL_LM_MODEL_MAP.update(dict(internlm3=InternLM3GPTQForCausalLM))
 
     pretrained_model_dir = model
     quantized_model_dir = work_dir

diff --git a/lmdeploy/lite/modeling/internlm3_gptq.py b/lmdeploy/lite/modeling/internlm3_gptq.py
@@ -0,0 +1,14 @@
+# Copyright (c) OpenMMLab. All rights reserved.
+from auto_gptq.modeling import BaseGPTQForCausalLM
+
+
+class InternLM3GPTQForCausalLM(BaseGPTQForCausalLM):
+    layer_type = 'InternLM3DecoderLayer'
+    layers_block_name = 'model.layers'
+    outside_layer_modules = ['model.embed_tokens', 'model.norm']
+    inside_layer_modules = [
+        ['self_attn.k_proj', 'self_attn.v_proj', 'self_attn.q_proj'],
+        ['self_attn.o_proj'],
+        ['mlp.up_proj', 'mlp.gate_proj'],
+        ['mlp.down_proj'],
+    ]