From f65e116fb29a43ec1e1b4183a699ecd1d6b8f82b Mon Sep 17 00:00:00 2001 From: Benjamin Bossan Date: Thu, 17 Oct 2024 16:37:40 +0200 Subject: [PATCH] DOC Improve docs for layers_pattern argument Addresses part of #2155. Also fix type annotations where appropriate. --- src/peft/tuners/boft/config.py | 10 ++++++---- src/peft/tuners/fourierft/config.py | 10 ++++++---- src/peft/tuners/hra/config.py | 10 ++++++---- src/peft/tuners/loha/config.py | 10 ++++++---- src/peft/tuners/lokr/config.py | 10 ++++++---- src/peft/tuners/lora/config.py | 8 +++++--- src/peft/tuners/oft/config.py | 10 ++++++---- src/peft/tuners/vblora/config.py | 8 +++++--- src/peft/tuners/vera/config.py | 22 ++++++++++++---------- 9 files changed, 58 insertions(+), 40 deletions(-) diff --git a/src/peft/tuners/boft/config.py b/src/peft/tuners/boft/config.py index dcae4c0841..7559303272 100644 --- a/src/peft/tuners/boft/config.py +++ b/src/peft/tuners/boft/config.py @@ -53,9 +53,10 @@ class BOFTConfig(PeftConfig): The layer indexes to transform, if this argument is specified, it will apply the BOFT transformations on the layer indexes that are specified in this list. If a single integer is passed, it will apply the BOFT transformations on the layer at this index. - layers_pattern (`str`): + layers_pattern (`Optional[Union[List[str], str]]`): The layer pattern name, used only if `layers_to_transform` is different from `None` and if the layer - pattern is not in the common layers pattern. + pattern is not in the common layers pattern. This should target the `nn.ModuleList` of the model, which is + often called `'layers'` or `'h'`. """ boft_block_size: int = field( @@ -129,10 +130,11 @@ class BOFTConfig(PeftConfig): "help": "The layer indexes to transform, is this argument is specified, PEFT will transform only the layers indexes that are specified inside this list. If a single integer is passed, PEFT will transform only the layer at this index." }, ) - layers_pattern: Optional[str] = field( + layers_pattern: Optional[Union[list[str], str]] = field( default=None, metadata={ - "help": "The layer pattern name, used only if `layers_to_transform` is different to None and if the layer pattern is not in the common layers pattern." + "help": "The layer pattern name, used only if `layers_to_transform` is different to None and if the layer pattern is not in the common layers pattern. " + "This should target the `nn.ModuleList` of the model, which is often called `'layers'` or `'h'`." }, ) diff --git a/src/peft/tuners/fourierft/config.py b/src/peft/tuners/fourierft/config.py index 1efaa22f37..93aaf85f02 100644 --- a/src/peft/tuners/fourierft/config.py +++ b/src/peft/tuners/fourierft/config.py @@ -68,9 +68,10 @@ class FourierFTConfig(PeftConfig): The layer indexes to transform, is this argument is specified, PEFT will transform only the layers indexes that are specified inside this list. If a single integer is passed, PEFT will transform only the layer at this index. - layers_pattern (`str`): + layers_pattern (`Optional[Union[List[str], str]]`): The layer pattern name, used only if `layers_to_transform` is different to None and if the layer pattern is - not in the common layers pattern. + not in the common layers pattern. This should target the `nn.ModuleList` of the model, which is often + called `'layers'` or `'h'`. n_frequency_pattern (`dict`): The mapping from layer names or regexp expression to n_frequency which are different from the default specified. For example, `{model.decoder.layers.0.encoder_attn.k_proj: 1000`}. @@ -154,12 +155,13 @@ class FourierFTConfig(PeftConfig): ) }, ) - layers_pattern: Optional[str] = field( + layers_pattern: Optional[Union[list[str], str]] = field( default=None, metadata={ "help": ( "The layer pattern name, used only if `layers_to_transform` is different to None and if the layer" - " pattern is not in the common layers pattern." + " pattern is not in the common layers pattern. This should target the `nn.ModuleList` of the " + "model, which is often called `'layers'` or `'h'`." ) }, ) diff --git a/src/peft/tuners/hra/config.py b/src/peft/tuners/hra/config.py index 01e90471a5..81997bbfeb 100644 --- a/src/peft/tuners/hra/config.py +++ b/src/peft/tuners/hra/config.py @@ -50,8 +50,9 @@ class HRAConfig(PeftConfig): The layer indices to transform. If a list of ints is passed, it will apply the adapter to the layer indices that are specified in this list. If a single integer is passed, it will apply the transformations on the layer at this index. - layers_pattern (`str`): - The layer pattern name, used only if `layers_to_transform` is different from `None`. + layers_pattern (`Optional[Union[List[str], str]]`): + The layer pattern name, used only if `layers_to_transform` is different from `None`. This should target the + `nn.ModuleList` of the model, which is often called `'layers'` or `'h'`. rank_pattern (`dict`): The mapping from layer names or regexp expression to ranks which are different from the default rank specified by `r`. @@ -96,10 +97,11 @@ class HRAConfig(PeftConfig): "help": "The layer indexes to transform, is this argument is specified, PEFT will transform only the layers indexes that are specified inside this list. If a single integer is passed, PEFT will transform only the layer at this index." }, ) - layers_pattern: Optional[str] = field( + layers_pattern: Optional[Union[list[str], str]] = field( default=None, metadata={ - "help": "The layer pattern name, used only if `layers_to_transform` is different to None and if the layer pattern is not in the common layers pattern." + "help": "The layer pattern name, used only if `layers_to_transform` is different to None and if the layer pattern is not in the common layers pattern. " + "This should target the `nn.ModuleList` of the model, which is often called `'layers'` or `'h'`." }, ) bias: str = field(default="none", metadata={"help": "Bias type for HRA. Can be 'none', 'all' or 'hra_only'"}) diff --git a/src/peft/tuners/loha/config.py b/src/peft/tuners/loha/config.py index 3f47444eff..aa05121be2 100644 --- a/src/peft/tuners/loha/config.py +++ b/src/peft/tuners/loha/config.py @@ -55,8 +55,9 @@ class LoHaConfig(LycorisConfig): The layer indices to transform. If a list of ints is passed, it will apply the adapter to the layer indices that are specified in this list. If a single integer is passed, it will apply the transformations on the layer at this index. - layers_pattern (`str`): - The layer pattern name, used only if `layers_to_transform` is different from `None`. + layers_pattern (`Optional[Union[List[str], str]]`): + The layer pattern name, used only if `layers_to_transform` is different from `None`. This should target the + `nn.ModuleList` of the model, which is often called `'layers'` or `'h'`. rank_pattern (`dict`): The mapping from layer names or regexp expression to ranks which are different from the default rank specified by `r`. @@ -108,10 +109,11 @@ class LoHaConfig(LycorisConfig): "help": "The layer indexes to transform, is this argument is specified, PEFT will transform only the layers indexes that are specified inside this list. If a single integer is passed, PEFT will transform only the layer at this index." }, ) - layers_pattern: Optional[str] = field( + layers_pattern: Optional[Union[list[str], str]] = field( default=None, metadata={ - "help": "The layer pattern name, used only if `layers_to_transform` is different to None and if the layer pattern is not in the common layers pattern." + "help": "The layer pattern name, used only if `layers_to_transform` is different to None and if the layer pattern is not in the common layers pattern. " + "This should target the `nn.ModuleList` of the model, which is often called `'layers'` or `'h'`." }, ) modules_to_save: Optional[list[str]] = field( diff --git a/src/peft/tuners/lokr/config.py b/src/peft/tuners/lokr/config.py index 0f8e991556..b4879af558 100644 --- a/src/peft/tuners/lokr/config.py +++ b/src/peft/tuners/lokr/config.py @@ -59,8 +59,9 @@ class LoKrConfig(LycorisConfig): The layer indices to transform. If a list of ints is passed, it will apply the adapter to the layer indices that are specified in this list. If a single integer is passed, it will apply the transformations on the layer at this index. - layers_pattern (`str`): - The layer pattern name, used only if `layers_to_transform` is different from `None`. + layers_pattern (`Optional[Union[List[str], str]]`): + The layer pattern name, used only if `layers_to_transform` is different from `None`. This should target the + `nn.ModuleList` of the model, which is often called `'layers'` or `'h'`. rank_pattern (`dict`): The mapping from layer names or regexp expression to ranks which are different from the default rank specified by `r`. @@ -117,10 +118,11 @@ class LoKrConfig(LycorisConfig): "help": "The layer indexes to transform, is this argument is specified, PEFT will transform only the layers indexes that are specified inside this list. If a single integer is passed, PEFT will transform only the layer at this index." }, ) - layers_pattern: Optional[str] = field( + layers_pattern: Optional[Union[list[str], str]] = field( default=None, metadata={ - "help": "The layer pattern name, used only if `layers_to_transform` is different to None and if the layer pattern is not in the common layers pattern." + "help": "The layer pattern name, used only if `layers_to_transform` is different to None and if the layer pattern is not in the common layers pattern. " + "This should target the `nn.ModuleList` of the model, which is often called `'layers'` or `'h'`." }, ) modules_to_save: Optional[list[str]] = field( diff --git a/src/peft/tuners/lora/config.py b/src/peft/tuners/lora/config.py index 6fb383a274..10522dcc42 100644 --- a/src/peft/tuners/lora/config.py +++ b/src/peft/tuners/lora/config.py @@ -123,8 +123,9 @@ class LoraConfig(PeftConfig): The layer indices to transform. If a list of ints is passed, it will apply the adapter to the layer indices that are specified in this list. If a single integer is passed, it will apply the transformations on the layer at this index. - layers_pattern (`str`): - The layer pattern name, used only if `layers_to_transform` is different from `None`. + layers_pattern (`Optional[Union[List[str], str]]`): + The layer pattern name, used only if `layers_to_transform` is different from `None`. This should target the + `nn.ModuleList` of the model, which is often called `'layers'` or `'h'`. rank_pattern (`dict`): The mapping from layer names or regexp expression to ranks which are different from the default rank specified by `r`. @@ -229,7 +230,8 @@ class LoraConfig(PeftConfig): default=None, metadata={ "help": "The layer pattern name, used only if `layers_to_transform` is different to None and if the layer pattern is not in the common layers pattern." - "This only works when target_modules is a list of str." + "This only works when target_modules is a list of str. This should target the `nn.ModuleList` of the " + "model, which is often called `'layers'` or `'h'`." }, ) rank_pattern: Optional[dict] = field( diff --git a/src/peft/tuners/oft/config.py b/src/peft/tuners/oft/config.py index 85bfc9cc24..40aad205d3 100644 --- a/src/peft/tuners/oft/config.py +++ b/src/peft/tuners/oft/config.py @@ -54,8 +54,9 @@ class OFTConfig(PeftConfig): The layer indices to transform. If a list of ints is passed, it will apply the adapter to the layer indices that are specified in this list. If a single integer is passed, it will apply the transformations on the layer at this index. - layers_pattern (`str`): - The layer pattern name, used only if `layers_to_transform` is different from `None`. + layers_pattern (`Optional[Union[List[str], str]]`): + The layer pattern name, used only if `layers_to_transform` is different from `None`. This should target the + `nn.ModuleList` of the model, which is often called `'layers'` or `'h'`. rank_pattern (`dict`): The mapping from layer names or regexp expression to ranks which are different from the default rank specified by `r`. @@ -117,10 +118,11 @@ class OFTConfig(PeftConfig): "help": "The layer indexes to transform, is this argument is specified, PEFT will transform only the layers indexes that are specified inside this list. If a single integer is passed, PEFT will transform only the layer at this index." }, ) - layers_pattern: Optional[str] = field( + layers_pattern: Optional[Union[list[str], str]] = field( default=None, metadata={ - "help": "The layer pattern name, used only if `layers_to_transform` is different to None and if the layer pattern is not in the common layers pattern." + "help": "The layer pattern name, used only if `layers_to_transform` is different to None and if the layer pattern is not in the common layers pattern. " + "This should target the `nn.ModuleList` of the model, which is often called `'layers'` or `'h'`." }, ) modules_to_save: Optional[list[str]] = field( diff --git a/src/peft/tuners/vblora/config.py b/src/peft/tuners/vblora/config.py index 879f54ea11..a493c35312 100644 --- a/src/peft/tuners/vblora/config.py +++ b/src/peft/tuners/vblora/config.py @@ -79,8 +79,9 @@ class VBLoRAConfig(PeftConfig): The layer indices to transform. If a list of ints is passed, it will apply the adapter to the layer indices that are specified in this list. If a single integer is passed, it will apply the transformations on the layer at this index. - layers_pattern (`str`): - The layer pattern name, used only if `layers_to_transform` is different from `None`. + layers_pattern (`Optional[Union[List[str], str]]`): + The layer pattern name, used only if `layers_to_transform` is different from `None`. This should target the + `nn.ModuleList` of the model, which is often called `'layers'` or `'h'`. """ r: int = field(default=4, metadata={"help": "The rank of incremental matrices."}) @@ -169,7 +170,8 @@ class VBLoRAConfig(PeftConfig): default=None, metadata={ "help": "The layer indexes to transform, is this argument is specified, PEFT will transform only the layers indexes that are specified inside this list. If a single integer is passed, PEFT will transform only the layer at this index. " - "This only works when target_modules is a list of str." + "This only works when target_modules is a list of str. This should target the `nn.ModuleList` of the " + "model, which is often called `'layers'` or `'h'`." }, ) layers_pattern: Optional[Union[list[str], str]] = field( diff --git a/src/peft/tuners/vera/config.py b/src/peft/tuners/vera/config.py index 1601c8c0e6..45ef903756 100644 --- a/src/peft/tuners/vera/config.py +++ b/src/peft/tuners/vera/config.py @@ -11,10 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import annotations import warnings from dataclasses import dataclass, field -from typing import List, Optional, Union +from typing import Optional, Union from peft.config import PeftConfig from peft.utils import PeftType @@ -61,14 +62,14 @@ class VeraConfig(PeftConfig): The layer indexes to transform, if this argument is specified, it will apply the Vera transformations on the layer indexes that are specified in this list. If a single integer is passed, it will apply the Vera transformations on the layer at this index. - layers_pattern (`str`): - The layer pattern name, used only if `layers_to_transform` is different from `None` and if the layer - pattern is not in the common layers pattern. + layers_pattern (`Optional[Union[List[str], str]]`): + The layer pattern name, used only if `layers_to_transform` is different from `None`. This should target the + `nn.ModuleList` of the model, which is often called `'layers'` or `'h'`. """ r: int = field(default=256, metadata={"help": "Vera attention dimension"}) - target_modules: Optional[Union[List[str], str]] = field( + target_modules: Optional[Union[list[str], str]] = field( default=None, metadata={ "help": ( @@ -104,7 +105,7 @@ class VeraConfig(PeftConfig): metadata={"help": "Set this to True if the layer to replace stores weight like (fan_in, fan_out)"}, ) bias: str = field(default="none", metadata={"help": "Bias type for Vera. Can be 'none', 'all' or 'vera_only'"}) - modules_to_save: Optional[List[str]] = field( + modules_to_save: Optional[list[str]] = field( default=None, metadata={ "help": ( @@ -123,7 +124,7 @@ class VeraConfig(PeftConfig): ), }, ) - layers_to_transform: Optional[Union[List[int], int]] = field( + layers_to_transform: Optional[Union[list[int], int]] = field( default=None, metadata={ "help": ( @@ -133,12 +134,13 @@ class VeraConfig(PeftConfig): ) }, ) - layers_pattern: Optional[str] = field( + layers_pattern: Optional[Union[list[str], str]] = field( default=None, metadata={ "help": ( - "The layer pattern name, used only if `layers_to_transform` is different to None and if the layer" - " pattern is not in the common layers pattern." + "The layer pattern name, used only if `layers_to_transform` is different to None and if the layer " + "pattern is not in the common layers pattern. This should target the `nn.ModuleList` of the " + "model, which is often called `'layers'` or `'h'`." ) }, )