Add truncation=True to prepare_for_model function

sentence_transformers/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.2.6"
+__version__ = "0.2.7"
 __DOWNLOAD_SERVER__ = 'https://public.ukp.informatik.tu-darmstadt.de/reimers/sentence-transformers/v0.2/'
 from .datasets import SentencesDataset, SentenceLabelDataset, ParallelSentencesDataset
 from .data_samplers import LabelSampler

sentence_transformers/models/ALBERT.py

@@ -61,7 +61,7 @@ def get_sentence_features(self, tokens: List[int], pad_seq_length: int):
         :return: embedding ids, segment ids and mask for the sentence
         """
         pad_seq_length = min(pad_seq_length, self.max_seq_length) + 3 #Add space for special tokens
-        return self.tokenizer.prepare_for_model(tokens, max_length=pad_seq_length, pad_to_max_length=True, return_tensors='pt')
+        return self.tokenizer.prepare_for_model(tokens, max_length=pad_seq_length, pad_to_max_length=True, return_tensors='pt', truncation=True)
 
 
     def get_config_dict(self):

sentence_transformers/models/BERT.py

@@ -28,7 +28,6 @@ def __init__(self, model_name_or_path: str, max_seq_length: int = 128, do_lower_
         self.tokenizer = BertTokenizer.from_pretrained(model_name_or_path, **tokenizer_args)
 
 
-
     def forward(self, features):
         """Returns token_embeddings, cls_token"""
         output_states = self.bert(**features)
@@ -62,7 +61,7 @@ def get_sentence_features(self, tokens: List[int], pad_seq_length: int):
         """
         pad_seq_length = min(pad_seq_length, self.max_seq_length) + 2  ##Add Space for CLS + SEP token
 
-        return self.tokenizer.prepare_for_model(tokens, max_length=pad_seq_length, pad_to_max_length=True, return_tensors='pt')
+        return self.tokenizer.prepare_for_model(tokens, max_length=pad_seq_length, pad_to_max_length=True, return_tensors='pt', truncation=True)
 
 
     def get_config_dict(self):

sentence_transformers/models/CamemBERT.py

@@ -63,7 +63,7 @@ def get_sentence_features(self, tokens: List[int], pad_seq_length: int):
         :return: embedding ids, segment ids and mask for the sentence
         """
         pad_seq_length = min(pad_seq_length, self.max_seq_length) + 3 #Add space for special tokens
-        return self.tokenizer.prepare_for_model(tokens, max_length=pad_seq_length, pad_to_max_length=True, return_tensors='pt')
+        return self.tokenizer.prepare_for_model(tokens, max_length=pad_seq_length, pad_to_max_length=True, return_tensors='pt', truncation=True)
 
     def get_config_dict(self):
         return {key: self.__dict__[key] for key in self.config_keys}

sentence_transformers/models/DistilBERT.py

@@ -62,7 +62,7 @@ def get_sentence_features(self, tokens: List[int], pad_seq_length: int):
         :return: embedding ids, segment ids and mask for the sentence
         """
         pad_seq_length = min(pad_seq_length, self.max_seq_length) + 2 #Add space for special tokens
-        return self.tokenizer.prepare_for_model(tokens, max_length=pad_seq_length, pad_to_max_length=True, return_tensors='pt')
+        return self.tokenizer.prepare_for_model(tokens, max_length=pad_seq_length, pad_to_max_length=True, return_tensors='pt', truncation=True)
 
     def get_config_dict(self):
         return {key: self.__dict__[key] for key in self.config_keys}

sentence_transformers/models/RoBERTa.py

@@ -61,7 +61,7 @@ def get_sentence_features(self, tokens: List[int], pad_seq_length: int):
         :return: embedding ids, segment ids and mask for the sentence
         """
         pad_seq_length = min(pad_seq_length, self.max_seq_length) + 2 ##Add Space for CLS + SEP token
-        return self.tokenizer.prepare_for_model(tokens, max_length=pad_seq_length, pad_to_max_length=True, return_tensors='pt')
+        return self.tokenizer.prepare_for_model(tokens, max_length=pad_seq_length, pad_to_max_length=True, return_tensors='pt', truncation=True)
 
     def get_config_dict(self):
         return {key: self.__dict__[key] for key in self.config_keys}

sentence_transformers/models/T5.py

@@ -61,7 +61,7 @@ def get_sentence_features(self, tokens: List[int], pad_seq_length: int):
         """
 
         pad_seq_length = min(pad_seq_length, self.max_seq_length)
-        return self.tokenizer.prepare_for_model(tokens, max_length=pad_seq_length, pad_to_max_length=True, return_tensors='pt')
+        return self.tokenizer.prepare_for_model(tokens, max_length=pad_seq_length, pad_to_max_length=True, return_tensors='pt', truncation=True)
 
     def get_config_dict(self):
         return {key: self.__dict__[key] for key in self.config_keys}

sentence_transformers/models/Transformer.py

@@ -58,7 +58,7 @@ def get_sentence_features(self, tokens: List[int], pad_seq_length: int):
         :return: embedding ids, segment ids and mask for the sentence
         """
         pad_seq_length = min(pad_seq_length, self.max_seq_length) + 3 #Add space for special tokens
-        return self.tokenizer.prepare_for_model(tokens, max_length=pad_seq_length, pad_to_max_length=True, return_tensors='pt')
+        return self.tokenizer.prepare_for_model(tokens, max_length=pad_seq_length, pad_to_max_length=True, return_tensors='pt', truncation=True)
 
     def get_config_dict(self):
         return {key: self.__dict__[key] for key in self.config_keys}

sentence_transformers/models/XLMRoBERTa.py

@@ -63,7 +63,7 @@ def get_sentence_features(self, tokens: List[int], pad_seq_length: int):
         :return: embedding ids, segment ids and mask for the sentence
         """
         pad_seq_length = min(pad_seq_length, self.max_seq_length) + 2 #Add space for special tokens
-        return self.tokenizer.prepare_for_model(tokens, max_length=pad_seq_length, pad_to_max_length=True, return_tensors='pt')
+        return self.tokenizer.prepare_for_model(tokens, max_length=pad_seq_length, pad_to_max_length=True, return_tensors='pt', truncation=True)
 
     def get_config_dict(self):
         return {key: self.__dict__[key] for key in self.config_keys}

sentence_transformers/models/XLNet.py

@@ -58,7 +58,7 @@ def get_sentence_features(self, tokens: List[int], pad_seq_length: int) -> Dict[
         :return: embedding ids, segment ids and mask for the sentence
         """
         pad_seq_length = min(pad_seq_length, self.max_seq_length) + 3 #Add space for special tokens
-        return self.tokenizer.prepare_for_model(tokens, max_length=pad_seq_length, pad_to_max_length=True, return_tensors='pt')
+        return self.tokenizer.prepare_for_model(tokens, max_length=pad_seq_length, pad_to_max_length=True, return_tensors='pt', truncation=True)
 
     def get_config_dict(self):
         return {key: self.__dict__[key] for key in self.config_keys}