From d2e779f0e3b83d8c76b635a6367e320622255218 Mon Sep 17 00:00:00 2001 From: jp Date: Sat, 19 Aug 2023 09:18:39 +0800 Subject: [PATCH] set TOKENIZERS_PARALLELISM if not --- utils/pruning.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/utils/pruning.py b/utils/pruning.py index 2c838f6..f632a89 100644 --- a/utils/pruning.py +++ b/utils/pruning.py @@ -1,3 +1,4 @@ +import os from typing import Dict, List, Tuple from sentence_transformers import SentenceTransformer import spacy @@ -5,6 +6,8 @@ import torch import torch.nn.functional as F +if os.getenv("TOKENIZERS_PARALLELISM") is None: + os.environ["TOKENIZERS_PARALLELISM"] = "false" encoder = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2") nlp = spacy.load("en_core_web_sm")