set TOKENIZERS_PARALLELISM if not

defog-ai · Aug 19, 2023 · d2e779f · d2e779f
1 parent 3259525
commit d2e779f
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/utils/pruning.py b/utils/pruning.py
@@ -1,10 +1,13 @@
+import os
 from typing import Dict, List, Tuple
 from sentence_transformers import SentenceTransformer
 import spacy
 import pickle
 import torch
 import torch.nn.functional as F
 
+if os.getenv("TOKENIZERS_PARALLELISM") is None:
+    os.environ["TOKENIZERS_PARALLELISM"] = "false"
 encoder = SentenceTransformer("sentence-transformers/all-MiniLM-L6-v2")
 nlp = spacy.load("en_core_web_sm")