Update tokenization.py

bigdata-ustc · Mar 10, 2024 · 8fd96b2 · 8fd96b2
1 parent dbe8936
commit 8fd96b2
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/EduNLP/SIF/tokenization/text/tokenization.py b/EduNLP/SIF/tokenization/text/tokenization.py
@@ -77,7 +77,7 @@ def tokenize(text,
                 token for token in word_tokenize(text)
                 if token not in stopwords and token.strip()
             ]
-        except:
+        except OSError:
             nltk.download('punkt')
         return [
             token for token in word_tokenize(text)
@@ -101,7 +101,7 @@ def tokenize(text,
             huggingface_tokenizer.models.BPE())
         try:
             tokenizer.load(bpe_json, pretty=True)
-        except:
+        except OSError:
             if (bpe_trainfile is None):
                 raise OSError("bpe train file not found, using %s." %
                               bpe_trainfile)