More efficient DocBin creation in conversion

rojaAchary · Mar 5, 2021 · ef00ec7 · ef00ec7
1 parent 88d1628
commit ef00ec7
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 6 deletions.
diff --git a/pipelines/textcat_demo/scripts/convert.py b/pipelines/textcat_demo/scripts/convert.py
@@ -10,12 +10,11 @@
 
 def convert(lang: str, input_path: Path, output_path: Path):
     nlp = spacy.blank(lang)
-    docs = []
+    db = DocBin()
     for line in srsly.read_jsonl(input_path):
         doc = nlp.make_doc(line["text"])
         doc.cats = line["cats"]
-        docs.append(doc)
-    db = DocBin(docs=docs)
+        db.add(doc)
     db.to_disk(output_path)
 
 

diff --git a/pipelines/textcat_multilabel_demo/scripts/convert.py b/pipelines/textcat_multilabel_demo/scripts/convert.py
@@ -10,12 +10,11 @@
 
 def convert(lang: str, input_path: Path, output_path: Path):
     nlp = spacy.blank(lang)
-    docs = []
+    db = DocBin()
     for line in srsly.read_jsonl(input_path):
         doc = nlp.make_doc(line["text"])
         doc.cats = line["cats"]
-        docs.append(doc)
-    db = DocBin(docs=docs)
+        db.add(doc)
     db.to_disk(output_path)