Rename module

naivenlp · Nov 19, 2021 · 6dc3d2f · 6dc3d2f
1 parent df43ede
commit 6dc3d2f
Show file tree

Hide file tree

Showing 10 changed files with 24 additions and 7 deletions.
diff --git a/README.md b/README.md
@@ -1,2 +1,9 @@
 # smile-datasets
-La**S**t **mile** Dataset: Use tf.data to solve the last mile data loading problem
+La**S**t **mile** Datasets: Use `tf.data` to solve the last mile data loading problem for tensorflow.
+
+If you want to load public datasets, try:
+
+* [tensorflow/datasets](https://github.com/tensorflow/datasets)
+* [huggingface/datasets](https://github.com/huggingface/datasets)
+
+If you want to load local, personal dataset with minimized boilerplate, use **Smile Dataset**!
diff --git a/datasets/__init__.py → smile_datasets/__init__.py b/datasets/__init__.py → smile_datasets/__init__.py
diff --git a/datasets/dataset.py → smile_datasets/dataset.py b/datasets/dataset.py → smile_datasets/dataset.py
diff --git a/datasets/qa/__init__.py → smile_datasets/qa/__init__.py b/datasets/qa/__init__.py → smile_datasets/qa/__init__.py
diff --git a/datasets/qa/dataset.py → smile_datasets/qa/dataset.py b/datasets/qa/dataset.py → smile_datasets/qa/dataset.py
@@ -3,8 +3,8 @@
 from typing import Dict, List
 
 import tensorflow as tf
-from datasets import utils
-from datasets.dataset import AbcDataset
+from smile_datasets import utils
+from smile_datasets.dataset import AbcDataset
 from tokenizers import BertWordPieceTokenizer
 
 from .example import ExampleForQuestionAnswering

diff --git a/datasets/qa/example.py → smile_datasets/qa/example.py b/datasets/qa/example.py → smile_datasets/qa/example.py
diff --git a/datasets/qa/parsers.py → smile_datasets/qa/parsers.py b/datasets/qa/parsers.py → smile_datasets/qa/parsers.py
diff --git a/datasets/qa/readers.py → smile_datasets/qa/readers.py b/datasets/qa/readers.py → smile_datasets/qa/readers.py
diff --git a/datasets/utils.py → smile_datasets/utils.py b/datasets/utils.py → smile_datasets/utils.py
diff --git a/tests/qa_tests/dataset_test.py b/tests/qa_tests/dataset_test.py
@@ -1,10 +1,10 @@
 import os
 import unittest
 
-from datasets.qa import readers
-from datasets.qa.dataset import DatapipeForQuestionAnswering, DatasetForQuestionAnswering
-from datasets.qa.example import ExampleForQuestionAnswering
-from datasets.qa.parsers import ParserForQuestionAnswering
+from smile_datasets.qa import readers
+from smile_datasets.qa.dataset import DatapipeForQuestionAnswering, DatasetForQuestionAnswering
+from smile_datasets.qa.example import ExampleForQuestionAnswering
+from smile_datasets.qa.parsers import ParserForQuestionAnswering
 
 DUREADER_RUBOST_INPUT_FILE = os.path.join(os.environ["DUREADER_ROBUST_PATH"], "dev.json")
 DUREADER_CHECKLIST_INPUT_FILE = os.path.join(os.environ["DUREADER_CHECKLIST_PATH"], "dev.json")
@@ -39,6 +39,11 @@ def test_dataset_save_dureader_rubost_tfrecord(self):
             print(e)
             if idx == 5:
                 break
+
+        dataset = DatapipeForQuestionAnswering.from_dataset(d)
+        print()
+        print(next(iter(dataset)))
+
         d.save_tfrecord("testdata/dureader_rubost_dev.tfrecord")
 
         dataset = DatapipeForQuestionAnswering.from_tfrecord_files("testdata/dureader_rubost_dev.tfrecord")
@@ -51,6 +56,11 @@ def test_dataset_save_dureader_checklist_tfrecord(self):
             print(e)
             if idx == 5:
                 break
+
+        dataset = DatapipeForQuestionAnswering.from_dataset(d)
+        print()
+        print(next(iter(dataset)))
+
         d.save_tfrecord("testdata/dureader_checklist_dev.tfrecord")
 
         dataset = DatapipeForQuestionAnswering.from_tfrecord_files("testdata/dureader_checklist_dev.tfrecord")