Count Bytes and Docs (#186)

* added option to count bytes

* version

* added document counter

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "dolma"
-version = "1.0.9"
+version = "1.0.10"
 description = "Data filters"
 license = { text = "Apache-2.0" }
 readme = "README.md"

python/dolma/taggers/length.py

@@ -18,6 +18,19 @@
 from ..core.utils import split_paragraphs
 
 
+@TaggerRegistry.add("bytes_length_v1")
+class BytesLengthV1(BaseTagger):
+    def predict(self, doc: Document) -> DocResult:
+        score = len(doc.text.encode("utf-8"))
+        return DocResult(doc=doc, spans=[Span(start=0, end=len(doc.text), type="bytes", score=score)])
+
+
+@TaggerRegistry.add("doc_count_v1")
+class DocCountLengthV1(BaseTagger):
+    def predict(self, doc: Document) -> DocResult:
+        return DocResult(doc=doc, spans=[Span(start=0, end=len(doc.text), type="docs", score=1)])
+
+
 @TaggerRegistry.add("char_length_v1")
 class CharLengthV1(BaseTagger):
     def predict(self, doc: Document) -> DocResult: