Always use inferred extension (#183)

* Always use inferred extension * fmt * bump version
allenai · Aug 6, 2024 · 4322ad0 · 4322ad0
1 parent abb67f7
commit 4322ad0
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 3 deletions.
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "dolma"
-version = "1.0.7"
+version = "1.0.8"
 description = "Data filters"
 license = { text = "Apache-2.0" }
 readme = "README.md"

diff --git a/src/shard.rs b/src/shard.rs
@@ -113,8 +113,8 @@ impl Shard {
             }
             if !shard_inputs.is_empty() {
                 let output = format!(
-                    "{}/{}-{:04}.json.gz",
-                    stream_config.output.path, stream_config.name, stream_shard_count
+                    "{}/{}-{:04}.json{}",
+                    stream_config.output.path, stream_config.name, stream_shard_count, output_ext
                 );
                 let shard = Shard {
                     inputs: shard_inputs.clone(),