data_download.py consistent with process_data.py (mlcommons#249)

PenghuiCheng · Apr 4, 2019 · 25cdb0b · 25cdb0b
1 parent 7c2fb6f
commit 25cdb0b
Showing 1 changed file with 2 additions and 1 deletion.
diff --git a/translation/tensorflow/transformer/data_download.py b/translation/tensorflow/transformer/data_download.py
@@ -73,6 +73,7 @@
 
 # Strings to inclue in the generated files.
 _PREFIX = "wmt32k"
+_ENCODE_TAG = "encoded"
 _TRAIN_TAG = "train"
 _EVAL_TAG = "dev"  # Following WMT and Tensor2Tensor conventions, in which the
                    # evaluation datasets are tagged as "dev" for development.
@@ -307,7 +308,7 @@ def encode_and_save_files(
 def shard_filename(path, tag, shard_num, total_shards):
   """Create filename for data shard."""
   return os.path.join(
-      path, "%s-%s-%.5d-of-%.5d" % (_PREFIX, tag, shard_num, total_shards))
+      path, "%s-%s-%s-%.5d-of-%.5d" % (_PREFIX, _ENCODE_TAG, tag, shard_num, total_shards))
 
 
 def shuffle_records(fname):