From 4b824c99baa26cbc5f06722a81555675054dae39 Mon Sep 17 00:00:00 2001
From: Evildoor <evildoor256@gmail.com>
Date: Tue, 25 Aug 2020 18:55:01 +0300
Subject: [PATCH] Add a script comparing normal and batch modes.

---
 Utils/Dataflow/data4es/batch_stage/inp      | 10 ++++++++++
 Utils/Dataflow/data4es/batch_stage/run.sh   | 15 +++++++++++++++
 Utils/Dataflow/data4es/batch_stage/stage.py | 10 ++++++++++
 3 files changed, 35 insertions(+)
 create mode 100644 Utils/Dataflow/data4es/batch_stage/inp
 create mode 100755 Utils/Dataflow/data4es/batch_stage/run.sh

diff --git a/Utils/Dataflow/data4es/batch_stage/inp b/Utils/Dataflow/data4es/batch_stage/inp
new file mode 100644
index 000000000..c67599f38
--- /dev/null
+++ b/Utils/Dataflow/data4es/batch_stage/inp
@@ -0,0 +1,10 @@
+{}
+{"!@#$": true}
+{"df": 1}
+{"df": "abc"}
+{"df": "abc1"}
+{"df": "absdfg"}
+{"df": "Some text here."}
+{"df": "abc", "fd": true}
+{"d": "abc"}
+{"df": "abc"}
diff --git a/Utils/Dataflow/data4es/batch_stage/run.sh b/Utils/Dataflow/data4es/batch_stage/run.sh
new file mode 100755
index 000000000..f5959c4c8
--- /dev/null
+++ b/Utils/Dataflow/data4es/batch_stage/run.sh
@@ -0,0 +1,15 @@
+source ../../shell_lib/eop_filter
+
+cmd="./stage.py -m s"
+cmd_batch2="./stage.py -b 2 -m s"
+cmd_batch100="./stage.py -b 100 -m s"
+
+# Various tests that should produce the same results.
+
+# Stage chains.
+# these differ by size without eop_filter at the end
+cat inp | $cmd | eop_filter | $cmd | eop_filter > outp1
+cat inp | $cmd_batch2 | eop_filter | $cmd_batch2 | eop_filter > outp2
+cat inp | $cmd_batch100 | eop_filter | $cmd_batch100 | eop_filter > outp100
+cat inp | $cmd | eop_filter | $cmd_batch2 | eop_filter > outp12
+cat inp | $cmd_batch2 | eop_filter | $cmd | eop_filter > outp21
diff --git a/Utils/Dataflow/data4es/batch_stage/stage.py b/Utils/Dataflow/data4es/batch_stage/stage.py
index 96157aa34..45aca1489 100755
--- a/Utils/Dataflow/data4es/batch_stage/stage.py
+++ b/Utils/Dataflow/data4es/batch_stage/stage.py
@@ -63,12 +63,22 @@ def main(args):
     stage.set_output_message_type(messageType.JSON)
     stage.set_default_arguments(bnc='')
 
+    # Accept batch size from command line.
+    # This is cheating because batch size is supposed to be set by
+    # stage developer, not received from command line (so,
+    # from supervisor). However, this is done in this illustrative
+    # stage to simplify a process of comparing the results of
+    # normal mode and batch mode with different batch sizes.
+    stage.add_argument('-b', action='store', type=int, help='Batch size.',
+                       default=1, dest='bsize')
+
     stage.process = process
 
     exit_code = 0
     exc_info = None
     try:
         stage.configure(args)
+        stage.set_batch_size(stage.ARGS.bsize)
         stage.run()
     except (DataflowException, RuntimeError), err:
         if str(err):