From 18419cfaeb28ef0a7ff57e0016b6689c8d2c996b Mon Sep 17 00:00:00 2001
From: CihatAltiparmak <cihataltiparmak1@gmail.com>
Date: Wed, 24 Jul 2024 23:02:16 +0300
Subject: [PATCH 01/11] Added scripts related to visualization, running
 benchmarks etc.

---
 .../rmw_cyclonedds/config_rmw_cyclonedds.sh   |  0
 .../rmw_fastrtps/config_rmw_fastrtps.sh       |  0
 .../rmw_zenoh/config_rmw_zenoh.sh             |  5 ++
 scripts/box_plot_visualizer.py                | 68 +++++++++++++++++++
 scripts/run_all_benchmarks.sh                 | 41 +++++++++++
 5 files changed, 114 insertions(+)
 create mode 100644 middleware_configurations/rmw_cyclonedds/config_rmw_cyclonedds.sh
 create mode 100644 middleware_configurations/rmw_fastrtps/config_rmw_fastrtps.sh
 create mode 100644 middleware_configurations/rmw_zenoh/config_rmw_zenoh.sh
 create mode 100644 scripts/box_plot_visualizer.py
 create mode 100644 scripts/run_all_benchmarks.sh

diff --git a/middleware_configurations/rmw_cyclonedds/config_rmw_cyclonedds.sh b/middleware_configurations/rmw_cyclonedds/config_rmw_cyclonedds.sh
new file mode 100644
index 0000000..e69de29
diff --git a/middleware_configurations/rmw_fastrtps/config_rmw_fastrtps.sh b/middleware_configurations/rmw_fastrtps/config_rmw_fastrtps.sh
new file mode 100644
index 0000000..e69de29
diff --git a/middleware_configurations/rmw_zenoh/config_rmw_zenoh.sh b/middleware_configurations/rmw_zenoh/config_rmw_zenoh.sh
new file mode 100644
index 0000000..af68bf6
--- /dev/null
+++ b/middleware_configurations/rmw_zenoh/config_rmw_zenoh.sh
@@ -0,0 +1,5 @@
+echo "The configurations for rmw_zenoh_cpp is started!"
+# sudo sysctl -w "net.ipv4.tcp_rmem=4096 4096 4096"
+# sudo sysctl -w "net.ipv4.tcp_wmem=4096 4096 4096"
+# sudo sysctl -w "net.ipv4.tcp_mem=4096 4096 4096"
+echo "The configurations for rmw_zenoh_cpp is finished!"
\ No newline at end of file
diff --git a/scripts/box_plot_visualizer.py b/scripts/box_plot_visualizer.py
new file mode 100644
index 0000000..9f6412e
--- /dev/null
+++ b/scripts/box_plot_visualizer.py
@@ -0,0 +1,68 @@
+import matplotlib.pyplot as plt
+import numpy as np
+import json
+import os
+import sys
+
+BENCHMARK_RESULTS_DIR=sys.argv[1]
+
+middleware_colors = {
+    'rmw_zenoh_cpp' : 'orange',
+    'rmw_cyclonedds_cpp' : 'peachpuff',
+    'rmw_fastrtps_cpp' : 'tomato'}
+
+middleware_list = ["rmw_zenoh_cpp", "rmw_cyclonedds_cpp", "rmw_fastrtps_cpp"]
+
+def read_benchmark_json(file_name):
+    benchmark_json_data = None
+    with open(file_name) as f:
+        benchmark_json_data = json.load(f)
+    return benchmark_json_data
+
+def get_real_time_list_from_benchmark_json(benchmark_json_data):
+    real_time_list = []
+    for benchmark_info in benchmark_json_data["benchmarks"]:
+        if benchmark_info["run_type"] == "iteration":
+            real_time_list.append(benchmark_info["real_time"])
+    return real_time_list
+
+def get_middleware_dataset_for_scenario(scenario_name):
+    middleware_datasets = []
+    for middleware_name in middleware_list:
+        file_name = os.path.join(BENCHMARK_RESULTS_DIR, scenario_name, f"{middleware_name}.json")
+        benchmark_json_data = read_benchmark_json(file_name)
+        dataset = get_real_time_list_from_benchmark_json(benchmark_json_data)
+        middleware_datasets.append(
+            {
+                "name" : middleware_name,
+                "dataset" : dataset,
+                "color" : middleware_colors[middleware_name],
+            })
+    return middleware_datasets
+
+def plot_dataset_of_scenario(plt, middleware_datasets):
+    labels = []
+    colors = []
+    datasets = []
+
+    for x in middleware_datasets:
+        labels.append(x["name"])
+        colors.append(x["color"])
+        datasets.append(x["dataset"])
+
+    fig, ax = plt.subplots()
+    ax.set_ylabel('real time (ns)')
+
+    bplot = ax.boxplot(datasets,
+                       patch_artist=True,
+                       tick_labels=labels)
+
+    # fill with colors
+    for patch, color in zip(bplot['boxes'], colors):
+        patch.set_facecolor(color)
+
+for scenario_name in os.listdir(BENCHMARK_RESULTS_DIR):
+    middleware_datasets = get_middleware_dataset_for_scenario(scenario_name)
+    plot_dataset_of_scenario(plt, middleware_datasets)
+
+plt.show()
\ No newline at end of file
diff --git a/scripts/run_all_benchmarks.sh b/scripts/run_all_benchmarks.sh
new file mode 100644
index 0000000..69236f6
--- /dev/null
+++ b/scripts/run_all_benchmarks.sh
@@ -0,0 +1,41 @@
+# Inspired from https://unix.stackexchange.com/questions/31414/how-can-i-pass-a-command-line-argument-into-a-shell-script
+helpFunction()
+{
+   echo ""
+   echo "Usage: $0 -i initial_script -m middleware_name -d benchmark_results_directory"
+   echo -i "\t-i initial_script to run once all benchmarks are started"
+   echo -e "\t-m selected middleware to benchmark"
+   echo -e "\t-c the directory the benchmark results is saved"
+   exit 1 # Exit script after printing help
+}
+
+while getopts "i:m:d:" opt
+do
+   case "$opt" in
+      i ) initial_script="$OPTARG" ;;
+      m ) middleware_name="$OPTARG" ;;
+      d ) benchmark_results_directory="$OPTARG" ;;
+      ? ) helpFunction ;; # Print helpFunction in case parameter is non-existent
+   esac
+done
+
+# Print helpFunction in case parameters are empty
+if [ -z "$initial_script" ] || [ -z "$middleware_name" ] || [ -z "$benchmark_results_directory" ]
+then
+   echo "Some or all of the parameters are empty";
+   helpFunction
+fi
+
+echo "middleware name is $middleware_name"
+echo "benchmark results directory is $benchmark_results_directory"
+
+echo "Benchmarking is starting!"
+echo "Starting initial sctipts before bechmarks run!"
+sh "$initial_script"
+echo "Initial script has finished! Now starting to benchmark middleware with scenarios!"
+
+# ros2 daemon stop
+# ros2 launch moveit_middleware_benchmark scenario_basic_service_client_benchmark.launch.py benchmark_command_args:="--benchmark_out=middleware_bechmark_results.json --benchmark_out_format=json"
+
+# ros2 daemon stop
+# ros2 launch moveit_middleware_benchmark scenario_perception_pipeline_benchmark.launch.py benchmark_command_args:="--benchmark_out=middleware_bechmark_results.json --benchmark_out_format=json"
\ No newline at end of file

From 35d9c9798c5e6562d7c82193ff120fd17008a150 Mon Sep 17 00:00:00 2001
From: CihatAltiparmak <cihataltiparmak1@gmail.com>
Date: Thu, 25 Jul 2024 00:11:30 +0300
Subject: [PATCH 02/11] Improved run_all_benchmarks script and fixed some typos
 in this script

---
 .../rmw_zenoh/config_rmw_zenoh.sh             |  2 +-
 scripts/box_plot_visualizer.py                | 37 +++++++++++--------
 scripts/run_all_benchmarks.sh                 | 12 +++---
 3 files changed, 30 insertions(+), 21 deletions(-)

diff --git a/middleware_configurations/rmw_zenoh/config_rmw_zenoh.sh b/middleware_configurations/rmw_zenoh/config_rmw_zenoh.sh
index af68bf6..e104f70 100644
--- a/middleware_configurations/rmw_zenoh/config_rmw_zenoh.sh
+++ b/middleware_configurations/rmw_zenoh/config_rmw_zenoh.sh
@@ -2,4 +2,4 @@ echo "The configurations for rmw_zenoh_cpp is started!"
 # sudo sysctl -w "net.ipv4.tcp_rmem=4096 4096 4096"
 # sudo sysctl -w "net.ipv4.tcp_wmem=4096 4096 4096"
 # sudo sysctl -w "net.ipv4.tcp_mem=4096 4096 4096"
-echo "The configurations for rmw_zenoh_cpp is finished!"
\ No newline at end of file
+echo "The configurations for rmw_zenoh_cpp is finished!"
diff --git a/scripts/box_plot_visualizer.py b/scripts/box_plot_visualizer.py
index 9f6412e..a77a237 100644
--- a/scripts/box_plot_visualizer.py
+++ b/scripts/box_plot_visualizer.py
@@ -4,21 +4,24 @@
 import os
 import sys
 
-BENCHMARK_RESULTS_DIR=sys.argv[1]
+BENCHMARK_RESULTS_DIR = sys.argv[1]
 
 middleware_colors = {
-    'rmw_zenoh_cpp' : 'orange',
-    'rmw_cyclonedds_cpp' : 'peachpuff',
-    'rmw_fastrtps_cpp' : 'tomato'}
+    "rmw_zenoh_cpp": "orange",
+    "rmw_cyclonedds_cpp": "peachpuff",
+    "rmw_fastrtps_cpp": "tomato",
+}
 
 middleware_list = ["rmw_zenoh_cpp", "rmw_cyclonedds_cpp", "rmw_fastrtps_cpp"]
 
+
 def read_benchmark_json(file_name):
     benchmark_json_data = None
     with open(file_name) as f:
         benchmark_json_data = json.load(f)
     return benchmark_json_data
 
+
 def get_real_time_list_from_benchmark_json(benchmark_json_data):
     real_time_list = []
     for benchmark_info in benchmark_json_data["benchmarks"]:
@@ -26,20 +29,25 @@ def get_real_time_list_from_benchmark_json(benchmark_json_data):
             real_time_list.append(benchmark_info["real_time"])
     return real_time_list
 
+
 def get_middleware_dataset_for_scenario(scenario_name):
     middleware_datasets = []
     for middleware_name in middleware_list:
-        file_name = os.path.join(BENCHMARK_RESULTS_DIR, scenario_name, f"{middleware_name}.json")
+        file_name = os.path.join(
+            BENCHMARK_RESULTS_DIR, scenario_name, f"{middleware_name}.json"
+        )
         benchmark_json_data = read_benchmark_json(file_name)
         dataset = get_real_time_list_from_benchmark_json(benchmark_json_data)
         middleware_datasets.append(
             {
-                "name" : middleware_name,
-                "dataset" : dataset,
-                "color" : middleware_colors[middleware_name],
-            })
+                "name": middleware_name,
+                "dataset": dataset,
+                "color": middleware_colors[middleware_name],
+            }
+        )
     return middleware_datasets
 
+
 def plot_dataset_of_scenario(plt, middleware_datasets):
     labels = []
     colors = []
@@ -51,18 +59,17 @@ def plot_dataset_of_scenario(plt, middleware_datasets):
         datasets.append(x["dataset"])
 
     fig, ax = plt.subplots()
-    ax.set_ylabel('real time (ns)')
+    ax.set_ylabel("real time (ns)")
 
-    bplot = ax.boxplot(datasets,
-                       patch_artist=True,
-                       tick_labels=labels)
+    bplot = ax.boxplot(datasets, patch_artist=True, tick_labels=labels)
 
     # fill with colors
-    for patch, color in zip(bplot['boxes'], colors):
+    for patch, color in zip(bplot["boxes"], colors):
         patch.set_facecolor(color)
 
+
 for scenario_name in os.listdir(BENCHMARK_RESULTS_DIR):
     middleware_datasets = get_middleware_dataset_for_scenario(scenario_name)
     plot_dataset_of_scenario(plt, middleware_datasets)
 
-plt.show()
\ No newline at end of file
+plt.show()
diff --git a/scripts/run_all_benchmarks.sh b/scripts/run_all_benchmarks.sh
index 69236f6..4faff3c 100644
--- a/scripts/run_all_benchmarks.sh
+++ b/scripts/run_all_benchmarks.sh
@@ -4,8 +4,8 @@ helpFunction()
    echo ""
    echo "Usage: $0 -i initial_script -m middleware_name -d benchmark_results_directory"
    echo -i "\t-i initial_script to run once all benchmarks are started"
-   echo -e "\t-m selected middleware to benchmark"
-   echo -e "\t-c the directory the benchmark results is saved"
+   echo -m "\t-m selected middleware to benchmark"
+   echo -d "\t-d the directory the benchmark results is saved"
    exit 1 # Exit script after printing help
 }
 
@@ -30,12 +30,14 @@ echo "middleware name is $middleware_name"
 echo "benchmark results directory is $benchmark_results_directory"
 
 echo "Benchmarking is starting!"
-echo "Starting initial sctipts before bechmarks run!"
+echo "Starting initial scripts before benchmarks run!"
 sh "$initial_script"
 echo "Initial script has finished! Now starting to benchmark middleware with scenarios!"
 
 # ros2 daemon stop
 # ros2 launch moveit_middleware_benchmark scenario_basic_service_client_benchmark.launch.py benchmark_command_args:="--benchmark_out=middleware_bechmark_results.json --benchmark_out_format=json"
 
-# ros2 daemon stop
-# ros2 launch moveit_middleware_benchmark scenario_perception_pipeline_benchmark.launch.py benchmark_command_args:="--benchmark_out=middleware_bechmark_results.json --benchmark_out_format=json"
\ No newline at end of file
+ros2 daemon stop
+export RMW_IMPLEMENTATION=${middleware_name}
+mkdir ${benchmark_results_directory}/scenario_perception_pipeline -p
+ros2 launch moveit_middleware_benchmark scenario_perception_pipeline_benchmark.launch.py benchmark_command_args:="--benchmark_out=${benchmark_results_directory}/scenario_perception_pipeline/${middleware_name}.json --benchmark_out_format=json" selected_test_case_index:=1

From b9d154f3776863d38a5bb4104b2cab92d7994631 Mon Sep 17 00:00:00 2001
From: CihatAltiparmak <cihataltiparmak1@gmail.com>
Date: Sun, 28 Jul 2024 02:31:58 +0300
Subject: [PATCH 03/11] Added basic service client benchmark to
 run_all_benchmarks.sh

---
 scripts/run_all_benchmarks.sh | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/scripts/run_all_benchmarks.sh b/scripts/run_all_benchmarks.sh
index 4faff3c..bd2f6df 100644
--- a/scripts/run_all_benchmarks.sh
+++ b/scripts/run_all_benchmarks.sh
@@ -34,10 +34,12 @@ echo "Starting initial scripts before benchmarks run!"
 sh "$initial_script"
 echo "Initial script has finished! Now starting to benchmark middleware with scenarios!"
 
-# ros2 daemon stop
-# ros2 launch moveit_middleware_benchmark scenario_basic_service_client_benchmark.launch.py benchmark_command_args:="--benchmark_out=middleware_bechmark_results.json --benchmark_out_format=json"
+export RMW_IMPLEMENTATION=${middleware_name}
 
+mkdir ${benchmark_results_directory}/scenario_basic_service_client -p
 ros2 daemon stop
-export RMW_IMPLEMENTATION=${middleware_name}
+ros2 launch moveit_middleware_benchmark scenario_basic_service_client_benchmark.launch.py benchmark_command_args:="--benchmark_out=${benchmark_results_directory}/scenario_basic_service_client/${middleware_name}.json --benchmark_out_format=json"
+
 mkdir ${benchmark_results_directory}/scenario_perception_pipeline -p
-ros2 launch moveit_middleware_benchmark scenario_perception_pipeline_benchmark.launch.py benchmark_command_args:="--benchmark_out=${benchmark_results_directory}/scenario_perception_pipeline/${middleware_name}.json --benchmark_out_format=json" selected_test_case_index:=1
+ros2 daemon stop
+ros2 launch moveit_middleware_benchmark scenario_perception_pipeline_benchmark.launch.py benchmark_command_args:="--benchmark_out=${benchmark_results_directory}/scenario_perception_pipeline/${middleware_name}.json --benchmark_out_format=json"

From 28782e28f4b952b71e5c9c51e2fa77cb0736d655 Mon Sep 17 00:00:00 2001
From: CihatAltiparmak <cihataltiparmak1@gmail.com>
Date: Wed, 31 Jul 2024 22:53:33 +0300
Subject: [PATCH 04/11] Added documentations for scripts and made the configs
 for rmw_zenoh enabled

---
 docs/how_to_run.md                            |  86 ++++++++++++++++++
 docs/pictures/box_plot_example.png            | Bin 0 -> 13780 bytes
 .../rmw_zenoh/config_rmw_zenoh.sh             |   6 +-
 3 files changed, 89 insertions(+), 3 deletions(-)
 create mode 100644 docs/pictures/box_plot_example.png

diff --git a/docs/how_to_run.md b/docs/how_to_run.md
index 2e894d6..8d65d0e 100644
--- a/docs/how_to_run.md
+++ b/docs/how_to_run.md
@@ -1,3 +1,89 @@
+## Run All Benchmarks
+If you haven't taken a look at [How To Install](how_to_install.md) section, please firstly start with [How To Install](how_to_install.md) Because it's assumed that you applied [How To Install](how_to_install.md) section in this section.
+
+To run all benchmarks, just select your middleware implementation and go ahaed with `run_all_benchmarks.sh` bash script. This command initially will run the script you chose. In this example, it's used the default config scripts which the necessary middleware configuration is applied. This property serves the functionality for the users to apply some custom configurations freely.
+
+```shell
+# go to workspace this repository is built
+cd ws
+source /opt/ros/rolling/setup.sh
+source install/setup.sh
+# select your middleware
+export RMW_IMPLEMENTATION=rmw_cyclonedds_cpp
+sh src/moveit_middleware_benchmark/scripts/run_all_benchmarks.sh -i ./src/moveit_middleware_benchmark/middleware_configurations/rmw_cyclonedds/config_rmw_cyclonedds.sh -d /benchmark_results -m rmw_cyclonedds_cpp
+```
+
+Let's explain all operations at `run_all_benchmarks.sh`.
+
+#### selection of initial script
+```
+-i ./src/moveit_middleware_benchmark/middleware_configurations/rmw_cyclonedds/config_rmw_cyclonedds.sh
+```
+
+This argument is for selecting the initial scripts to be run. In this repository, These initial scripts is used for configuring middleware so that middleware is used more effectively. For example, you can use the initial script to configure TCP settings for rmw_zenoh like below.
+
+```shell
+echo "The configurations for rmw_zenoh_cpp is started!"
+sudo sysctl -w "net.ipv4.tcp_rmem=4096 4096 4096"
+sudo sysctl -w "net.ipv4.tcp_wmem=4096 4096 4096"
+sudo sysctl -w "net.ipv4.tcp_mem=4096 4096 4096"
+echo "The configurations for rmw_zenoh_cpp is finished!"
+```
+
+#### directory selection to save benchmark results
+```
+-d /benchmark_results
+```
+
+This argument indicates where the benchmark results are saved. For scenario_perception_benchmark and scenario_basic_service_client, the results of these scenarios are written in the shape of the below directory tree. It should be added that `run_all_benchmarks.sh` script uses json format to save the benchmark results.
+
+```
+benchmark_results/
+├── scenario_basic_service_client
+│   ├── rmw_cyclonedds_cpp.json
+│   ├── rmw_fastrtps_cpp.json
+│   └── rmw_zenoh_cpp.json
+└── scenario_perception_pipeline
+    ├── rmw_cyclonedds_cpp.json
+    ├── rmw_fastrtps_cpp.json
+    └── rmw_zenoh_cpp.json
+```
+
+#### middleware selection
+```
+-m rmw_cyclonedds_cpp
+```
+
+This argument is important for both saving the benchmark results correctly and run `export RMW_IMPLEMENTATION=...` command correctly.
+
+## Plot Visualization of Benchmark Results
+
+After running `run_all_benchmarks.sh`, you can also visualize the box plots of benchmark results. Suppose that you have some benchmark results stored in below directory and the directory named `benchmark_results` is located in `ws` directory which this repository is built.
+
+
+```
+benchmark_results/
+├── scenario_basic_service_client
+│   ├── rmw_cyclonedds_cpp.json
+│   ├── rmw_fastrtps_cpp.json
+│   └── rmw_zenoh_cpp.json
+└── scenario_perception_pipeline
+    ├── rmw_cyclonedds_cpp.json
+    ├── rmw_fastrtps_cpp.json
+    └── rmw_zenoh_cpp.json
+```
+
+Just give the directory of benchmark results as argument and then visualize the benchmark results in plot.
+```shell
+cd ws
+python3 src/moveit_middleware_benchmark/scripts/box_plot_visualizer.py benchmark_results
+
+```
+
+**NOTE THAT THE BELOW PICTURE DOESN'T PRESENT REAL RESULTS. IT'S JUST FOR SHOWCASE**
+
+![](./pictures/box_plot_example.png)
+
 ## Scenarios
 
 ### [Perception Pipeline Benchmark](scenarios/perception_pipeline_benchmark.md)
diff --git a/docs/pictures/box_plot_example.png b/docs/pictures/box_plot_example.png
new file mode 100644
index 0000000000000000000000000000000000000000..eaa3808761dad9838a1bcf19356c4a21a5970dae
GIT binary patch
literal 13780
zcmd^mXH*sGwrwHifNdj<pkhKmP?RJY%rqcDGDuL7oRlaTBds9lMnI6H<gAh;ZcLz{
zAZ&sX1SNySO^$Dtr^k8soZEMdcgDNt{<%LKD!Zz7)mQ7AYlg*bRb{yiYuVQl1hGN>
z_Y-OavCM}c7}{5_#7{(OS~~HUxZ_D3M|C?>N0;;VCWO*?$4l0Bj@A|zb~>BbJ6PD+
z3h|5cAKJgu+|lupgT%puHh({W-_G9bpz!kFb8(S1mwwlEAc&3U$v+H9vWXVB{!#f8
zM>VcQ47R%J2b(M|jZSUqBA)Z?&fp2(+D);_zZR6-rPJUq@*7)Qh@H~I@cO)GmrvQ;
zx6ZS-wp=@AW$LHcCQvTOtVC1uZ}~;x<U3&p#?8lW4ou0(Ry&?L`fF#1Lvd7KxofXy
zTccBJv_t2%uZC0h`ej@8;C^&sLuf1TcZ}O6f`K6RA19U*#MAW*zYs*J;PNB*r1F>l
zEgz)qPSwtA_+!(ax@-&ee0}}{hDFob6H9%G(GTw3i>Zo~Om!Y?Vi&jXWV4wZY)*V+
zSE4)El%t-mmm_@s)eYX{?F8}pCZTgOV25l*hC!iaLz-?+JvA=w#MP@;*+p$5R&M0*
zXgK`#?OV&wPakusM5W;%tpt67IQOhcnYEoSqh!|7z;mMB#P%RRe?#=ryjk(WWV3|Z
zc#?!ue;k{TDGxutGDRcL*5G55^jS)_^Pt{#X}58IfhV!CA`acB<!iUv6`kp-#(VsH
zm95lL(D{c>`q?~XdQWv@_s2&fmUTQEwjMs+=0U$pi^oBDv#Cdwl>;b2K@(ryhb>Bd
z7z#QmisS9qH>P+mEh;E0I~Pdd?%0?Zw67mHo1273!2{FvjP&)>R&HPy(_lfZurNGj
z-zMpN+RMvJK~XU%Iy$;Bx4r$GZ^|-)$lXn7UAJv5NXD6~n6`cLVPK5lUxlg*AeU*o
zWP%-ap&_fteabdIF_BM7N*|Y}9ms2WB4AwB`^79;GtahlydzN7^6ed#{@QrITN`)q
z%QuyH7Ai-GJ$~)GF*PRUNMBzcC9aac@Pp7$pHq3jc3Pn9V$%4y<ILP#s&RQh{e%Vs
zab!b}VA+;LzmXNm$;s@(=AYOG><R}D$H&F>cD@gCn;$I~F{|1(*jlK)Y15{H?|1lw
zZJLCghFWxMtMgpHWSLd78PAVJzw5G(d{%5y8B$SIl~KUWNDv<Ef7>W6D}fXLxL;mt
zmIkZp>7@))mkf99+GSQOhU4`;`~`S<cvNW#N}8!5kq+mVmnuvz&JDA$uyi@H5k$LG
zXLomsE4^De^xNuX=l0&XapU*dnN>W(!gb#)3F64f&Ym7VUfv42d&?=DD(3lFR9N@{
zu^JC(Oz%7E%rTAoc)I1kWA~~z_u&P;xVLuQx*RN{ee-4(mVJ_vXKQL|)~;W_KhMUO
zY5V5Qn>B*>E~^i~9gJ8=I{#1&<d9CWF3Qdpb{hU#Tv5gtaRhiWoz1jeg6F`2^p?YH
zY;1P|0~=fN9axx|9jZos!mGHKol{ez{Kl`Jv|ZAf=g=Wl&$-EZe>S1I63;pNx#E#m
z>%yhobM5WzH*ebXxNT{1N<l$E?Z}ZMBAQIZi%-|CUE`H9W@To6{evT|otgD&nqH2=
z?c2BYs9wb(2aPg<xK%9Q-P<MMHGd_{6PqAWcA-xtT+rl>prGLR>_Fx~q|4WuJ%Lhc
zcZ2bMG-RV6GU(@8hc((ODBRh-XHRfQNWxXdRZ%(BJ<;A7UJJ8*-H+@-I8VySS&jGB
z%zpPf$bY36AgMfdo2_)|L1-wq{MwBh4`wN~m3Wfp9;D8zQL?QX`41QrXf*0OD1~|6
z@$(Cgjvk)CSxFWi7gmmhV%vzEf3<vKpiw1K!ihJl^`mH8+VxdGH=$rzZ(Xu_K~I!>
zh4k`>2M^*bXbA>?Jjd=$xm*2_;Ns%49B4=nm-Y6_A5>FU7xA2RTo|<TMu9!bzVz*w
z=kzyUw;#{SD}D%B)N*!pb-8ln=jUr0$e>(T^r#E@kUNC3S25dd!cxTUYf($C_e`ac
z<3M^|f0g9$15R=eln3r;^j@0s9;o(S%(ASP5ZYSUYMb9Wv`$SDCFwMJn=tpU@tkT+
zJn8q_J`oX3;Gu+B^s85|gpNf44a@)`gi#kW!yy5IGxY9{v{gGiV(#7hWw4EEMA`h?
zZ{0Hr(b(=ux>=@pvlPy|TLg|(OHV}<I1MCJ9;E8GdCjFSj)hXq-~6#DYUgTf5<idK
zL=}mEn^{?Va%9BCb=Ry}Ly3!s*p|sgeILYK*=27r*7?4v&?S@ouyxX=J*Rqab4Mj*
zkgK-so8Y!#DZ81O85Fj<p<&dq<HsWbNtdDy+cd>-NV|<lQ$yOP@-`Z-SWgf>WB=Z+
zm=noQyx4+8umo^*<?iF<RdaUE1)Q^N*`j0HOl}(<o+Mty<wQ&c+OYgzZ80K-7dN>e
zWgHq3qGkKY-+%qv@^Y)8mi#-nZ;SU_0qCTy_VI~}iHTWT{rT;iH-`#W5lJ;KR-@1V
zBVzr#TK~_7{DckcIMV)7Au#Cv{cjUyCMQpvNJ7DA3jRhA)Rhds$Q<D2mJ<*V;N8DJ
z?YTflM@Q<#h?tn2CKoUMvW$TsAtu^+r2Qxh)@n#-BVRWqN#U-hhR3llZ3{8I-FG+l
zPG;8Wut_?wEVZxSE_LM|ASuat`4OF6-4q@tYPL;$d0*o~EpHM4((Ljkl#!3gY9CZ)
z$Swjnr*jL(qth+h`TzVu+24T6ZK6LO)qL{QDa+1x_bWbpkTRpj$_4VG*p<SC?>7%>
z6*>=QJNDIDCT+ori(@rnc>e^c{`%BV<w%KmIS$to$8W8#0jnT8Ue=`e<?lB#3i^3*
z)&7O4Htj6akJ)C`9QEKQZhq*WNv1NJh|mLPyK^Ealqc`r{T?45Z_!?Q)ohSnw$-k8
z<lF1nd3v>GqVjfmHAlzsoBU|hrI%$cub!3#2FO!hy?$M3O(!>XKP@rvS7zq=q%(V*
z`s?Iri7MLi^72l>Yp~B&0}SWmYire%9v(WnFgu~1s2o{god-1Yob5|2=n6TD=Th^W
z8_X-1`s(g9)+w(PCfJomBAlka{tw-%>gvp~S!_Cf;plZf0Oh6vr&N$4b=yK*lzg^&
zZ6jb|wA*;^<6`%zelRV5A%bh~UW=w|3)Uyrco&u(>b^eM9I1O1L(ID>f-S!mx#rtE
z61I2(F802<x;l%Wi|f?f)Ntc%X6~vcv)VZMipom5l)ZyPYPJ_H(BIC?@M%lzGvjj0
znkOe`$7w3{DVnJclYNbu(IpF$XMxgg4JIZg$?`#5V^dSIRy*>V(?5Mu&BQVvQR?^`
zY&rB))fc(aH~;$UlZCk{tEI&SUS8g@Y)~?PE~W5J-NlX_OG)4(R`b)N6%`e=zB{~<
z`e=!K0s=L+6H#QUHXh%%m4ZkpQoeuxUev(N&CR!K*HOxYprBa2oW)3&*e6fMYvgwH
zfeUDd@*C8p=?dN5A#1o%cK(u6@P?o7Vzp1%Lx)^>B_(NNgJ>DgY&4^Fs+;g$-&S%n
zF`#+HeE<I|=zo0+{7;;bkPCgns@HKYblJKo1>c1T{<nLcUdm)&RT#m~NSxcZY^TT9
zR_S?`AQ8n1`vQu$Urn_W1JS;m{COKqxi?`cD(Ru3o|wTMxw5Kl3kzSkoziJH2_o$T
zaihA~`$G~>QrPCNO?_%ocP4|)mjpUU9;WywLOf(xPSoj$i;GKK895hU{{G#&k=d;0
zyLa!dd-C?JL(`g(Xe_x8`#uu7wl}3d?M>v}DkLLQqENPG?RwR+fYZlz^6NoG9k~na
z19mOM<v)vLQ9t-X&6p*U_srz8-6vPA;(hAeWy=<PdHmsbZJU~x*H^9ad0|fW9TC0L
z#pj#?SErAQEoe~JrZ8Bp<Odf1lROjiVroaz%kf_F0yw6ee|-xK#HD}}Wf$H3ZvDBV
zN3WVzN2P!MtaJ0$EnW!;J%B7JK=g90lM9Cm6t^C>zVt+xJX(8@VI;pNO5M>h8<Ic-
zj|>H*1Aa}GG}^JSNyTOm6a^Bho}Pa_4w|CjavNvLq&ID3u>1V<D5;;~Lk-;;2C<zR
z2J_l7v<+QPzJLFI_D5EAU9L^DWyY_lSQkxq!UM3uS64OYJP_qRlwVqp%Y5#6Qm|G`
zR@TUm^<c*H;R|Gk3fs29NA&BJSVONF6M3~WXYRXA{zxWejzNKA&9xQlc%_uV9{&>v
zr(If_q4$tP0H92r+=r{zi3O}I1%g^Z=1}6$dfsn|1yJ|DWi_gU#4!AJSu0&H{>_^c
z=|)t;%Fu(2SX7aV?|#`X>op(YnU$3_0bS(Qk1LHh1$<p~=-|O>hc+tIK%WbBE)qOc
z*kk&V^!TUaEF5NEo|_m}?%TKTj=#T#o?iG$rfoW=R0`$SR{P2W;fHO)W#)bek<zUw
z>L<#4?_GaRL*qe1`q{*e@<1z`%`SNH56-@K_xG$_y;^MeGJR}}voAt4*b5-R!p4>?
zIrJj-SjTUytbDSvZ38F?RhpOtqZmje`3-BCn6xTG4zRGYa`*Kde;q05B9keg_|GIW
zT*Qhbz<kou#jlLH|Ax6enye(Cys9Ct3>QzOxD0=-`1rA2Q`;b3FUKqDo^@-%M5e^5
zkx+Du9H#-Tblt3k4~JK*SWz(1aCV}}Ds$iN-A`T_tAq>7Gjl7C@9+7kR{D$j@g^_b
z>1bQgl4Yhc@#724sueA&+A57i<@c1mdc|%z;i7hiriHjyUWkl6r~F6K_zoNh=B&>!
zI2#rgM(Wb4-X7<|F6YAP)Wg7aznifPPa!L@%6iW(++^A&*1qO^BH5T&*vq%X&J7i+
z`2BW3&C)VOFUL~Tb`D(19E)N&Knm|`*Oq;Z5D(Io2C~vnVOVu-5PrzZ4D{po(0Pqw
z=vTO>D4FSY#+x^8m;>`EaZjHfTCsAa0;SBKE!Om7IKEq+fq}s+Vuwzqp|m{jRMAnC
zhf%vZ5h3ec^3`+7z`)_7NHDwE<c}|u&ip?*vpYL4?&akCodPx_)w2b}<MW3aqIY;W
z6YNvNZJY52@#eUMgpfk%y?f(poye+P9-w?%dZd&wT-?F*=CW=0QOd1YAs?%Tw3>=w
zzKKll1x0MQ<o>$kjvUW#uWmAFg5k6m;-Mz`>uD2>M#C4;%ZY}|!UPvLw`9IM{=;`!
zcR!ZUBt@EFf*~-_m?>*kNUq64f#aYWDYsH^(2p~NzCLny+2h+5hFn0FX{0fN_a}lo
z#Y0Qjcm9!U*Onn-)nMLN8_zD|@gmA=wy&R`Y)OgO;6)t)lACwz$OJV|87`s+7<x`<
z{v;pz{Gk%gKaw6F_qBX|YXewn^3WK*&m(FN>(v*nwqUP(GBSn~h^fRUPj-u#e`@zX
z`w|DlSc<!`j98pMe;#+^P_UkSkV`Cw$Ork0##heRUK1-f5$HXCJcvuF_S<XUz4DI^
z+thw7cHf?Xb4x|ZtC6+{aPH>Ko20gNX>|$(HOhf->gwz?uZj>?4#5U(584*V5FvK?
z$*;fuN`fBG&Yh)Mi{VxoH6U=?)5wdth_e%-)}LbvhfC)CCq9OY&=wcwwkLs8mQLZL
zR@=NWI=xyaIygA~)`qPTWZ(7~b`T~E-3N2ru&GJ%BrogDdI@$6hu8FZEIHNT`A<~t
zGABGHznD4w_$&a*l#-P6`?F`yENKaMj~zQU{)SmaOqU%OtF|BKZAW<?Fe=d}YXc8P
zo&gzd043@Cd_Ptzy4#$PN{6%4-$td<Oz;z(m!rgs^RHK?>*uS%pHa22NN&u9)~tF=
zR)9xnEW@SQ@xE{=*XK=ncA2N{vSVww<}Qp4<I^Mw;AxnUS^r(pud2t6T?_ho!M8Q3
zK34T9@n9$7x~Oj-Csg88n>Q8PrR&d19RFqH`bL=~ypjQmBL5k%JCdnp4z6S6cT5rl
z>-r@2Ili=XcXB{1oy~9=qhFoaL;B~-m$6;}TSb-mv@_~rj$X$fM*ew%nCEqw8qq#I
zX8_M+Z^C`YDH-p?N}_^XHu^`=oO3UiZP+2x@?H==Yu!&mz$I`eS-TMpUoe29v5i*J
zkfY4H?7{0AVhaaHh9ZCdoA>BZBA*wHPn|q@Rju`0qT(nIFK_*#aao_BkG;5C?Hed^
z5%fZ|TM1q4HXlvYhsWq!)*Oc(Ww>m=!y_@%U;f;J#^uWkVm4xcmAb!2PU3Hy5A~3E
zI_!Jr5O6!qakNE@+@JMNn#_pTuU`kFLjL$;b$zB0wZ(mDJ9ZU2A1?9qc+RR)SP-O@
z6h(EQo8epgHPNqgn{Day(z|v&E^yfh0V6F}%`i@<)8JWD>3H{k+6AF2{`YEQcbJrn
zXJuurY>OT~15AGL%eMRtAN&rM@W|We>(7?UE<SE=_x8T2A}@biE7LF?m??JVXLUJn
z5;9Bkbc$CLgGX1eb`jIm?A#n5$V6t**d1olOq`aITh>0*TDVvK`pONpuWqi>q<VRB
zSK!f}QexYdKez8v;Qo1wdrQ`kjr}jo<<mJv%b&yiTAMdm8(+C1jXX@bo`X18_(L#X
zKSPkZqI3LkXk|QwLb1$XBDWS`M#^(zY03i_BRAKvk}4YJlm^Tf5#zG;Id*L&A7nC%
zC&RlxgvG)MRiKbc1H?QPolVnl2QKk8u8Hv+`a-J52hCKieNb)1ZWFc1>WSm;_5_;4
zIU?Q0{)Tk-1{u;ily=LWog7T{@gYd8B#9764cK}XZzi^M`-N}4cVkqVaSADSzrR_N
zInik7ZgX??;Mv>{R^og2p1gnmKI|S^=~c$+rk?b?uVZ?P<g%|=_nSF9PIe*F51C0E
zz&cal^-xoeVxy6Fajx~3?@op>m0grLx(rD?R4AJ{I1InZ0JbDt$1djoN5v(ZIg2%q
z=<KyQR~EP<6P%_GU$%Db+FINxY4f1}s^d|5L`<>UxlTWf!B)ldDzgktjlABN3V(@R
z>dK`s8>}+v&bXd9c`{P_F0QF;FHB^>P7airO#%hhNYy$3DWwQOqzwvIc6Y0bq49t4
zqi)Vg&2$FAnJW41fL@%BkB?w81hKrUt80{;t?iI&J!$1|%57u_eei&Xlk)_H^!G@&
zbg(&38*d~yENmvVe5<5$TF8FgCm?RAC=Hf=`Uj!v-uK+mb==V&_EmFC1wXwgBZHsB
z<xLIoIa!_Cb77*<vOXo4pE_?ZJyE9)UDueJS&{;RrF~_avHwuvu*?Cy?BB5rI85{A
zEP0UI_uJg021_=pj_R2sq1SS=9Ai8DVW;`&%ks6cfifv=i<5cmhtB!HLXL6y>YDhf
zgA)cEB<U=;;h*YIkpk;$^Oh|<@_3v6U|9)qah&qCa5Tposf(I$z(|6fsB-Dj(`(nS
zr;b?wCG|CclE;0SCgzKk9vyCaX6(zfLndu@V!%CPdBz|Am2R|jogj?@ucf&*_lYog
znhDC0vjb6B*CQM3&$O%>X||)*BeoE^)PCfquW#s#$Yqn{-{+Q!Qp6-B=d=fbE2Y|E
z=z?g)j@Z>H28GTA-QiZ*c5O29wcA}<9<SoKQu@9YOW_d`%P>NUQ}BOMw8ELGo_%Cn
z(C2SAg^!QaUjnF?@9<7TarFA1&CRxH)^eZxaXE<>pSE@Pu5TahC@(f$RF?FZJ~Y*g
zZ+>wEkIcAo{ig_V2dPT_P(@f6ZvDxLFz0u!A|Vf`$+64}KOt|;vZL&-1fF%V$9qw(
zIoF1U=8=Q^gIke2Qa7NAaE5!KtE)@4wv56d39xp3(UwcsOJ@7|4D8?URSqF(QP#Fc
z3{tLRap=Jbq%!edq(?`R5M3$}@R4XpR+oDuYUgZ8K2n4I__osN=_|#Hvjc8p@3|xQ
zGZL!TO}0sE^$FZj4(6HDx&P!kBa2RR^{AWAx{qMY{dN0_N6WU(b_S``#ma50Pe0pe
zHRn(*-J@y(x@vIZ#Eomq7&lWb{b=zKyLb*ALDZle%JATQ4e<h8n(Wk+2f@KzIUl?1
z#n1do#N0Bkd2&<G`C@<j%j>}$e0%mB9~~VHmvqUvaNz>4v>~pimu-Hg@3POjx46Wg
z_?q;M{9*i2J}#~@ApwSIK{-1{JfBa$9054}UlZ)X9t)yS>Qc22g1hqw3aagq3(zn%
z?G~Ra1|RMJ^!WO+Wy|XF9n9d%XS1gu0lgHWvb^-q763xrIM!83lClkKf+t{rAI$n0
z0Dw^r1V|2j(l+vpM?)taKX_VJB3kH1stcBq21hh?{xUuI5GEvI{rM0n4=Hiw9PXz{
zfeCjz4OQ56tq~NmeKYB{sQ?}+aU&xmg3W1wT2Qx1JyFMARWz@mYYk)x3;(q#2lWr;
z@y^O1_SD^MP&~monB%E0UVP4qh>kXzxrPp%684XQL2zIs+9o(-^&Vc{l!_@Fa_RAm
z27xf|C*%h6X_HY@SFTOVi)UWb-&SQh$<B{lN0O;;8V$E>9~3bC-N%pVlmhbVK`P6S
z82gvOs)M%fqp8SZG1IHQB&tNqkOHl-psO-875_1ryj<A2cEg4VH|(-HVVTQJ=Pq1$
zLPj$iWu}ji{DwaLmk2sN+QG7QYpPkarxvWX#`zg~&gG6%#k2jX0n*?;q|+zlgRD*q
zu0CA0x;S~}>JG#^quDSZYLF&LRg%#Xe7y1g)XWsC#tgTmg{hn?qu=chZo?CdtdJ34
zH1`)bdsJ6fC)9A4T|@zkSwOJhGyNhYHORyXmVehiB3<$%CK%X!(x=Xht}*eRy8h&m
zs9H>O+b9-F%xL+M>1+@~i0yrgFWHns?Qa2tVWNu}tRza0k)NW?@&F`OZRAKB*ZGH_
zO}%gQr)PE2Pa%+zD9t<x>&psmXz<GPhYz>>8PyRn1^+F+`2U0{_^*7&Xc+MVw1<q9
z&|8BmyK29`^{f6uk}tK!3<H+Uo8#c&_D(k2HKGsldpyHiW*Q44uAu8X4K`_#f>?Df
z$Ewlt!$W}rh+6cv!z7B#j#VCXYBX>$XjedmL}(%~O{svRoeru|@M^FTZccBn`CTX$
zN?nR3Un?TAa0Fd95!VpyPuhiqe5hJ5avwZ;lp3!XTCpW|hu8E`5TKHb{O<6d`}hC4
zJ33p5xru?qysvJGqOFcW^9u|0FD@H{{{_3R7KaM~Q5+?wp`jt71t?J!N!#GloM+dF
z9Wz044hstlxP4*Iv$C9!tUdVh4=(w$3)nj;--LRIMLk&m{qV&(+<JG(tHFmRXAL%N
zIdlY(`ipR33;R<r#}DK+8??rQv}o+JG9%WlTZbfjiZ(^32XV3)NAN(*#OO_*CZwEy
zJglYBf}^64*cZF!k^c?~+SAwbI?uSwe>8iCz=a!R=nHa#ooJv}=$sA;nFx7KQcz_n
zK(deJHz8jiQQMZJqt{ouhHxuK@W>Yo<~Bb-=7F8;`YMJKW=axf01&f2=eW|&0G6mi
zo%v|@Vgr_8#8rlw==lD<LfY{Wg<@Y;Kzc$`tuAdV($?Y)eDA=;@#0hkmLI_@|En<<
z83&d_s%%(gFfzG(8$`xMi~Cjn48Hf~FeGjJ+EyB-e=D<C>^~)n!A~LoCVFdPE6QNK
zsZR~Hrdmr8J%LlAi_BDN7{hYTg?ZW8?Ck7+O4g+i+3_<KQLhRKFzT$C-?)tM3HNPl
z(_fAiKev6^PAtn`*&YA#2VodFQ4L}<=9Gs-WYWgfdHctQ2WixDn-fIZ4Y)z_YVagq
z2jdHk$T<s?7&(;?`b>OLOAXSYWNFbI%@sot;s#9oPv_@7PM$ouPgGR<@#DwJp^PzW
zM;CMb1Wp04ESs{yv!Bzg#Y_pl$513-%=t8vm~a@<W9ycUoGGFXo*($oX5n0Bh&SQ)
z_so^|-Jr8o#e;qJg{{h=%VO?~F5a^FGr0Uf;Ew+}wR@@sW^H)o+ls_WRTzITngTea
z^-Lc<c(56jk=bOqj9&}hvKD+G3{oVnVdfGuU{W>y>9v=kFw)8K0@Cp`4g}2`h20};
z-}!EBPM7S3k-Z+wt1C!Jhq)3-gOxshH)zpopf^dWEsPEgTP#~|$`iS>W$TVhj|CW4
z2flbAc6Ym^7BkmrRgkq7+dSM5=<yz>fd(av1kHZ5^NM9$#nfxGG#d&|x0#(ig`6ou
zy;0&2cO@AXyYcSZcb^lpF>tc`RJ><{%w$Q~@x7yiPlMK!V+D&h5p4i&ff3Y0Jt&DV
z=2&2v9l?~5;h<c=w&b&R#r0Y9i0;)PY%w?CgHGTGeH5CXiNM?dmKSLko0})u7LA@-
zfC7vfgcz(h2{lMqT4-C+#EPODkNe8bAH}PTIt-dl4q%|SL*pc2f{6?$r<$s3;}zl6
zO2Qts{1kH(6EZTfAz1eu^O|Evj~*=mi*|w?kvVAOy^w?1p*Uowc>O7IfgDXhvUo%F
zzBXA#Oe1LO=sd(Nj)zj`Pg0A{YT)}h+<yq%#f!wck!UOoe{1dFR*}lL<6-y0!&7i2
z`C2<K`c{l=#U~_y+#M%%I;p&{Hsk~Yri^m6+u%YO!C%%ocP@t1ZwOij0FdDlPAOkK
z=t`a=FIT|?i4uU*-({aaj`0Vc!-uutbh=?0V%%tH!iWqOvmt;|;vA^l{sQEgAlti;
z7h>ANvgO5PmRV7ZXf?dvD4Uj)#FO7@n`1#cl}TMJ=n=ULe+7$MAFs%|V)^pkJA4_R
zphl$^RI2s(5wh4c=W62J++y0E($dnhc5sKx2V%*{+B$6w6Vu_1(&P8OKohz`2>Ek5
zyk*gT2-o!?x>*=(<A+FggpgBY0dKO<=xFf8=+GeS#{S|M9ZLp0e|dWTx<J`OR>LbV
zPn<aM5WKS`E1EnN=EPvcgvUg^cG;lwBiY5#KrU(NR@nUYMdP)K-4Jxbs79{+`#%%`
zJfkK_{+Db}m^*_Q^{znKe%<G9re|itsEZSsrgM{n<jJR_Wm;(^h()?*dZI<kwy3MA
zUEqg-L!)|oXMwL}c`dlnAdi^0ZXK|Yg^R|#ZJV|2i*GBlOe%l-dH*N}I?R_VO;U@~
z9SNvwQv@QWJynrh;^JRm5C?^aTlx#Y%r3`&gf4$Sk!4)YB;+!rPeX{k=lwQu`!`#R
zKR`SyVG`v7boXgf(}XZ_hlFJ9%&0Z%*3p0T)G8hnftcdjv*-0i6O%BPHqXYm#6<0W
z5s_|GA<}@UI@ZmbFOrsZX5#@)&NrNU_lAA=aGH*RA9h(;S^8-Cj%bIaBwhZ5>Kos`
zUkC~dv)~MaiSrPBDGWcL15LsbqrIBQwI%O8D!Tk&$@DAS7|PJjvN6L3v6pxjCf~AL
z>83(f4g1wqRnNhu`2aiB9QxE0!vH;O4(J=FD7OOw4zK%d{~=5g2!8qUrTZqU;US=G
zR4#TTLSZU5ruU?o+1V{a_zf(PH5ivne_xk5rQu5MX=WA{6Es?L=b;u0nVC*zA@4;G
zV}Aj-sKS)fr%(IOOm_r+1pWSK=RJ3pdHZ%7%+{U9`eL!+bC!=IuJKr$=`n=@Pk?Qf
z08@=+LBJ<)H}PuE;lmA%0}UUuJZ4{a1*yo4rYR~Y2vR=4bC4bwDw(qt0x(wF;~73e
z57_t%m_jghV;1Mo;^HDUXQ>C;@hQstiV6ucX=HJSF=W8!Ft=lATFwFM<{lF5IjKv7
zz*5)K!z)jKdgu;0Yr8LN=^$e}iqm?2e*QEZbF(>kTmGE%u23U(V=pEj01g*|F<SRy
zUyiZ)-E){LIIXIBZw50F#gh)@+g%<)$Za+D4->Ya^nHF7+c=59{xcteZqcNV#KzDD
z&IVT|vE|oaFQ}=hRe0E%h&NiXZ{2DIn3l@?);CN3U5SxMVI&xmu_Z=}XG=x(t)tff
zaRn$J5X@BCh$0T8mojeLV(c%ks3=4^?d&|M+x*qJ*`~W-AcL;wB0-d3xXo}NC@84-
zt8ZfDfb=L*(s0Z*xE|9z8Oc-xjVL=XNn7GbWPB`8woStE-SU+y!_ffB)w4LseWXH8
z`|NqoOgWDUfY*u`zN^w{1j1CeZ56S^#Lvzg+twzlR&*$-N`wXqN=m^3r?94B<bJ~_
zkU4@1q+`K|BvY`0QCT6Fn=}B-nAlPWG7860<yM&Eqj<^G-M@ElXK1D7<3;L&`}ZFM
z-O%h^8xS#^LCmCn>eMYW10Z$Jp$D8I7PW8aEmVwFz#s2VNzG7$)72wD&R4eN)3TyT
z^s#F5oE;aB4ROc`L##4XL|5LxA*G`tGZSYv?~we?${rYZ#qc$4ZV}^lQobWqT6p!I
zMTXqd0e*c#->lJr#y=YeCSxuGVYfEEfb6TtY<{N{I1i@qWfrLc=^K*H#Ev7CbX9g0
z3yqR?*Lo2!_LUi$j?9tu@@&(|_!}W1Y4=u%o4UCbpq@5w-Fo($r?0PXDz>uWa5=gx
zsDoit7tHX8(Jw3Kej&caW2DtAEmb`+pl6srw9;y3yZ{3&)%N=O5g2cY1>);rnC1*g
z6w#pT>NEgE{SiA%6VV+*3X9L?+4eW(q>`pICM5y=N|;|`8S<8?a32Dr2;wBC_7qf9
zrlb(Cd6l{P1dBNJYmmp0x)-RDg3(!Wnx*H5%PvFa>D-@dg7nP-p{6=!MxJVA=qKY<
zZrCoN0oHtv6BN&4W~^%#Vjv#p_9LTL177F2PYpQ)c;@za$vk=T<Uv?iG8MyvaM0_Z
zZ&Fb<QS?{8e$9MwZ?#X7J-j+q%o0R-FS=H*n(Ny*hvx#z&@_+WHz**@>#p2<dX;1_
z$T?C@ywA^8D>HX5sQBH!ea6-{qw#hPDM_&L>XG`cy<0<4>Q-C4vm`V!!$6u73bGf)
zG2WjCM_R~o23T8RvqzpB2?oW;ga*+e9X*UDRO=d_Gz`ibEX0~1SL4fy!t5j*ukMIB
z-^i4H6i7pcK`JU(86ZzCHTkmB()MQ_hR+C$p=Ddp#-Y^Sx3#s=cW`-oduM(b9DC3>
zsXu(XwDbrm6#>>yFi(jMBw_RAd3?(P$s@ph$S^s7&?|o!h7u1SU!_$WhN8v_e%~)7
zqz*3AC2b|XY-fr-zJzS`#mL2@HcnzC5!fkVj3Q-ZX_8whJPvc%y#Vcc;4Y?XnaFu=
z==SmK{F&}Y6X+d%IgN%i%ufu}dxQVFVOD|+oG=VWX4s^P)SV)8OY)}jTI~JM%j3vV
zQz)x5*4EZHH<FQc@5@y?>d2K7ZSzo<bQ!7xOF>sZ*oO%70cR_m$yw|!I8qz;;pRzS
z8WDJ-7;v8U+@x}x^w~j2Ye<(0?c6n&m4(IP$7d2n>am%~>2n*5ksRC6&El%$U~eys
zPHbXo+K;<#P&`;12gBF>ZWfLr&2@5u1S!clo;k8+rC55340BZi-kx5GM?Qij2RvCX
zT=~fYw<W|;pT_%Wq_;zcu!yv0fkV>6p%BwHUyF1x4M0xg<Y(C<W<!wtP%BO6F^n>|
z49AcPZ<RRAbR+d{1iOM}=l>WfsSU1e2?~*4eE@+~UaRvXTk3){IoeX_GOPuiKnC@h
z&Sb1XUJ6zlFJ`mg&Kgb~PQxB2(*jSVd&q6nk#lCe_`Y@mLD#EHA&+}B?p@!%Z$e{0
z)-tRs<9Thg&yU$(d`|w8dctV|_~{dRq8L5i^Zw$+i&q+8C)5kuwoGf+KLCe8kYoi>
zK(Z1x3CARD>9Q>jpqk+$BbVrCmytEzFR!iWeqPBhgl@^Td-toBORqOX*-(pCgWr%{
zKBIV2PaO)K8Ep#LtjXkXo6!)RY}QJs66s2m>rbkHKt&XIB!Hq52AvWCcOT21>5*L`
z$Lx%83v7rqTs!WHR$Yk%J$IrNxjHa!*<yamf6MtO*KW4z0YiO#>%|dY*@poBj^SZD
zGz?2PeO7nw+~HjaFDokxcONcpY(YtJVN|^?)2K~q?A;zi)|=VZUusW3I{eX}U8G|9
ziWOmy_>YhoYniu!JLOj2$TC=N4_V;Q>eLv9Um9R$W3#|~hBf9ZI`W?y1ENfOKRr&s
zOcTjZ(HX-q>+qhoghZ<wokj5F%a>8lRE(C7VKYabc`ZHFVhn13jx%hZdcqdts)Z<>
zZ1em1^JA4t5ID(a^D^==+h4>Vw@(EfAb)9bP6@LC{!}Uzmcw0g1^fk|0H?{N;V&Ln
zBOldmYRIN!an?#@iMAv|k}%ZAVcWds^#HgZU$PRfWlDGiS3WdqX<wIgHKS-u!N9FA
z-VNmv$sb?l=jTU*lF}N|?j%RXU)@;av4M4DNqi#eckCjB*p47Xg4D;LmnR_;d8Sgi
z&T}Il6={H!lj`|}9OZT>z|e=~`YMJS#^f)EWH!goj|rfCI!q2Xq(34x>Yq_3J8#Lf
z=A_7qh>OkD!=y{vP2a%8gkRy976UsF0M=`sIa8yXchlDwziLqw&X+QCMd?h?mr43r
z4UNwHz`iel#RT@$t2V2BAj3nx{qkSUR{YCm!@n|D`>%L=f1BOyUShD;F29i@kfVV6
PCge{lpU60N?%MwW{5Gve

literal 0
HcmV?d00001

diff --git a/middleware_configurations/rmw_zenoh/config_rmw_zenoh.sh b/middleware_configurations/rmw_zenoh/config_rmw_zenoh.sh
index e104f70..fa0920d 100644
--- a/middleware_configurations/rmw_zenoh/config_rmw_zenoh.sh
+++ b/middleware_configurations/rmw_zenoh/config_rmw_zenoh.sh
@@ -1,5 +1,5 @@
 echo "The configurations for rmw_zenoh_cpp is started!"
-# sudo sysctl -w "net.ipv4.tcp_rmem=4096 4096 4096"
-# sudo sysctl -w "net.ipv4.tcp_wmem=4096 4096 4096"
-# sudo sysctl -w "net.ipv4.tcp_mem=4096 4096 4096"
+sudo sysctl -w "net.ipv4.tcp_rmem=4096 4096 4096"
+sudo sysctl -w "net.ipv4.tcp_wmem=4096 4096 4096"
+sudo sysctl -w "net.ipv4.tcp_mem=4096 4096 4096"
 echo "The configurations for rmw_zenoh_cpp is finished!"

From 67e2664a19167f213e500500bc7bed0a7aa048ef Mon Sep 17 00:00:00 2001
From: CihatAltiparmak <cihataltiparmak1@gmail.com>
Date: Thu, 1 Aug 2024 00:16:19 +0300
Subject: [PATCH 05/11] Fixed typo in documentation

---
 docs/how_to_run.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/how_to_run.md b/docs/how_to_run.md
index 8d65d0e..6928580 100644
--- a/docs/how_to_run.md
+++ b/docs/how_to_run.md
@@ -1,7 +1,7 @@
 ## Run All Benchmarks
 If you haven't taken a look at [How To Install](how_to_install.md) section, please firstly start with [How To Install](how_to_install.md) Because it's assumed that you applied [How To Install](how_to_install.md) section in this section.
 
-To run all benchmarks, just select your middleware implementation and go ahaed with `run_all_benchmarks.sh` bash script. This command initially will run the script you chose. In this example, it's used the default config scripts which the necessary middleware configuration is applied. This property serves the functionality for the users to apply some custom configurations freely.
+To run all benchmarks, just select your middleware implementation and go ahead with `run_all_benchmarks.sh` bash script. This command initially will run the script you chose. In this example, it's used the default config scripts which the necessary middleware configuration is applied. This property serves the functionality for the users to apply some custom configurations freely.
 
 ```shell
 # go to workspace this repository is built

From 6535984e25e3e72e43ef84eb481e94c1a97b8796 Mon Sep 17 00:00:00 2001
From: CihatAltiparmak <cihataltiparmak1@gmail.com>
Date: Sat, 3 Aug 2024 16:15:49 +0300
Subject: [PATCH 06/11] Removed the unnecessary parameter (-m) from
 run_all_benchmarks.sh

---
 docs/how_to_run.md                              | 17 ++++++-----------
 .../rmw_cyclonedds/config.sh                    |  3 +++
 .../rmw_cyclonedds/config_rmw_cyclonedds.sh     |  0
 .../rmw_fastrtps/config.sh                      |  3 +++
 .../rmw_fastrtps/config_rmw_fastrtps.sh         |  0
 .../{config_rmw_zenoh.sh => config.sh}          |  1 +
 scripts/run_all_benchmarks.sh                   | 13 ++++---------
 7 files changed, 17 insertions(+), 20 deletions(-)
 create mode 100644 middleware_configurations/rmw_cyclonedds/config.sh
 delete mode 100644 middleware_configurations/rmw_cyclonedds/config_rmw_cyclonedds.sh
 create mode 100644 middleware_configurations/rmw_fastrtps/config.sh
 delete mode 100644 middleware_configurations/rmw_fastrtps/config_rmw_fastrtps.sh
 rename middleware_configurations/rmw_zenoh/{config_rmw_zenoh.sh => config.sh} (86%)

diff --git a/docs/how_to_run.md b/docs/how_to_run.md
index 6928580..45ac6ce 100644
--- a/docs/how_to_run.md
+++ b/docs/how_to_run.md
@@ -8,22 +8,24 @@ To run all benchmarks, just select your middleware implementation and go ahead w
 cd ws
 source /opt/ros/rolling/setup.sh
 source install/setup.sh
-# select your middleware
-export RMW_IMPLEMENTATION=rmw_cyclonedds_cpp
-sh src/moveit_middleware_benchmark/scripts/run_all_benchmarks.sh -i ./src/moveit_middleware_benchmark/middleware_configurations/rmw_cyclonedds/config_rmw_cyclonedds.sh -d /benchmark_results -m rmw_cyclonedds_cpp
+# go to moveit_middleware_benchmark package's directory
+cd src/moveit_middleware_benchmark
+# conduct all benchmarks
+sh src/moveit_middleware_benchmark/scripts/run_all_benchmarks.sh -i ./src/moveit_middleware_benchmark/middleware_configurations/rmw_cyclonedds/config.sh -d /benchmark_results
 ```
 
 Let's explain all operations at `run_all_benchmarks.sh`.
 
 #### selection of initial script
 ```
--i ./src/moveit_middleware_benchmark/middleware_configurations/rmw_cyclonedds/config_rmw_cyclonedds.sh
+-i ./src/moveit_middleware_benchmark/middleware_configurations/rmw_cyclonedds/config.sh
 ```
 
 This argument is for selecting the initial scripts to be run. In this repository, These initial scripts is used for configuring middleware so that middleware is used more effectively. For example, you can use the initial script to configure TCP settings for rmw_zenoh like below.
 
 ```shell
 echo "The configurations for rmw_zenoh_cpp is started!"
+export RMW_IMPLEMENTATION=rmw_zenoh_cpp
 sudo sysctl -w "net.ipv4.tcp_rmem=4096 4096 4096"
 sudo sysctl -w "net.ipv4.tcp_wmem=4096 4096 4096"
 sudo sysctl -w "net.ipv4.tcp_mem=4096 4096 4096"
@@ -49,13 +51,6 @@ benchmark_results/
     └── rmw_zenoh_cpp.json
 ```
 
-#### middleware selection
-```
--m rmw_cyclonedds_cpp
-```
-
-This argument is important for both saving the benchmark results correctly and run `export RMW_IMPLEMENTATION=...` command correctly.
-
 ## Plot Visualization of Benchmark Results
 
 After running `run_all_benchmarks.sh`, you can also visualize the box plots of benchmark results. Suppose that you have some benchmark results stored in below directory and the directory named `benchmark_results` is located in `ws` directory which this repository is built.
diff --git a/middleware_configurations/rmw_cyclonedds/config.sh b/middleware_configurations/rmw_cyclonedds/config.sh
new file mode 100644
index 0000000..3d70b54
--- /dev/null
+++ b/middleware_configurations/rmw_cyclonedds/config.sh
@@ -0,0 +1,3 @@
+echo "The configurations for rmw_cyclonedds_cpp is started!"
+export RMW_IMPLEMENTATION=rmw_cyclonedds_cpp
+echo "The configurations for rmw_cyclonedds_cpp is finished!"
\ No newline at end of file
diff --git a/middleware_configurations/rmw_cyclonedds/config_rmw_cyclonedds.sh b/middleware_configurations/rmw_cyclonedds/config_rmw_cyclonedds.sh
deleted file mode 100644
index e69de29..0000000
diff --git a/middleware_configurations/rmw_fastrtps/config.sh b/middleware_configurations/rmw_fastrtps/config.sh
new file mode 100644
index 0000000..9dd57af
--- /dev/null
+++ b/middleware_configurations/rmw_fastrtps/config.sh
@@ -0,0 +1,3 @@
+echo "The configurations for rmw_fastrtps_cpp is started!"
+export RMW_IMPLEMENTATION=rmw_fastrtps_cpp
+echo "The configurations for rmw_fastrtps_cpp is finished!"
\ No newline at end of file
diff --git a/middleware_configurations/rmw_fastrtps/config_rmw_fastrtps.sh b/middleware_configurations/rmw_fastrtps/config_rmw_fastrtps.sh
deleted file mode 100644
index e69de29..0000000
diff --git a/middleware_configurations/rmw_zenoh/config_rmw_zenoh.sh b/middleware_configurations/rmw_zenoh/config.sh
similarity index 86%
rename from middleware_configurations/rmw_zenoh/config_rmw_zenoh.sh
rename to middleware_configurations/rmw_zenoh/config.sh
index fa0920d..ad969e5 100644
--- a/middleware_configurations/rmw_zenoh/config_rmw_zenoh.sh
+++ b/middleware_configurations/rmw_zenoh/config.sh
@@ -1,4 +1,5 @@
 echo "The configurations for rmw_zenoh_cpp is started!"
+export RMW_IMPLEMENTATION=rmw_zenoh_cpp
 sudo sysctl -w "net.ipv4.tcp_rmem=4096 4096 4096"
 sudo sysctl -w "net.ipv4.tcp_wmem=4096 4096 4096"
 sudo sysctl -w "net.ipv4.tcp_mem=4096 4096 4096"
diff --git a/scripts/run_all_benchmarks.sh b/scripts/run_all_benchmarks.sh
index bd2f6df..799dac8 100644
--- a/scripts/run_all_benchmarks.sh
+++ b/scripts/run_all_benchmarks.sh
@@ -2,9 +2,8 @@
 helpFunction()
 {
    echo ""
-   echo "Usage: $0 -i initial_script -m middleware_name -d benchmark_results_directory"
+   echo "Usage: $0 -i initial_script -d benchmark_results_directory"
    echo -i "\t-i initial_script to run once all benchmarks are started"
-   echo -m "\t-m selected middleware to benchmark"
    echo -d "\t-d the directory the benchmark results is saved"
    exit 1 # Exit script after printing help
 }
@@ -13,20 +12,18 @@ while getopts "i:m:d:" opt
 do
    case "$opt" in
       i ) initial_script="$OPTARG" ;;
-      m ) middleware_name="$OPTARG" ;;
       d ) benchmark_results_directory="$OPTARG" ;;
       ? ) helpFunction ;; # Print helpFunction in case parameter is non-existent
    esac
 done
 
 # Print helpFunction in case parameters are empty
-if [ -z "$initial_script" ] || [ -z "$middleware_name" ] || [ -z "$benchmark_results_directory" ]
+if [ -z "$initial_script" ] || [ -z "$benchmark_results_directory" ]
 then
    echo "Some or all of the parameters are empty";
    helpFunction
 fi
 
-echo "middleware name is $middleware_name"
 echo "benchmark results directory is $benchmark_results_directory"
 
 echo "Benchmarking is starting!"
@@ -34,12 +31,10 @@ echo "Starting initial scripts before benchmarks run!"
 sh "$initial_script"
 echo "Initial script has finished! Now starting to benchmark middleware with scenarios!"
 
-export RMW_IMPLEMENTATION=${middleware_name}
-
 mkdir ${benchmark_results_directory}/scenario_basic_service_client -p
 ros2 daemon stop
-ros2 launch moveit_middleware_benchmark scenario_basic_service_client_benchmark.launch.py benchmark_command_args:="--benchmark_out=${benchmark_results_directory}/scenario_basic_service_client/${middleware_name}.json --benchmark_out_format=json"
+ros2 launch moveit_middleware_benchmark scenario_basic_service_client_benchmark.launch.py benchmark_command_args:="--benchmark_out=${benchmark_results_directory}/scenario_basic_service_client/${RMW_IMPLEMENTATION}.json --benchmark_out_format=json"
 
 mkdir ${benchmark_results_directory}/scenario_perception_pipeline -p
 ros2 daemon stop
-ros2 launch moveit_middleware_benchmark scenario_perception_pipeline_benchmark.launch.py benchmark_command_args:="--benchmark_out=${benchmark_results_directory}/scenario_perception_pipeline/${middleware_name}.json --benchmark_out_format=json"
+ros2 launch moveit_middleware_benchmark scenario_perception_pipeline_benchmark.launch.py benchmark_command_args:="--benchmark_out=${benchmark_results_directory}/scenario_perception_pipeline/${RMW_IMPLEMENTATION}.json --benchmark_out_format=json" selected_test_case_index:=0

From 17a4aecc9abcb18a5e50a393d1760727653d3267 Mon Sep 17 00:00:00 2001
From: CihatAltiparmak <cihataltiparmak1@gmail.com>
Date: Sat, 3 Aug 2024 19:16:37 +0300
Subject: [PATCH 07/11] Modified benchmark_args in run_all_benchmarks.sh and
 some style fix

---
 middleware_configurations/rmw_cyclonedds/config.sh | 2 +-
 middleware_configurations/rmw_fastrtps/config.sh   | 2 +-
 scripts/run_all_benchmarks.sh                      | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/middleware_configurations/rmw_cyclonedds/config.sh b/middleware_configurations/rmw_cyclonedds/config.sh
index 3d70b54..f9eba3b 100644
--- a/middleware_configurations/rmw_cyclonedds/config.sh
+++ b/middleware_configurations/rmw_cyclonedds/config.sh
@@ -1,3 +1,3 @@
 echo "The configurations for rmw_cyclonedds_cpp is started!"
 export RMW_IMPLEMENTATION=rmw_cyclonedds_cpp
-echo "The configurations for rmw_cyclonedds_cpp is finished!"
\ No newline at end of file
+echo "The configurations for rmw_cyclonedds_cpp is finished!"
diff --git a/middleware_configurations/rmw_fastrtps/config.sh b/middleware_configurations/rmw_fastrtps/config.sh
index 9dd57af..5cfb878 100644
--- a/middleware_configurations/rmw_fastrtps/config.sh
+++ b/middleware_configurations/rmw_fastrtps/config.sh
@@ -1,3 +1,3 @@
 echo "The configurations for rmw_fastrtps_cpp is started!"
 export RMW_IMPLEMENTATION=rmw_fastrtps_cpp
-echo "The configurations for rmw_fastrtps_cpp is finished!"
\ No newline at end of file
+echo "The configurations for rmw_fastrtps_cpp is finished!"
diff --git a/scripts/run_all_benchmarks.sh b/scripts/run_all_benchmarks.sh
index 799dac8..d766760 100644
--- a/scripts/run_all_benchmarks.sh
+++ b/scripts/run_all_benchmarks.sh
@@ -33,8 +33,8 @@ echo "Initial script has finished! Now starting to benchmark middleware with sce
 
 mkdir ${benchmark_results_directory}/scenario_basic_service_client -p
 ros2 daemon stop
-ros2 launch moveit_middleware_benchmark scenario_basic_service_client_benchmark.launch.py benchmark_command_args:="--benchmark_out=${benchmark_results_directory}/scenario_basic_service_client/${RMW_IMPLEMENTATION}.json --benchmark_out_format=json"
+ros2 launch moveit_middleware_benchmark scenario_basic_service_client_benchmark.launch.py benchmark_command_args:="--benchmark_out=${benchmark_results_directory}/scenario_basic_service_client/${RMW_IMPLEMENTATION}.json --benchmark_out_format=json --benchmark_repetitions=6"
 
 mkdir ${benchmark_results_directory}/scenario_perception_pipeline -p
 ros2 daemon stop
-ros2 launch moveit_middleware_benchmark scenario_perception_pipeline_benchmark.launch.py benchmark_command_args:="--benchmark_out=${benchmark_results_directory}/scenario_perception_pipeline/${RMW_IMPLEMENTATION}.json --benchmark_out_format=json" selected_test_case_index:=0
+ros2 launch moveit_middleware_benchmark scenario_perception_pipeline_benchmark.launch.py benchmark_command_args:="--benchmark_out=${benchmark_results_directory}/scenario_perception_pipeline/${RMW_IMPLEMENTATION}.json --benchmark_out_format=json --benchmark_repetitions=6"

From 247dd21250d96c35326fd518b546bc280cbb03cd Mon Sep 17 00:00:00 2001
From: CihatAltiparmak <cihataltiparmak1@gmail.com>
Date: Sat, 3 Aug 2024 22:30:31 +0300
Subject: [PATCH 08/11] Added title section per plot and used dot instead of sh
 for running initial_script

---
 middleware_benchmark_results.json | 184 ++++++++++++++++++++++++++++++
 scripts/box_plot_visualizer.py    |   6 +-
 scripts/run_all_benchmarks.sh     |   2 +-
 3 files changed, 189 insertions(+), 3 deletions(-)
 create mode 100644 middleware_benchmark_results.json

diff --git a/middleware_benchmark_results.json b/middleware_benchmark_results.json
new file mode 100644
index 0000000..13932d7
--- /dev/null
+++ b/middleware_benchmark_results.json
@@ -0,0 +1,184 @@
+{
+  "context": {
+    "date": "2024-08-03T21:20:00+03:00",
+    "host_name": "jarbay51",
+    "executable": "/home/cihat/ws_moveit/install/moveit_middleware_benchmark/lib/moveit_middleware_benchmark/scenario_basic_service_client_benchmark_main",
+    "num_cpus": 8,
+    "mhz_per_cpu": 3500,
+    "cpu_scaling_enabled": true,
+    "caches": [
+      {
+        "type": "Data",
+        "level": 1,
+        "size": 32768,
+        "num_sharing": 2
+      },
+      {
+        "type": "Instruction",
+        "level": 1,
+        "size": 32768,
+        "num_sharing": 2
+      },
+      {
+        "type": "Unified",
+        "level": 2,
+        "size": 262144,
+        "num_sharing": 2
+      },
+      {
+        "type": "Unified",
+        "level": 3,
+        "size": 6291456,
+        "num_sharing": 8
+      }
+    ],
+    "load_avg": [0.214355,0.46875,0.448242],
+    "library_build_type": "release"
+  },
+  "benchmarks": [
+    {
+      "name": "ScenarioBasicServiceClientFixture/test_scenario_basic_service_client",
+      "family_index": 0,
+      "per_family_instance_index": 0,
+      "run_name": "ScenarioBasicServiceClientFixture/test_scenario_basic_service_client",
+      "run_type": "iteration",
+      "repetitions": 6,
+      "repetition_index": 0,
+      "threads": 1,
+      "iterations": 1,
+      "real_time": 4.5481033100004425e+09,
+      "cpu_time": 9.6490377400000000e+08,
+      "time_unit": "ns"
+    },
+    {
+      "name": "ScenarioBasicServiceClientFixture/test_scenario_basic_service_client",
+      "family_index": 0,
+      "per_family_instance_index": 0,
+      "run_name": "ScenarioBasicServiceClientFixture/test_scenario_basic_service_client",
+      "run_type": "iteration",
+      "repetitions": 6,
+      "repetition_index": 1,
+      "threads": 1,
+      "iterations": 1,
+      "real_time": 4.0113112649996767e+09,
+      "cpu_time": 9.7104653500000012e+08,
+      "time_unit": "ns"
+    },
+    {
+      "name": "ScenarioBasicServiceClientFixture/test_scenario_basic_service_client",
+      "family_index": 0,
+      "per_family_instance_index": 0,
+      "run_name": "ScenarioBasicServiceClientFixture/test_scenario_basic_service_client",
+      "run_type": "iteration",
+      "repetitions": 6,
+      "repetition_index": 2,
+      "threads": 1,
+      "iterations": 1,
+      "real_time": 4.1803860189993429e+09,
+      "cpu_time": 9.8092454400000012e+08,
+      "time_unit": "ns"
+    },
+    {
+      "name": "ScenarioBasicServiceClientFixture/test_scenario_basic_service_client",
+      "family_index": 0,
+      "per_family_instance_index": 0,
+      "run_name": "ScenarioBasicServiceClientFixture/test_scenario_basic_service_client",
+      "run_type": "iteration",
+      "repetitions": 6,
+      "repetition_index": 3,
+      "threads": 1,
+      "iterations": 1,
+      "real_time": 4.5593644450000286e+09,
+      "cpu_time": 9.7409754200000048e+08,
+      "time_unit": "ns"
+    },
+    {
+      "name": "ScenarioBasicServiceClientFixture/test_scenario_basic_service_client",
+      "family_index": 0,
+      "per_family_instance_index": 0,
+      "run_name": "ScenarioBasicServiceClientFixture/test_scenario_basic_service_client",
+      "run_type": "iteration",
+      "repetitions": 6,
+      "repetition_index": 4,
+      "threads": 1,
+      "iterations": 1,
+      "real_time": 3.8610252679991388e+09,
+      "cpu_time": 9.7080441899999940e+08,
+      "time_unit": "ns"
+    },
+    {
+      "name": "ScenarioBasicServiceClientFixture/test_scenario_basic_service_client",
+      "family_index": 0,
+      "per_family_instance_index": 0,
+      "run_name": "ScenarioBasicServiceClientFixture/test_scenario_basic_service_client",
+      "run_type": "iteration",
+      "repetitions": 6,
+      "repetition_index": 5,
+      "threads": 1,
+      "iterations": 1,
+      "real_time": 3.9889190239991875e+09,
+      "cpu_time": 9.3607051399999988e+08,
+      "time_unit": "ns"
+    },
+    {
+      "name": "ScenarioBasicServiceClientFixture/test_scenario_basic_service_client_mean",
+      "family_index": 0,
+      "per_family_instance_index": 0,
+      "run_name": "ScenarioBasicServiceClientFixture/test_scenario_basic_service_client",
+      "run_type": "aggregate",
+      "repetitions": 6,
+      "threads": 1,
+      "aggregate_name": "mean",
+      "aggregate_unit": "time",
+      "iterations": 6,
+      "real_time": 4.1915182218329697e+09,
+      "cpu_time": 9.6630788800000000e+08,
+      "time_unit": "ns"
+    },
+    {
+      "name": "ScenarioBasicServiceClientFixture/test_scenario_basic_service_client_median",
+      "family_index": 0,
+      "per_family_instance_index": 0,
+      "run_name": "ScenarioBasicServiceClientFixture/test_scenario_basic_service_client",
+      "run_type": "aggregate",
+      "repetitions": 6,
+      "threads": 1,
+      "aggregate_name": "median",
+      "aggregate_unit": "time",
+      "iterations": 6,
+      "real_time": 4.0958486419995098e+09,
+      "cpu_time": 9.7092547699999964e+08,
+      "time_unit": "ns"
+    },
+    {
+      "name": "ScenarioBasicServiceClientFixture/test_scenario_basic_service_client_stddev",
+      "family_index": 0,
+      "per_family_instance_index": 0,
+      "run_name": "ScenarioBasicServiceClientFixture/test_scenario_basic_service_client",
+      "run_type": "aggregate",
+      "repetitions": 6,
+      "threads": 1,
+      "aggregate_name": "stddev",
+      "aggregate_unit": "time",
+      "iterations": 6,
+      "real_time": 2.9844070423768336e+08,
+      "cpu_time": 1.5705217165175552e+07,
+      "time_unit": "ns"
+    },
+    {
+      "name": "ScenarioBasicServiceClientFixture/test_scenario_basic_service_client_cv",
+      "family_index": 0,
+      "per_family_instance_index": 0,
+      "run_name": "ScenarioBasicServiceClientFixture/test_scenario_basic_service_client",
+      "run_type": "aggregate",
+      "repetitions": 6,
+      "threads": 1,
+      "aggregate_name": "cv",
+      "aggregate_unit": "percentage",
+      "iterations": 6,
+      "real_time": 7.1201099086997149e-02,
+      "cpu_time": 1.6252808613289049e-02,
+      "time_unit": "ns"
+    }
+  ]
+}
diff --git a/scripts/box_plot_visualizer.py b/scripts/box_plot_visualizer.py
index a77a237..1eaa980 100644
--- a/scripts/box_plot_visualizer.py
+++ b/scripts/box_plot_visualizer.py
@@ -48,7 +48,7 @@ def get_middleware_dataset_for_scenario(scenario_name):
     return middleware_datasets
 
 
-def plot_dataset_of_scenario(plt, middleware_datasets):
+def plot_dataset_of_scenario(plt, scenario_name, middleware_datasets):
     labels = []
     colors = []
     datasets = []
@@ -59,7 +59,9 @@ def plot_dataset_of_scenario(plt, middleware_datasets):
         datasets.append(x["dataset"])
 
     fig, ax = plt.subplots()
+    ax.set_title(scenario_name)
     ax.set_ylabel("real time (ns)")
+    ax.set_xlabel("middlewares")
 
     bplot = ax.boxplot(datasets, patch_artist=True, tick_labels=labels)
 
@@ -70,6 +72,6 @@ def plot_dataset_of_scenario(plt, middleware_datasets):
 
 for scenario_name in os.listdir(BENCHMARK_RESULTS_DIR):
     middleware_datasets = get_middleware_dataset_for_scenario(scenario_name)
-    plot_dataset_of_scenario(plt, middleware_datasets)
+    plot_dataset_of_scenario(plt, scenario_name, middleware_datasets)
 
 plt.show()
diff --git a/scripts/run_all_benchmarks.sh b/scripts/run_all_benchmarks.sh
index d766760..800d8b6 100644
--- a/scripts/run_all_benchmarks.sh
+++ b/scripts/run_all_benchmarks.sh
@@ -28,7 +28,7 @@ echo "benchmark results directory is $benchmark_results_directory"
 
 echo "Benchmarking is starting!"
 echo "Starting initial scripts before benchmarks run!"
-sh "$initial_script"
+. "$initial_script"
 echo "Initial script has finished! Now starting to benchmark middleware with scenarios!"
 
 mkdir ${benchmark_results_directory}/scenario_basic_service_client -p

From f7c3cd6c44c6c3025f0bc5436fb6b48330b2dfa0 Mon Sep 17 00:00:00 2001
From: CihatAltiparmak <cihataltiparmak1@gmail.com>
Date: Mon, 5 Aug 2024 06:56:20 +0300
Subject: [PATCH 09/11] Delete unnecessary middleware_benchmark_results.json
 file

---
 middleware_benchmark_results.json | 184 ------------------------------
 1 file changed, 184 deletions(-)
 delete mode 100644 middleware_benchmark_results.json

diff --git a/middleware_benchmark_results.json b/middleware_benchmark_results.json
deleted file mode 100644
index 13932d7..0000000
--- a/middleware_benchmark_results.json
+++ /dev/null
@@ -1,184 +0,0 @@
-{
-  "context": {
-    "date": "2024-08-03T21:20:00+03:00",
-    "host_name": "jarbay51",
-    "executable": "/home/cihat/ws_moveit/install/moveit_middleware_benchmark/lib/moveit_middleware_benchmark/scenario_basic_service_client_benchmark_main",
-    "num_cpus": 8,
-    "mhz_per_cpu": 3500,
-    "cpu_scaling_enabled": true,
-    "caches": [
-      {
-        "type": "Data",
-        "level": 1,
-        "size": 32768,
-        "num_sharing": 2
-      },
-      {
-        "type": "Instruction",
-        "level": 1,
-        "size": 32768,
-        "num_sharing": 2
-      },
-      {
-        "type": "Unified",
-        "level": 2,
-        "size": 262144,
-        "num_sharing": 2
-      },
-      {
-        "type": "Unified",
-        "level": 3,
-        "size": 6291456,
-        "num_sharing": 8
-      }
-    ],
-    "load_avg": [0.214355,0.46875,0.448242],
-    "library_build_type": "release"
-  },
-  "benchmarks": [
-    {
-      "name": "ScenarioBasicServiceClientFixture/test_scenario_basic_service_client",
-      "family_index": 0,
-      "per_family_instance_index": 0,
-      "run_name": "ScenarioBasicServiceClientFixture/test_scenario_basic_service_client",
-      "run_type": "iteration",
-      "repetitions": 6,
-      "repetition_index": 0,
-      "threads": 1,
-      "iterations": 1,
-      "real_time": 4.5481033100004425e+09,
-      "cpu_time": 9.6490377400000000e+08,
-      "time_unit": "ns"
-    },
-    {
-      "name": "ScenarioBasicServiceClientFixture/test_scenario_basic_service_client",
-      "family_index": 0,
-      "per_family_instance_index": 0,
-      "run_name": "ScenarioBasicServiceClientFixture/test_scenario_basic_service_client",
-      "run_type": "iteration",
-      "repetitions": 6,
-      "repetition_index": 1,
-      "threads": 1,
-      "iterations": 1,
-      "real_time": 4.0113112649996767e+09,
-      "cpu_time": 9.7104653500000012e+08,
-      "time_unit": "ns"
-    },
-    {
-      "name": "ScenarioBasicServiceClientFixture/test_scenario_basic_service_client",
-      "family_index": 0,
-      "per_family_instance_index": 0,
-      "run_name": "ScenarioBasicServiceClientFixture/test_scenario_basic_service_client",
-      "run_type": "iteration",
-      "repetitions": 6,
-      "repetition_index": 2,
-      "threads": 1,
-      "iterations": 1,
-      "real_time": 4.1803860189993429e+09,
-      "cpu_time": 9.8092454400000012e+08,
-      "time_unit": "ns"
-    },
-    {
-      "name": "ScenarioBasicServiceClientFixture/test_scenario_basic_service_client",
-      "family_index": 0,
-      "per_family_instance_index": 0,
-      "run_name": "ScenarioBasicServiceClientFixture/test_scenario_basic_service_client",
-      "run_type": "iteration",
-      "repetitions": 6,
-      "repetition_index": 3,
-      "threads": 1,
-      "iterations": 1,
-      "real_time": 4.5593644450000286e+09,
-      "cpu_time": 9.7409754200000048e+08,
-      "time_unit": "ns"
-    },
-    {
-      "name": "ScenarioBasicServiceClientFixture/test_scenario_basic_service_client",
-      "family_index": 0,
-      "per_family_instance_index": 0,
-      "run_name": "ScenarioBasicServiceClientFixture/test_scenario_basic_service_client",
-      "run_type": "iteration",
-      "repetitions": 6,
-      "repetition_index": 4,
-      "threads": 1,
-      "iterations": 1,
-      "real_time": 3.8610252679991388e+09,
-      "cpu_time": 9.7080441899999940e+08,
-      "time_unit": "ns"
-    },
-    {
-      "name": "ScenarioBasicServiceClientFixture/test_scenario_basic_service_client",
-      "family_index": 0,
-      "per_family_instance_index": 0,
-      "run_name": "ScenarioBasicServiceClientFixture/test_scenario_basic_service_client",
-      "run_type": "iteration",
-      "repetitions": 6,
-      "repetition_index": 5,
-      "threads": 1,
-      "iterations": 1,
-      "real_time": 3.9889190239991875e+09,
-      "cpu_time": 9.3607051399999988e+08,
-      "time_unit": "ns"
-    },
-    {
-      "name": "ScenarioBasicServiceClientFixture/test_scenario_basic_service_client_mean",
-      "family_index": 0,
-      "per_family_instance_index": 0,
-      "run_name": "ScenarioBasicServiceClientFixture/test_scenario_basic_service_client",
-      "run_type": "aggregate",
-      "repetitions": 6,
-      "threads": 1,
-      "aggregate_name": "mean",
-      "aggregate_unit": "time",
-      "iterations": 6,
-      "real_time": 4.1915182218329697e+09,
-      "cpu_time": 9.6630788800000000e+08,
-      "time_unit": "ns"
-    },
-    {
-      "name": "ScenarioBasicServiceClientFixture/test_scenario_basic_service_client_median",
-      "family_index": 0,
-      "per_family_instance_index": 0,
-      "run_name": "ScenarioBasicServiceClientFixture/test_scenario_basic_service_client",
-      "run_type": "aggregate",
-      "repetitions": 6,
-      "threads": 1,
-      "aggregate_name": "median",
-      "aggregate_unit": "time",
-      "iterations": 6,
-      "real_time": 4.0958486419995098e+09,
-      "cpu_time": 9.7092547699999964e+08,
-      "time_unit": "ns"
-    },
-    {
-      "name": "ScenarioBasicServiceClientFixture/test_scenario_basic_service_client_stddev",
-      "family_index": 0,
-      "per_family_instance_index": 0,
-      "run_name": "ScenarioBasicServiceClientFixture/test_scenario_basic_service_client",
-      "run_type": "aggregate",
-      "repetitions": 6,
-      "threads": 1,
-      "aggregate_name": "stddev",
-      "aggregate_unit": "time",
-      "iterations": 6,
-      "real_time": 2.9844070423768336e+08,
-      "cpu_time": 1.5705217165175552e+07,
-      "time_unit": "ns"
-    },
-    {
-      "name": "ScenarioBasicServiceClientFixture/test_scenario_basic_service_client_cv",
-      "family_index": 0,
-      "per_family_instance_index": 0,
-      "run_name": "ScenarioBasicServiceClientFixture/test_scenario_basic_service_client",
-      "run_type": "aggregate",
-      "repetitions": 6,
-      "threads": 1,
-      "aggregate_name": "cv",
-      "aggregate_unit": "percentage",
-      "iterations": 6,
-      "real_time": 7.1201099086997149e-02,
-      "cpu_time": 1.6252808613289049e-02,
-      "time_unit": "ns"
-    }
-  ]
-}

From 6eb6b96da14dbe1ccc1773f15597857be7881df0 Mon Sep 17 00:00:00 2001
From: CihatAltiparmak <cihataltiparmak1@gmail.com>
Date: Mon, 5 Aug 2024 18:07:03 +0300
Subject: [PATCH 10/11] Update docs/how_to_run.md for directing to
 how_to_install

Co-authored-by: Henning Kayser <henningkayser@picknik.ai>
---
 docs/how_to_run.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/how_to_run.md b/docs/how_to_run.md
index 45ac6ce..9524eb9 100644
--- a/docs/how_to_run.md
+++ b/docs/how_to_run.md
@@ -1,5 +1,5 @@
 ## Run All Benchmarks
-If you haven't taken a look at [How To Install](how_to_install.md) section, please firstly start with [How To Install](how_to_install.md) Because it's assumed that you applied [How To Install](how_to_install.md) section in this section.
+Before running the following benchmarks, please read and apply the instructions outlined in the [How To Install](how_to_install.md) section for providing the necessary requirements.
 
 To run all benchmarks, just select your middleware implementation and go ahead with `run_all_benchmarks.sh` bash script. This command initially will run the script you chose. In this example, it's used the default config scripts which the necessary middleware configuration is applied. This property serves the functionality for the users to apply some custom configurations freely.
 

From 3f44a1acdc5b89fc41f5f8f78be3d0430473b09e Mon Sep 17 00:00:00 2001
From: CihatAltiparmak <cihataltiparmak1@gmail.com>
Date: Mon, 5 Aug 2024 18:07:35 +0300
Subject: [PATCH 11/11] Update docs/how_to_run.md for middleware-configuration
 related section

Co-authored-by: Henning Kayser <henningkayser@picknik.ai>
---
 docs/how_to_run.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/how_to_run.md b/docs/how_to_run.md
index 9524eb9..4b1118d 100644
--- a/docs/how_to_run.md
+++ b/docs/how_to_run.md
@@ -21,7 +21,7 @@ Let's explain all operations at `run_all_benchmarks.sh`.
 -i ./src/moveit_middleware_benchmark/middleware_configurations/rmw_cyclonedds/config.sh
 ```
 
-This argument is for selecting the initial scripts to be run. In this repository, These initial scripts is used for configuring middleware so that middleware is used more effectively. For example, you can use the initial script to configure TCP settings for rmw_zenoh like below.
+This argument is for selecting the initial scripts to be run. These initial scripts are used for configuring middleware-specific settings for improved performance. For example, you can use the initial script to configure TCP settings for rmw_zenoh like below.
 
 ```shell
 echo "The configurations for rmw_zenoh_cpp is started!"