Fix test_perf.sh script

ndsev · Jan 22, 2025 · 56a59ce · 56a59ce
1 parent 4757cf0
commit 56a59ce
Show file tree

Hide file tree

Showing 5 changed files with 102 additions and 58 deletions.
diff --git a/.github/workflows/build_linux_cpp.yml b/.github/workflows/build_linux_cpp.yml
@@ -135,3 +135,12 @@ jobs:
     - name: Run Zserio custom integration test
       run: |
         scripts/test_zs.sh cpp-${PACKAGE} -d test/data/others/gif/zs -s gif.zs
+
+    - name: Run Zserio performance test
+      run: |
+        scripts/test_perf.sh cpp-${PACKAGE} -d test/data/others/gif/zs -s gif.zs -b gif.GifFile \
+              -f test/data/others/gif/data/1pix.gif -c READ
+        scripts/test_perf.sh cpp-${PACKAGE} -d test/data/others/gif/zs -s gif.zs -b gif.GifFile \
+              -f test/data/others/gif/data/1pix.gif -c WRITE
+        scripts/test_perf.sh cpp-${PACKAGE} -d test/data/others/gif/zs -s gif.zs -b gif.GifFile \
+              -f test/data/others/gif/data/1pix.gif -c READ_WRITE
diff --git a/.github/workflows/build_windows_cpp.yml b/.github/workflows/build_windows_cpp.yml
@@ -119,5 +119,16 @@ jobs:
         scripts/test.sh cpp-${PACKAGE}
 
     - name: Run Zserio custom integration test
+      shell: bash
       run: |
         scripts/test_zs.sh cpp-${PACKAGE} -d test/data/others/gif/zs -s gif.zs
+
+    - name: Run Zserio performance test
+      shell: bash
+      run: |
+        scripts/test_perf.sh cpp-${PACKAGE} -d test/data/others/gif/zs -s gif.zs -b gif.GifFile \
+              -f test/data/others/gif/data/1pix.gif -c READ
+        scripts/test_perf.sh cpp-${PACKAGE} -d test/data/others/gif/zs -s gif.zs -b gif.GifFile \
+              -f test/data/others/gif/data/1pix.gif -c WRITE
+        scripts/test_perf.sh cpp-${PACKAGE} -d test/data/others/gif/zs -s gif.zs -b gif.GifFile \
+              -f test/data/others/gif/data/1pix.gif -c READ_WRITE
diff --git a/scripts/test_perf.sh b/scripts/test_perf.sh
@@ -17,11 +17,6 @@ generate_performance_test()
     local TEST_CONFIG="$1"; shift
     local PROFILE="$1"; shift
 
-    # use host paths in generated files
-    local DISABLE_SLASHES_CONVERSION=1
-    posix_to_host_path "${JSON_FILE}" HOST_JSON_FILE ${DISABLE_SLASHES_CONVERSION}
-    posix_to_host_path "${BLOB_FILE}" HOST_BLOB_FILE ${DISABLE_SLASHES_CONVERSION}
-
     local BLOB_INCLUDE_PATH=${BLOB_FULL_NAME//.//}.h
     local BLOB_CLASS_FULL_NAME=${BLOB_FULL_NAME//./::}
     local TOP_LEVEL_PACKAGE_NAME=${BLOB_FULL_NAME%%.*}
@@ -33,6 +28,10 @@ generate_performance_test()
         local INPUT_FILE="${JSON_FILE}"
     fi
 
+    # use host paths in generated files (needed for Windows)
+    local DISABLE_SLASHES_CONVERSION=1
+    posix_to_host_path "${INPUT_FILE}" INPUT_FILE ${DISABLE_SLASHES_CONVERSION}
+
     cat > "${SRC_FILE}" << EOF
 #include <fstream>
 #include <iostream>
@@ -270,19 +269,23 @@ EOF
     case "${TEST_CONFIG}" in
         "READ")
             cat >> "${SRC_FILE}" << EOF
-        zserio::deserialize(bitBuffer, readData[i]);
+        zserio::BitStreamReader reader(bitBuffer);
+        zserio::detail::read(reader, readData[i]);
 EOF
             ;;
         "READ_WRITE")
             cat >> "${SRC_FILE}" << EOF
-        auto readView = zserio::deserialize(bitBuffer, readData[i]);
-        zserio::serialize(readView);
+        zserio::BitStreamReader reader(bitBuffer);
+        auto readView = zserio::detail::read(reader, readData[i]);
+        zserio::BitStreamWriter writer(bitBuffer);
+        zserio::detail::write(writer, readView);
 EOF
             ;;
 
         "WRITE")
             cat >> "${SRC_FILE}" << EOF
-        zserio::serialize(readView);
+        zserio::BitStreamWriter writer(bitBuffer);
+        zserio::detail::write(writer, readView);
 EOF
             ;;
     esac
@@ -368,40 +371,31 @@ test_perf()
     local SWITCH_RUN_ONLY="$1"; shift
     local SWITCH_PROFILE="$1"; shift
 
-    # run C++ performance test
-    local TEST_SRC_DIR="${TEST_OUT_DIR}/src"
-    mkdir -p "${TEST_SRC_DIR}"
-    local TEST_SRC_FILE="${TEST_SRC_DIR}/PerformanceTest.cpp"
-    local TEST_LOG_DIR="${TEST_OUT_DIR}/log"
-    mkdir -p "${TEST_LOG_DIR}"
-    if [[ ${SWITCH_RUN_ONLY} == 0 ]] ; then
-        generate_performance_test "${TEST_SRC_FILE}" "${SWITCH_BLOB_NAME}" \
-                "${SWITCH_JSON_FILE}" "${SWITCH_BLOB_FILE}" "${TEST_LOG_DIR}/PerformanceTest.log" \
-                ${SWITCH_NUM_ITERATIONS} ${SWITCH_TEST_CONFIG} ${SWITCH_PROFILE}
-    fi
-    local CMAKE_ARGS=()
-    local CTEST_ARGS=("--verbose")
-    if [[ ${SWITCH_PROFILE} == 1 ]] ; then
-        CMAKE_ARGS=("-DCMAKE_BUILD_TYPE=RelWithDebInfo")
-        CTEST_ARGS+=("-T memcheck")
-    fi
-
-    # run test
-    test_zs "${ZSERIO_CPP17_DISTR_DIR}" "${ZSERIO_CPP17_PROJECT_ROOT}" "${ZSERIO_CPP17_BUILD_DIR}" \
-        "${TEST_OUT_DIR}" PARAM_CPP_TARGET_ARRAY[@] "${SWITCH_SOURCE}" "${SWITCH_DIRECTORY}" \
-        "${SWITCH_TEST_NAME}" "${TEST_SRC_FILE}"
-    if [ $? -ne 0 ] ; then
-        return 1
-    fi
+    # run C++ performance tests
+    local TEST_LOG_FILE_SUBDIR="log"
+    local TEST_LOG_FILE_NAME="PerformanceTest.log"
+    for CPP_TARGET in "${CPP_TARGETS[@]}" ; do
+        local TARGET_TEST_OUT_DIR="${TEST_OUT_DIR}/${CPP_TARGET}"
+        local TEST_SRC_DIR="${TARGET_TEST_OUT_DIR}/src"
+        mkdir -p "${TEST_SRC_DIR}"
+        local TEST_SRC_FILE="${TEST_SRC_DIR}/PerformanceTest.cpp"
+        local TEST_LOG_DIR="${TARGET_TEST_OUT_DIR}/${TEST_LOG_FILE_SUBDIR}"
+        mkdir -p "${TEST_LOG_DIR}"
+        mkdir -p "${TARGET_TEST_OUT_DIR}"
+        if [[ ${SWITCH_RUN_ONLY} == 0 ]] ; then
+            generate_performance_test "${TEST_SRC_FILE}" "${SWITCH_BLOB_NAME}" \
+                    "${SWITCH_JSON_FILE}" "${SWITCH_BLOB_FILE}" "${TEST_LOG_DIR}/${TEST_LOG_FILE_NAME}" \
+                    ${SWITCH_NUM_ITERATIONS} ${SWITCH_TEST_CONFIG} ${SWITCH_PROFILE}
+        fi
 
-    if [[ ${SWITCH_PROFILE} == 1 ]] ; then
-        echo
-        echo "C++ profiling finished, use one of the following commands for analysis:"
-        for CPP_TARGET in "${CPP_TARGETS[@]}" ; do
-            local CALLGRIND_FILE=$(${FIND} "${TEST_OUT_DIR}/cpp/${CPP_TARGET}" -name "callgrind.out")
-            echo "    kcachegrind ${CALLGRIND_FILE}"
-        done
-    fi
+        # run external integration test
+        test_zs "${ZSERIO_CPP17_DISTR_DIR}" "${ZSERIO_CPP17_PROJECT_ROOT}" "${ZSERIO_CPP17_BUILD_DIR}" \
+            "${TEST_OUT_DIR}" CPP_TARGET "${SWITCH_SOURCE}" "${SWITCH_DIRECTORY}" \
+            "${SWITCH_TEST_NAME}" "${TEST_SRC_FILE}" ${SWITCH_PROFILE}
+        if [ $? -ne 0 ] ; then
+            return 1
+        fi
+    done
 
     # collect results
     echo
@@ -417,7 +411,7 @@ test_perf()
            "Generator" "Total Duration" "Iterations" "Step Duration" "Blob Size" "Blob in Memory"
     echo -n "|" ; for i in {1..101} ; do echo -n "-" ; done ; echo "|"
     for CPP_TARGET in "${CPP_TARGETS[@]}" ; do
-        local PERF_TEST_FILE=$(${FIND} "${TEST_OUT_DIR}/cpp/${CPP_TARGET}" -name "PerformanceTest.log")
+        local PERF_TEST_FILE="${TEST_OUT_DIR}/${CPP_TARGET}/${TEST_LOG_FILE_SUBDIR}/${TEST_LOG_FILE_NAME}"
         local RESULTS=($(cat ${PERF_TEST_FILE}))
         printf "| %-21s | %14s | %10s | %15s | %10s | %14s |\n" \
                "C++ (${CPP_TARGET})" ${RESULTS[0]} ${RESULTS[1]} ${RESULTS[2]} ${RESULTS[3]} ${RESULTS[4]}
@@ -436,15 +430,15 @@ Description:
     Runs performance tests on given zserio sources using C++17 extension from distr directory.
 
 Usage:
-    $0 [-h] [-e] [-p] [-r] [--profile] [-o <dir>] [-d <dir>] [-t <name>] -[n <num>] [-c <config>]
+    $0 [-h] [-e] [-p] [-r] [-l] [-o <dir>] [-d <dir>] [-t <name>] -[n <num>] [-c <config>]
         target... -s <source> -b <blobname> [-f <blobfile> | -j <jsonfile>]
 
 Arguments:
     -h, --help              Show this help.
     -e, --help-env          Show help for enviroment variables.
     -p, --purge             Purge test build directory.
     -r, --run-only          Run already compiled PerformanceTests again.
-    --profile               Run the test in profiling mode and produce profiling data.
+    -l, --profile           Run the test in profiling mode and produce profiling data.
     -o <dir>, --output-directory <dir>
                             Output directory where tests will be run.
     -d <dir>, --source-dir <dir>
@@ -629,7 +623,7 @@ parse_arguments()
                 shift
                 ;;
 
-            "--profile")
+            "-l" | "--profile")
                 eval ${SWITCH_PROFILE_OUT}=1
                 shift
                 ;;

diff --git a/scripts/test_zs.sh b/scripts/test_zs.sh
@@ -7,7 +7,7 @@ source "${SCRIPT_DIR}/test.sh"
 # Run custom integration test
 test_zs()
 {
-    exit_if_argc_ne $# 9
+    exit_if_argc_ne $# 10
     local ZSERIO_CPP17_DISTR_DIR="$1"; shift
     local ZSERIO_CPP17_PROJECT_ROOT="$1"; shift
     local ZSERIO_CPP17_BUILD_DIR="$1"; shift
@@ -18,20 +18,26 @@ test_zs()
     local SWITCH_DIRECTORY="$1"; shift
     local SWITCH_TEST_NAME="$1"; shift
     local SWITCH_TEST_FILE="$1"; shift
+    local SWITCH_PROFILE="$1"; shift
 
     # run custom integration tests
     local MESSAGE="Zserio C++17 custom integration test"
     echo "STARTING - ${MESSAGE}"
 
     local TEST_SUITES_LIST="external"
     local CMAKE_ARGS=("-DZSERIO_RELEASE_ROOT=${ZSERIO_CPP17_BUILD_DIR}/extension/download"
-                      "-DZSERIO_CPP17_RELEASE_ROOT=${ZSERIO_CPP17_DISTR_DIR}"
-                      "-DZSERIO_TEST_SUITES=${TEST_SUITES_LIST}"
-                      "-DZSERIO_TEST_EXTERN_SCHEMA=${SWITCH_SOURCE}"
-                      "-DZSERIO_TEST_EXTERN_SCHEMA_ROOT=${SWITCH_DIRECTORY}"
-                      "-DZSERIO_TEST_EXTERN_SCHEMA_NAME=${SWITCH_TEST_NAME}"
-                      "-DZSERIO_TEST_EXTERN_SCHEMA_TEST=${SWITCH_TEST_FILE}")
+            "-DZSERIO_CPP17_RELEASE_ROOT=${ZSERIO_CPP17_DISTR_DIR}"
+            "-DZSERIO_TEST_SUITES=${TEST_SUITES_LIST}"
+            "-DZSERIO_TEST_EXTERN_SCHEMA=${SWITCH_SOURCE}"
+            "-DZSERIO_TEST_EXTERN_SCHEMA_ROOT=${SWITCH_DIRECTORY}"
+            "-DZSERIO_TEST_EXTERN_SCHEMA_NAME=${SWITCH_TEST_NAME}"
+            "-DZSERIO_TEST_EXTERN_SCHEMA_TEST=${SWITCH_TEST_FILE}")
     local CTEST_ARGS=(--verbose)
+    if [[ ${SWITCH_PROFILE} == 1 ]] ; then
+        CMAKE_ARGS+=("-DPROFILING_ENABLED=ON"
+                "-DCMAKE_BUILD_TYPE=RelWithDebInfo")
+        CTEST_ARGS+=("-T memcheck")
+    fi
     local CPP_TARGET="all"
     local TEST_SRC_DIR="${ZSERIO_CPP17_PROJECT_ROOT}/test"
     compile_cpp "${ZSERIO_CPP17_PROJECT_ROOT}" "${TEST_OUT_DIR}" "${TEST_SRC_DIR}" CPP_TARGETS[@] \
@@ -41,6 +47,12 @@ test_zs()
         return 1
     fi
 
+    if [[ ${SWITCH_PROFILE} == 1 ]] ; then
+        echo "C++ profiling finished, use the following commands for analysis:"
+        local CALLGRIND_FILE=$(${FIND} "${TEST_OUT_DIR}" -name "callgrind.out")
+        echo "    kcachegrind ${CALLGRIND_FILE}"
+    fi
+
     echo -e "FINISHED - ${MESSAGE}\n"
 
     return 0
@@ -54,12 +66,13 @@ Description:
     Runs given integration test on given zserio sources using C++17 extension from distr directory.
 
 Usage:
-    $0 [-h] [-e] [-p] [-o <dir>] [-d <dir>] [-s <source>] [-t <name>] [-f <file>] target
+    $0 [-h] [-e] [-p] [-l] [-o <dir>] [-d <dir>] [-s <source>] [-t <name>] [-f <file>] target
 
 Arguments:
     -h, --help            Show this help.
     -e, --help-env        Show help for enviroment variables.
     -p, --purge           Purge test build directory.
+    -l, --profile         Run the test in profiling mode and produce profiling data.
     -o <dir>, --output-directory <dir>
                           Output directory where tests will be run.
     -d <dir>, --source-dir <dir>
@@ -96,20 +109,22 @@ EOF
 # 3 - Environment help switch is present. Arguments after help switch have not been checked.
 parse_arguments()
 {
-    exit_if_argc_lt $# 7
+    exit_if_argc_lt $# 8
     local PARAM_CPP_TARGET_ARRAY_OUT="$1"; shift
     local SWITCH_OUT_DIR_OUT="$1"; shift
     local SWITCH_PURGE_OUT="$1"; shift
     local SWITCH_DIRECTORY_OUT="$1"; shift
     local SWITCH_SOURCE_OUT="$1"; shift
     local SWITCH_TEST_NAME_OUT="$1"; shift
     local SWITCH_TEST_FILE_OUT="$1"; shift
+    local SWITCH_PROFILE_OUT="$1"; shift
 
     eval ${SWITCH_PURGE_OUT}=0
     eval ${SWITCH_DIRECTORY_OUT}="."
     eval ${SWITCH_SOURCE_OUT}=""
     eval ${SWITCH_TEST_NAME_OUT}=""
     eval ${SWITCH_TEST_FILE_OUT}=""
+    eval ${SWITCH_PROFILE_OUT}=0
 
     local NUM_PARAMS=0
     local PARAM_ARRAY=();
@@ -179,6 +194,11 @@ parse_arguments()
                 shift 2
                 ;;
 
+            "-l" | "--profile")
+                eval ${SWITCH_PROFILE_OUT}=1
+                shift
+                ;;
+
             "-"*)
                 stderr_echo "Invalid switch '${ARG}'!"
                 echo
@@ -248,8 +268,9 @@ main()
     local SWITCH_SOURCE
     local SWITCH_TEST_NAME
     local SWITCH_TEST_FILE
+    local SWITCH_PROFILE
     parse_arguments PARAM_CPP_TARGET_ARRAY SWITCH_OUT_DIR SWITCH_PURGE SWITCH_DIRECTORY SWITCH_SOURCE \
-            SWITCH_TEST_NAME SWITCH_TEST_FILE "$@"
+            SWITCH_TEST_NAME SWITCH_TEST_FILE SWITCH_PROFILE "$@"
     local PARSE_RESULT=$?
     if [ ${PARSE_RESULT} -eq 2 ] ; then
         print_help
@@ -306,8 +327,8 @@ main()
     # run test
     local ZSERIO_CPP17_DISTR_DIR="${SWITCH_OUT_DIR}/distr"
     test_zs "${ZSERIO_CPP17_DISTR_DIR}" "${ZSERIO_CPP17_PROJECT_ROOT}" "${ZSERIO_CPP17_BUILD_DIR}" \
-            "${TEST_OUT_DIR}" PARAM_CPP_TARGET_ARRAY[@] "${SWITCH_SOURCE}" "${SWITCH_DIRECTORY}" \
-            "${SWITCH_TEST_NAME}" "${SWITCH_TEST_FILE}"
+            "${TEST_OUT_DIR}" PARAM_CPP_TARGET_ARRAY[@] "${SWITCH_SOURCE}" \
+            "${SWITCH_DIRECTORY}" "${SWITCH_TEST_NAME}" "${SWITCH_TEST_FILE}" ${SWITCH_PROFILE}
     if [ $? -ne 0 ] ; then
         return 1
     fi

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
@@ -12,6 +12,7 @@ enable_testing()
 
 # input parameters
 set(SANITIZERS_ENABLED OFF CACHE BOOL "Whether the compiler sanitizers are enabled.")
+set(PROFILING_ENABLED OFF CACHE BOOL "Whether the valgrind profiling in tests is enabled (for gcc only).")
 set(CLANG_TIDY_BIN "" CACHE STRING "Name of clang-tidy binary. If empty, clang-tidy tool is not called.")
 set(CLANG_FORMAT_BIN "" CACHE STRING "Name of clang-format binary. If empty, clang-format tool is not called.")
 
@@ -50,6 +51,14 @@ if (SANITIZERS_ENABLED)
     compiler_set_undefined_sanitizer()
 endif ()
 
+if (PROFILING_ENABLED)
+    string(CONCAT MEMORYCHECK_COMMAND_OPTIONS
+        "--tool=callgrind -v --instr-atstart=no --collect-atstart=no --collect-jumps=yes --dump-instr=yes "
+        "--callgrind-out-file=callgrind.out"
+    )
+    include(CTest)
+endif ()
+
 # add test utility library
 add_subdirectory(test_utils test_utils)