From fa0705694ac643871ed457aec590eb4f6142dc43 Mon Sep 17 00:00:00 2001 From: ParthShitole Date: Tue, 22 Oct 2024 20:36:47 +0530 Subject: [PATCH 1/6] backends/ebpf: Prevent compiling backend tests if dependencies are not installed - The added code checks for required dependencies before adding tests. - This makes sures the Test don't fail due to missing dependencies. Signed-off-by: Parth Shitole --- CMakeLists.txt | 25 +++++++++++++++++ backends/ebpf/CMakeLists.txt | 52 +++++++++++++++++++++++------------- 2 files changed, 58 insertions(+), 19 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7ddf63af630..b9a6341cd49 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -282,6 +282,31 @@ endif() # enable CTest enable_testing () +# Macro to check dependencies before adding tests + +macro(CHECK_DEPENDENCIES TEST_DEPENDENCY_PROGRAMS TEST_DEPENDENCY_LIBRARIES) + foreach(PROG ${TEST_DEPENDENCY_PROGRAMS}) + find_program(${PROG}_FOUND ${PROG}) + if (${PROG}_FOUND) + message(STATUS "Found program ${PROG} at ${${PROG}_FOUND}") + else() + message(WARNING "Missing program ${PROG}, disabling relevant tests." + " Please install ${PROG} and ensure it is in your PATH.") + set(TEST_DEPENDENCY_PRESENT FALSE) + endif() + endforeach() + + foreach(LIB ${TEST_DEPENDENCY_LIBRARIES}) + find_library(${LIB}_FOUND ${LIB} HINTS "${CMAKE_CURRENT_SOURCE_DIR}/runtime/usr/lib64/") + if (${LIB}_FOUND) + message(STATUS "Found library ${LIB} at ${${LIB}_FOUND}") + else() + message(WARNING "Missing library ${LIB}, disabling relevant tests." + " Please install ${LIB}.") + set(TEST_DEPENDENCY_PRESENT FALSE) + endif() + endforeach() +endmacro() # if we want to manage versions in CMake ... # include (cmake/P4CVersion.cmake) diff --git a/backends/ebpf/CMakeLists.txt b/backends/ebpf/CMakeLists.txt index 5f6c896d4be..fbe0da20b64 100644 --- a/backends/ebpf/CMakeLists.txt +++ b/backends/ebpf/CMakeLists.txt @@ -221,6 +221,16 @@ else() set (SUPPORTS_KERNEL False) endif() +# List of executable dependencies +set(TEST_DEPENDENCY_PROGRAMS scapy tcpdump pcap) + +# List of library dependencies +set(TEST_DEPENDENCY_LIBRARIES libpcap-dev gcc-multilib) + +set(TEST_DEPENDENCY_PRESENT TRUE) + +CHECK_DEPENDENCIES("${TEST_DEPENDENCY_PROGRAMS}" "${TEST_DEPENDENCY_LIBRARIES}") + # check for the libbpf library find_library(LIBBPF NAMES bpf HINTS "${CMAKE_CURRENT_SOURCE_DIR}/runtime/usr/lib64/") if (LIBBPF) @@ -232,25 +242,29 @@ else() set (SUPPORTS_KERNEL False) endif() -# Only add the kernel tests if the two requirements are met -if (SUPPORTS_KERNEL) - p4c_add_tests("ebpf-kernel" ${EBPF_DRIVER_KERNEL} ${EBPF_TEST_SUITES} "${XFAIL_TESTS_KERNEL}") - p4c_add_tests("ebpf-kernel" ${EBPF_DRIVER_KERNEL} ${EBPF_KERNEL_TEST_SUITES} "${XFAIL_TESTS_KERNEL}") - # These are special tests with args that are not included - # in the default ebpf tests - p4c_add_test_with_args("ebpf-kernel" ${EBPF_DRIVER_KERNEL} FALSE "testdata/p4_16_samples/ebpf_conntrack_extern.p4" "testdata/p4_16_samples/ebpf_conntrack_extern.p4" "--extern-file ${P4C_SOURCE_DIR}/testdata/extern_modules/extern-conntrack-ebpf.c" "") - p4c_add_test_with_args("ebpf-kernel" ${EBPF_DRIVER_KERNEL} FALSE "testdata/p4_16_samples/ebpf_checksum_extern.p4" "testdata/p4_16_samples/ebpf_checksum_extern.p4" "--extern-file ${P4C_SOURCE_DIR}/testdata/extern_modules/extern-checksum-ebpf.c" "") -endif() -# ToDo Add check which verifies that BCC is installed -# Ideally, this is done via check for the python package -p4c_add_tests("ebpf-bcc" ${EBPF_DRIVER_BCC} ${EBPF_TEST_SUITES} "${XFAIL_TESTS_BCC}") -p4c_add_tests("ebpf" ${EBPF_DRIVER_TEST} ${EBPF_TEST_SUITES} "${XFAIL_TESTS_TEST}") -p4c_add_tests("ebpf-errors" ${EBPF_DRIVER_TEST} ${EBPF_ERRORS_SUITES} "${XFAIL_TESTS_TEST}") +if(TEST_DEPENDENCY_PRESENT) + # Only add the kernel tests if the two requirements are met + if (SUPPORTS_KERNEL) + p4c_add_tests("ebpf-kernel" ${EBPF_DRIVER_KERNEL} ${EBPF_TEST_SUITES} "${XFAIL_TESTS_KERNEL}") + p4c_add_tests("ebpf-kernel" ${EBPF_DRIVER_KERNEL} ${EBPF_KERNEL_TEST_SUITES} "${XFAIL_TESTS_KERNEL}") + # These are special tests with args that are not included + # in the default ebpf tests + p4c_add_test_with_args("ebpf-kernel" ${EBPF_DRIVER_KERNEL} FALSE "testdata/p4_16_samples/ebpf_conntrack_extern.p4" "testdata/p4_16_samples/ebpf_conntrack_extern.p4" "--extern-file ${P4C_SOURCE_DIR}/testdata/extern_modules/extern-conntrack-ebpf.c" "") + p4c_add_test_with_args("ebpf-kernel" ${EBPF_DRIVER_KERNEL} FALSE "testdata/p4_16_samples/ebpf_checksum_extern.p4" "testdata/p4_16_samples/ebpf_checksum_extern.p4" "--extern-file ${P4C_SOURCE_DIR}/testdata/extern_modules/extern-checksum-ebpf.c" "") + endif() + # ToDo Add check which verifies that BCC is installed + # Ideally, this is done via check for the python package + p4c_add_tests("ebpf-bcc" ${EBPF_DRIVER_BCC} ${EBPF_TEST_SUITES} "${XFAIL_TESTS_BCC}") + p4c_add_tests("ebpf" ${EBPF_DRIVER_TEST} ${EBPF_TEST_SUITES} "${XFAIL_TESTS_TEST}") + p4c_add_tests("ebpf-errors" ${EBPF_DRIVER_TEST} ${EBPF_ERRORS_SUITES} "${XFAIL_TESTS_TEST}") -# These are special tests with args that are not included in the default ebpf tests -p4c_add_test_with_args("ebpf" ${EBPF_DRIVER_TEST} FALSE "testdata/p4_16_samples/ebpf_checksum_extern.p4" "testdata/p4_16_samples/ebpf_checksum_extern.p4" "--extern-file ${P4C_SOURCE_DIR}/testdata/extern_modules/extern-checksum-ebpf.c" "") -# FIXME:This does not work yet -# We do not have support for dynamic addition of tables in the test framework -p4c_add_test_with_args("ebpf" ${EBPF_DRIVER_TEST} TRUE "testdata/p4_16_samples/ebpf_conntrack_extern.p4" "testdata/p4_16_samples/ebpf_conntrack_extern.p4" "--extern-file ${P4C_SOURCE_DIR}/testdata/extern_modules/extern-conntrack-ebpf.c" "") + # These are special tests with args that are not included in the default ebpf tests + p4c_add_test_with_args("ebpf" ${EBPF_DRIVER_TEST} FALSE "testdata/p4_16_samples/ebpf_checksum_extern.p4" "testdata/p4_16_samples/ebpf_checksum_extern.p4" "--extern-file ${P4C_SOURCE_DIR}/testdata/extern_modules/extern-checksum-ebpf.c" "") + # FIXME:This does not work yet + # We do not have support for dynamic addition of tables in the test framework + p4c_add_test_with_args("ebpf" ${EBPF_DRIVER_TEST} TRUE "testdata/p4_16_samples/ebpf_conntrack_extern.p4" "testdata/p4_16_samples/ebpf_conntrack_extern.p4" "--extern-file ${P4C_SOURCE_DIR}/testdata/extern_modules/extern-conntrack-ebpf.c" "") +else() + message(WARNING "Skipped adding Tests due to missing dependencies. Please install the dependencies and try again") +endif() message(STATUS "Done with configuring BPF back end") From ae735d856088ecb4df46cabf444128ad6f3a7d86 Mon Sep 17 00:00:00 2001 From: Parth Shitole Date: Wed, 23 Oct 2024 20:10:40 +0530 Subject: [PATCH 2/6] Macro converted to a function and placed in P4CUtils - The Previous implementation of a macro is now converted to a function and placed in the P4CUtils.cmake file - Other minor changes are made to improve the functionality. Signed-off-by: Parth Shitole --- CMakeLists.txt | 26 -------------------------- backends/ebpf/CMakeLists.txt | 10 ++++------ cmake/P4CUtils.cmake | 33 +++++++++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 32 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b9a6341cd49..981722f0ca9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -282,32 +282,6 @@ endif() # enable CTest enable_testing () -# Macro to check dependencies before adding tests - -macro(CHECK_DEPENDENCIES TEST_DEPENDENCY_PROGRAMS TEST_DEPENDENCY_LIBRARIES) - foreach(PROG ${TEST_DEPENDENCY_PROGRAMS}) - find_program(${PROG}_FOUND ${PROG}) - if (${PROG}_FOUND) - message(STATUS "Found program ${PROG} at ${${PROG}_FOUND}") - else() - message(WARNING "Missing program ${PROG}, disabling relevant tests." - " Please install ${PROG} and ensure it is in your PATH.") - set(TEST_DEPENDENCY_PRESENT FALSE) - endif() - endforeach() - - foreach(LIB ${TEST_DEPENDENCY_LIBRARIES}) - find_library(${LIB}_FOUND ${LIB} HINTS "${CMAKE_CURRENT_SOURCE_DIR}/runtime/usr/lib64/") - if (${LIB}_FOUND) - message(STATUS "Found library ${LIB} at ${${LIB}_FOUND}") - else() - message(WARNING "Missing library ${LIB}, disabling relevant tests." - " Please install ${LIB}.") - set(TEST_DEPENDENCY_PRESENT FALSE) - endif() - endforeach() -endmacro() - # if we want to manage versions in CMake ... # include (cmake/P4CVersion.cmake) # set (CPACK_PACKAGE_VERSION_MAJOR ${__P4C_VERSION_MAJOR}) diff --git a/backends/ebpf/CMakeLists.txt b/backends/ebpf/CMakeLists.txt index fbe0da20b64..b1154c7e624 100644 --- a/backends/ebpf/CMakeLists.txt +++ b/backends/ebpf/CMakeLists.txt @@ -222,14 +222,12 @@ else() endif() # List of executable dependencies -set(TEST_DEPENDENCY_PROGRAMS scapy tcpdump pcap) +set(TEST_DEPENDENCY_PROGRAMS tcpdump) # List of library dependencies -set(TEST_DEPENDENCY_LIBRARIES libpcap-dev gcc-multilib) +set(TEST_DEPENDENCY_LIBRARIES libpcap-dev gcc-multilib scapy) -set(TEST_DEPENDENCY_PRESENT TRUE) - -CHECK_DEPENDENCIES("${TEST_DEPENDENCY_PROGRAMS}" "${TEST_DEPENDENCY_LIBRARIES}") +CHECK_DEPENDENCIES(TEST_DEPENDENCY_PRESENT "${TEST_DEPENDENCY_PROGRAMS}" "${TEST_DEPENDENCY_LIBRARIES}") # check for the libbpf library find_library(LIBBPF NAMES bpf HINTS "${CMAKE_CURRENT_SOURCE_DIR}/runtime/usr/lib64/") @@ -264,7 +262,7 @@ if(TEST_DEPENDENCY_PRESENT) # We do not have support for dynamic addition of tables in the test framework p4c_add_test_with_args("ebpf" ${EBPF_DRIVER_TEST} TRUE "testdata/p4_16_samples/ebpf_conntrack_extern.p4" "testdata/p4_16_samples/ebpf_conntrack_extern.p4" "--extern-file ${P4C_SOURCE_DIR}/testdata/extern_modules/extern-conntrack-ebpf.c" "") else() - message(WARNING "Skipped adding Tests due to missing dependencies. Please install the dependencies and try again") + message(WARNING "Skipped adding Tests due to missing dependencies" "Please install the dependencies and try again") endif() message(STATUS "Done with configuring BPF back end") diff --git a/cmake/P4CUtils.cmake b/cmake/P4CUtils.cmake index 509c90f7b16..944e1b8b7e3 100644 --- a/cmake/P4CUtils.cmake +++ b/cmake/P4CUtils.cmake @@ -420,3 +420,36 @@ function(get_all_targets _result _dir) get_directory_property(_sub_targets DIRECTORY "${_dir}" BUILDSYSTEM_TARGETS) set(${_result} ${${_result}} ${_sub_targets} PARENT_SCOPE) endfunction() + +# Checks for presence of programs and dependencies +function(CHECK_DEPENDENCIES OUT_VAR TEST_DEPENDENCY_PROGRAMS TEST_DEPENDENCY_LIBRARIES) + set(ALL_FOUND TRUE) + + foreach(PROG ${TEST_DEPENDENCY_PROGRAMS}) + find_program(PROG_PATH ${PROG}) + if (NOT PROG_PATH) + message(WARNING "Missing program ${PROG}." + " Please install ${PROG} and ensure it is in your PATH.") + set(ALL_FOUND FALSE) + else() + message(STATUS "Found program ${PROG} at ${PROG_PATH}") + endif() + endforeach() + + foreach(LIB ${TEST_DEPENDENCY_LIBRARIES}) + find_library(LIB_PATH ${LIB} HINTS "${CMAKE_CURRENT_SOURCE_DIR}/runtime/usr/lib64/") + if (NOT LIB_PATH) + message(WARNING "Missing library ${LIB}." + " Please install ${LIB}.") + set(ALL_FOUND FALSE) + else() + message(STATUS "Found library ${LIB} at ${LIB_PATH}") + endif() + endforeach() + + if (ALL_FOUND) + set(${OUT_VAR} TRUE PARENT_SCOPE) + else() + set(${OUT_VAR} FALSE PARENT_SCOPE) + endif() +endfunction() \ No newline at end of file From 90939afa729a70f7c01eb3910a060e1ebb2a5ac8 Mon Sep 17 00:00:00 2001 From: ParthShitole <101716280+ParthShitole@users.noreply.github.com> Date: Wed, 23 Oct 2024 20:25:25 +0530 Subject: [PATCH 3/6] Update CMakeLists.txt Removing Unintended changes Signed-off-by: Parth Shitole --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 981722f0ca9..7ddf63af630 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -282,6 +282,7 @@ endif() # enable CTest enable_testing () + # if we want to manage versions in CMake ... # include (cmake/P4CVersion.cmake) # set (CPACK_PACKAGE_VERSION_MAJOR ${__P4C_VERSION_MAJOR}) From 038dd22ff0089d9cdf6e3a33ef77b0d738d0dc8e Mon Sep 17 00:00:00 2001 From: ParthShitole Date: Thu, 24 Oct 2024 20:00:55 +0530 Subject: [PATCH 4/6] New optional parameter HINTS_DIRS added to CHECK_DEPENDECIES - Intially the HINTS variable for find_libraries was hard coded. - The path may change for different Operating Systems. - The HINTS parameter can now be populated by the caller, and will only be used if provided. - Minor Typos are also fixed. Signed-off-by: Parth Shitole --- backends/ebpf/CMakeLists.txt | 2 +- cmake/P4CUtils.cmake | 14 ++++++++++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/backends/ebpf/CMakeLists.txt b/backends/ebpf/CMakeLists.txt index b1154c7e624..a04b0e398af 100644 --- a/backends/ebpf/CMakeLists.txt +++ b/backends/ebpf/CMakeLists.txt @@ -262,7 +262,7 @@ if(TEST_DEPENDENCY_PRESENT) # We do not have support for dynamic addition of tables in the test framework p4c_add_test_with_args("ebpf" ${EBPF_DRIVER_TEST} TRUE "testdata/p4_16_samples/ebpf_conntrack_extern.p4" "testdata/p4_16_samples/ebpf_conntrack_extern.p4" "--extern-file ${P4C_SOURCE_DIR}/testdata/extern_modules/extern-conntrack-ebpf.c" "") else() - message(WARNING "Skipped adding Tests due to missing dependencies" "Please install the dependencies and try again") + message(WARNING "Skipped adding Tests due to missing dependencies" " Please install the dependencies and try again") endif() message(STATUS "Done with configuring BPF back end") diff --git a/cmake/P4CUtils.cmake b/cmake/P4CUtils.cmake index 944e1b8b7e3..68c19854df0 100644 --- a/cmake/P4CUtils.cmake +++ b/cmake/P4CUtils.cmake @@ -421,7 +421,11 @@ function(get_all_targets _result _dir) set(${_result} ${${_result}} ${_sub_targets} PARENT_SCOPE) endfunction() -# Checks for presence of programs and dependencies +# Checks for presence of programs and dependencies +# This function supports a optional parameter HINTS_DIRS. +# In case cmake is unable to find a library even if it is present, +# you can pass the exact directory path as the last parameter of this function. +# Eg. CHECK_DEPENDENCIES(VAR "${TEST_PROGRAMS}" "${TEST_LIBRARIES}" "${HINTS_DIRS}") function(CHECK_DEPENDENCIES OUT_VAR TEST_DEPENDENCY_PROGRAMS TEST_DEPENDENCY_LIBRARIES) set(ALL_FOUND TRUE) @@ -437,7 +441,13 @@ function(CHECK_DEPENDENCIES OUT_VAR TEST_DEPENDENCY_PROGRAMS TEST_DEPENDENCY_LIB endforeach() foreach(LIB ${TEST_DEPENDENCY_LIBRARIES}) - find_library(LIB_PATH ${LIB} HINTS "${CMAKE_CURRENT_SOURCE_DIR}/runtime/usr/lib64/") + if (ARGC GREATER 3) + set(HINTS_DIRS_ARG ${ARGV4}) + find_library(LIB_PATH ${LIB} HINTS ${HINTS_DIRS}) + else() + find_library(LIB_PATH ${LIB}) + endif() + if (NOT LIB_PATH) message(WARNING "Missing library ${LIB}." " Please install ${LIB}.") From 43252ef39a4e16933e1b79c0fed164126fbc7ec5 Mon Sep 17 00:00:00 2001 From: ParthShitole Date: Sun, 27 Oct 2024 12:00:12 +0530 Subject: [PATCH 5/6] Fix the cache stallness in CHECK_DEPENDENCIES function - The current implementation of the above function may suffer from stallness. - Required changes are made to avoid this. - Some minor typos are also fixed. Signed-off-by: ParthShitole --- backends/ebpf/CMakeLists.txt | 2 +- cmake/P4CUtils.cmake | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/backends/ebpf/CMakeLists.txt b/backends/ebpf/CMakeLists.txt index a04b0e398af..bb5bdab632f 100644 --- a/backends/ebpf/CMakeLists.txt +++ b/backends/ebpf/CMakeLists.txt @@ -262,7 +262,7 @@ if(TEST_DEPENDENCY_PRESENT) # We do not have support for dynamic addition of tables in the test framework p4c_add_test_with_args("ebpf" ${EBPF_DRIVER_TEST} TRUE "testdata/p4_16_samples/ebpf_conntrack_extern.p4" "testdata/p4_16_samples/ebpf_conntrack_extern.p4" "--extern-file ${P4C_SOURCE_DIR}/testdata/extern_modules/extern-conntrack-ebpf.c" "") else() - message(WARNING "Skipped adding Tests due to missing dependencies" " Please install the dependencies and try again") + message(WARNING "Skipped adding Tests due to missing dependencies." " Please install the dependencies and try again") endif() message(STATUS "Done with configuring BPF back end") diff --git a/cmake/P4CUtils.cmake b/cmake/P4CUtils.cmake index 68c19854df0..bd411a4156f 100644 --- a/cmake/P4CUtils.cmake +++ b/cmake/P4CUtils.cmake @@ -430,30 +430,30 @@ function(CHECK_DEPENDENCIES OUT_VAR TEST_DEPENDENCY_PROGRAMS TEST_DEPENDENCY_LIB set(ALL_FOUND TRUE) foreach(PROG ${TEST_DEPENDENCY_PROGRAMS}) - find_program(PROG_PATH ${PROG}) - if (NOT PROG_PATH) + find_program(${PROG}_PROG_PATH ${PROG} NO_CACHE) + if (NOT ${PROG}_PROG_PATH) message(WARNING "Missing program ${PROG}." " Please install ${PROG} and ensure it is in your PATH.") set(ALL_FOUND FALSE) else() - message(STATUS "Found program ${PROG} at ${PROG_PATH}") + message(STATUS "Found program ${PROG}") endif() endforeach() foreach(LIB ${TEST_DEPENDENCY_LIBRARIES}) if (ARGC GREATER 3) set(HINTS_DIRS_ARG ${ARGV4}) - find_library(LIB_PATH ${LIB} HINTS ${HINTS_DIRS}) + find_library(${LIB}_LIB_PATH NAMES ${LIB} NO_CACHE HINTS ${HINTS_DIRS}) else() - find_library(LIB_PATH ${LIB}) + find_library(${LIB}_LIB_PATH NAMES ${LIB} NO_CACHE) endif() - if (NOT LIB_PATH) + if (NOT ${LIB}_LIB_PATH) message(WARNING "Missing library ${LIB}." " Please install ${LIB}.") set(ALL_FOUND FALSE) else() - message(STATUS "Found library ${LIB} at ${LIB_PATH}") + message(STATUS "Found library ${LIB}") endif() endforeach() From 85a7c0cffcb1266291f80c786d7e767a6657f846 Mon Sep 17 00:00:00 2001 From: ParthShitole Date: Mon, 18 Nov 2024 22:13:25 +0530 Subject: [PATCH 6/6] Correct the dependencies checked. - Corrected the specific dependencies to be checked before adding tests for ebpf Signed-off-by: Parth Shitole --- backends/ebpf/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backends/ebpf/CMakeLists.txt b/backends/ebpf/CMakeLists.txt index bb5bdab632f..ab4bbd98733 100644 --- a/backends/ebpf/CMakeLists.txt +++ b/backends/ebpf/CMakeLists.txt @@ -222,10 +222,10 @@ else() endif() # List of executable dependencies -set(TEST_DEPENDENCY_PROGRAMS tcpdump) +set(TEST_DEPENDENCY_PROGRAMS gcc-multilib) # List of library dependencies -set(TEST_DEPENDENCY_LIBRARIES libpcap-dev gcc-multilib scapy) +set(TEST_DEPENDENCY_LIBRARIES python3-six libgmp-dev libjansson-dev) CHECK_DEPENDENCIES(TEST_DEPENDENCY_PRESENT "${TEST_DEPENDENCY_PROGRAMS}" "${TEST_DEPENDENCY_LIBRARIES}")