diff --git a/build/build_edge.sh b/build/build_edge.sh index 8c1ea547464..5ec3109609b 100755 --- a/build/build_edge.sh +++ b/build/build_edge.sh @@ -45,6 +45,7 @@ install_recipes() if [ $? != 0 ]; then echo "inherit externalsrc" > $XRT_BB echo "EXTERNALSRC = \"$XRT_REPO_DIR/src\"" >> $XRT_BB + echo "EXTRA_OECMAKE += \"-DMY_VITIS=$XILINX_VITIS\"" >> $XRT_BB echo 'EXTERNALSRC_BUILD = "${WORKDIR}/build"' >> $XRT_BB echo 'PACKAGE_CLASSES = "package_rpm"' >> $XRT_BB echo 'LICENSE = "GPLv2 & Apache-2.0"' >> $XRT_BB @@ -307,7 +308,10 @@ fi if [ -f $SETTINGS_FILE ]; then source $SETTINGS_FILE fi -source $PETALINUX/settings.sh +source $PETALINUX/settings.sh + +VITIS_FILE="${THIS_SCRIPT_DIR}/vitis.build" +source $VITIS_FILE if [[ $AARCH = $aarch64_dir ]]; then if [[ -f $PETALINUX/../../bsp/release/zynqmp-common-v$PETALINUX_VER-final.bsp ]]; then diff --git a/build/vitis.build b/build/vitis.build new file mode 100644 index 00000000000..66e90af624e --- /dev/null +++ b/build/vitis.build @@ -0,0 +1 @@ +source /proj/xbuilds/2023.2_daily_latest/installs/lin64/Vitis/2023.2/settings64.sh diff --git a/src/runtime_src/core/edge/ps_kernels/CMakeLists.txt b/src/runtime_src/core/edge/ps_kernels/CMakeLists.txt index 84d74da5fdf..1d4357d078a 100644 --- a/src/runtime_src/core/edge/ps_kernels/CMakeLists.txt +++ b/src/runtime_src/core/edge/ps_kernels/CMakeLists.txt @@ -1,9 +1,34 @@ # SPDX-License-Identifier: Apache-2.0 # Copyright (C) 2019-2022 Xilinx, Inc. All rights reserved. -# Copyright (C) 2022 Advanced Micro Devices, Inc. All rights reserved. -if (DEFINED XRT_AIE_BUILD) - add_subdirectory(sample) - add_subdirectory(profiling) -endif() +# Copyright (C) 2022-2023 Advanced Micro Devices, Inc. All rights reserved. +set(PS_KERNEL_INSTALL_DIR "${XRT_INSTALL_LIB_DIR}/ps_kernels_lib") + +function(generate_ps_kernel_xclbin PS_KERNEL_NAME) + set(XCLBIN_NAME "${PS_KERNEL_NAME}.xclbin") + set(XCLBIN_TARGET "${PS_KERNEL_NAME}_xclbin") + set(PS_KERNEL_LOCATION "lib${PS_KERNEL_NAME}.so") + set(XCLBIN_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/${XCLBIN_NAME}") + + add_custom_command( + OUTPUT ${XCLBIN_LOCATION} + COMMAND "${MY_VITIS}/bin/xclbinutil" --output ${XCLBIN_LOCATION} --add-pskernel ${PS_KERNEL_LOCATION} --force + DEPENDS ${PS_KERNEL_NAME} + ) + + add_custom_target(${XCLBIN_TARGET} ALL + DEPENDS ${XCLBIN_LOCATION} + ) -add_subdirectory(xrt) + install (FILES ${XCLBIN_LOCATION} + DESTINATION ${PS_KERNEL_INSTALL_DIR} + ) +endfunction() + +if (DEFINED MY_VITIS) + if (DEFINED XRT_AIE_BUILD) + add_subdirectory(sample) + add_subdirectory(profiling) + endif() + + add_subdirectory(xrt) +endif() diff --git a/src/runtime_src/core/edge/ps_kernels/xrt/CMakeLists.txt b/src/runtime_src/core/edge/ps_kernels/xrt/CMakeLists.txt index f312610c536..350eecca7e4 100644 --- a/src/runtime_src/core/edge/ps_kernels/xrt/CMakeLists.txt +++ b/src/runtime_src/core/edge/ps_kernels/xrt/CMakeLists.txt @@ -1,4 +1,5 @@ # SPDX-License-Identifier: Apache-2.0 -# Copyright (C) 2022 Advanced Micro Devices, Inc. All rights reserved. +# Copyright (C) 2022-2023 Advanced Micro Devices, Inc. All rights reserved. # add_subdirectory(instance_query) +add_subdirectory(tests) diff --git a/src/runtime_src/core/edge/ps_kernels/xrt/instance_query/CMakeLists.txt b/src/runtime_src/core/edge/ps_kernels/xrt/instance_query/CMakeLists.txt index 739d59e812f..9db668001c2 100644 --- a/src/runtime_src/core/edge/ps_kernels/xrt/instance_query/CMakeLists.txt +++ b/src/runtime_src/core/edge/ps_kernels/xrt/instance_query/CMakeLists.txt @@ -3,8 +3,8 @@ # set(PS_KERNEL_INSTALL_DIR "${XRT_INSTALL_LIB_DIR}/ps_kernels_lib") -add_library(instance_query OBJECT - instance_query.cpp +add_library(instance_query SHARED + "instance_query.cpp" ) set_target_properties(instance_query PROPERTIES @@ -12,6 +12,11 @@ set_target_properties(instance_query PROPERTIES SOVERSION ${XRT_SOVERSION} ) +target_link_libraries(instance_query + PRIVATE + ${Boost_FILESYSTEM_LIBRARY} + ) + install (TARGETS instance_query EXPORT xrt-targets LIBRARY DESTINATION ${PS_KERNEL_INSTALL_DIR} diff --git a/src/runtime_src/core/edge/ps_kernels/xrt/tests/CMakeLists.txt b/src/runtime_src/core/edge/ps_kernels/xrt/tests/CMakeLists.txt new file mode 100644 index 00000000000..0256f523d15 --- /dev/null +++ b/src/runtime_src/core/edge/ps_kernels/xrt/tests/CMakeLists.txt @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (C) 2023 Advanced Micro Devices, Inc. All rights reserved. +# +add_subdirectory(validate) diff --git a/src/runtime_src/core/edge/ps_kernels/xrt/tests/validate/CMakeLists.txt b/src/runtime_src/core/edge/ps_kernels/xrt/tests/validate/CMakeLists.txt new file mode 100644 index 00000000000..f105353c5f0 --- /dev/null +++ b/src/runtime_src/core/edge/ps_kernels/xrt/tests/validate/CMakeLists.txt @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (C) 2023 Advanced Micro Devices, Inc. All rights reserved. +# +set(PS_KERNEL_INSTALL_DIR "${XRT_INSTALL_LIB_DIR}/ps_kernels_lib") + +add_subdirectory(ps_aie_test) +add_subdirectory(ps_bandwidth_test) +add_subdirectory(ps_validate_test) + +install (FILES test_dependencies.json + DESTINATION ${PS_KERNEL_INSTALL_DIR} + ) diff --git a/src/runtime_src/core/edge/ps_kernels/xrt/tests/validate/ps_aie_test/CMakeLists.txt b/src/runtime_src/core/edge/ps_kernels/xrt/tests/validate/ps_aie_test/CMakeLists.txt new file mode 100644 index 00000000000..b45e07b652c --- /dev/null +++ b/src/runtime_src/core/edge/ps_kernels/xrt/tests/validate/ps_aie_test/CMakeLists.txt @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (C) 2022-2023 Advanced Micro Devices, Inc. All rights reserved. +set(PS_KERNEL_NAME "ps_aie") + +add_library(${PS_KERNEL_NAME} SHARED + "ps_aie.cpp" + ) + +set_target_properties(${PS_KERNEL_NAME} PROPERTIES + VERSION ${XRT_VERSION_STRING} + SOVERSION ${XRT_SOVERSION} + ) + +target_link_libraries(${PS_KERNEL_NAME} + PRIVATE + xrt_coreutil + xrt_core +) + +generate_ps_kernel_xclbin(${PS_KERNEL_NAME}) diff --git a/src/runtime_src/core/edge/ps_kernels/xrt/tests/validate/ps_aie_test/ps_aie.cpp b/src/runtime_src/core/edge/ps_kernels/xrt/tests/validate/ps_aie_test/ps_aie.cpp new file mode 100644 index 00000000000..1ba6b6e3abf --- /dev/null +++ b/src/runtime_src/core/edge/ps_kernels/xrt/tests/validate/ps_aie_test/ps_aie.cpp @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) 2022 Xilinx, Inc +// Copyright (C) 2023 Advanced Micro Devices, Inc. - All rights reserved + +#include "core/edge/include/sk_types.h" +#include +#include +#include +#include +#include +#include +#include + +#include "xrt/xrt_aie.h" +#include "xrt/xrt_bo.h" +#include "xrt/xrt_graph.h" +#include "xrt/xrt_kernel.h" + +#define SAMPLES 256 + +#ifdef __cplusplus +extern "C" { +#endif + +class xrtHandles : public pscontext { + public: + xrt::device dhdl; + xrt::graph graphhdl; + xrtHandles(xclDeviceHandle dhdl_in, const xuid_t xclbin_uuid) : dhdl(dhdl_in), graphhdl(dhdl, xclbin_uuid, "mm") { + graphhdl.run(); + } +}; + +__attribute__((visibility("default"))) xrtHandles* +aie_kernel_init(xclDeviceHandle dhdl, const xuid_t xclbin_uuid) { + xrtHandles* handles = new xrtHandles(dhdl, xclbin_uuid); + + return handles; +} + +__attribute__((visibility("default"))) int +aie_kernel( + float* in_boA, float* in_boB, float* out_bo, int input_size, int output_size, xrtHandles* handles) { + auto out_bohdl = xrt::aie::bo(handles->dhdl, output_size, 0, 0); + auto in_bohdlA = xrt::aie::bo(handles->dhdl, input_size, 0, 0); + auto in_bohdlB = xrt::aie::bo(handles->dhdl, input_size, 0, 0); + + auto inA_bo_map = in_bohdlA.map(); + auto inB_bo_map = in_bohdlB.map(); + auto out_bo_map = out_bohdl.map(); + + memcpy(inA_bo_map, in_boA, input_size); + memcpy(inB_bo_map, in_boB, input_size); + + std::string gmioportA, gmioportB, gmioportC; + int gmio_id = 0; + gmioportA = std::to_string(gmio_id++); + gmioportB = std::to_string(gmio_id++); + in_bohdlB.sync("in_source2", XCL_BO_SYNC_BO_GMIO_TO_AIE, input_size, 0); + in_bohdlA.sync("in_source1", XCL_BO_SYNC_BO_GMIO_TO_AIE, input_size, 0); + + gmioportC = std::to_string(gmio_id++); + out_bohdl.sync("out_sink", XCL_BO_SYNC_BO_AIE_TO_GMIO, output_size, 0); + + memcpy(out_bo, out_bo_map, output_size); + + return 0; +} + +__attribute__((visibility("default"))) int +aie_kernel_fini(xrtHandles* handles) { + std::cout << "Releasing remaining XRT objects...\n"; + handles->graphhdl.end(); + delete handles; + return 0; +} + +#ifdef __cplusplus +} +#endif diff --git a/src/runtime_src/core/edge/ps_kernels/xrt/tests/validate/ps_bandwidth_test/CMakeLists.txt b/src/runtime_src/core/edge/ps_kernels/xrt/tests/validate/ps_bandwidth_test/CMakeLists.txt new file mode 100644 index 00000000000..8666943da80 --- /dev/null +++ b/src/runtime_src/core/edge/ps_kernels/xrt/tests/validate/ps_bandwidth_test/CMakeLists.txt @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (C) 2022-2023 Advanced Micro Devices, Inc. All rights reserved. +set(PS_KERNEL_NAME "ps_bandwidth") + +add_library(${PS_KERNEL_NAME} SHARED + "ps_bandwidth.cpp" + ) + +set_target_properties(${PS_KERNEL_NAME} PROPERTIES + VERSION ${XRT_VERSION_STRING} + SOVERSION ${XRT_SOVERSION} + ) + +target_link_libraries(${PS_KERNEL_NAME} + PRIVATE + xrt_coreutil + xrt_core +) + +generate_ps_kernel_xclbin(${PS_KERNEL_NAME}) diff --git a/src/runtime_src/core/edge/ps_kernels/xrt/tests/validate/ps_bandwidth_test/ps_bandwidth.cpp b/src/runtime_src/core/edge/ps_kernels/xrt/tests/validate/ps_bandwidth_test/ps_bandwidth.cpp new file mode 100644 index 00000000000..9b5f2efa3f8 --- /dev/null +++ b/src/runtime_src/core/edge/ps_kernels/xrt/tests/validate/ps_bandwidth_test/ps_bandwidth.cpp @@ -0,0 +1,109 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "xrt/xrt_kernel.h" + +constexpr unsigned long long int operator "" _Ki(unsigned long long int n) { + return n * 1024; +} + +constexpr unsigned long long int operator "" _Mi(unsigned long long int n) { + return n * 1024 * 1024; +} + +#ifdef __cplusplus +extern "C" { +#endif +#include "core/edge/include/sk_types.h" + +// User private data structure container (context object) definition +class xrtHandles : public pscontext +{ +public: + xrt::device dhdl; + xrt::kernel bandwidth_kernel; + xrtHandles(xclDeviceHandle dhdl_in, const xuid_t xclbin_uuid) + : dhdl(dhdl_in) + , bandwidth_kernel(dhdl,xclbin_uuid,"bandwidth") + { + } +}; + + __attribute__((visibility("default"))) + pscontext *bandwidth_kernel_init(xclDeviceHandle dhdl, const xuid_t xclbin_uuid) { + xrtHandles *handles = new xrtHandles(dhdl, xclbin_uuid); + + return(handles); + } + +__attribute__((visibility("default"))) +int bandwidth_kernel(int reps, double *max_throughput, xrtHandles *xrtHandle) +{ + double max_throughput_int = 0; + + // Starting at 4K and going up to 16M with increments of power of 2 + for (uint32_t i = 4_Ki; i <= 16_Mi; i *= 2) { + unsigned int data_size = i; + + // These commands will allocate memory on the FPGA. The xrt::bo objects + // can be used to reference the memory locations on the device. + // Creating Buffers + auto input_buffer = xrt::bo(xrtHandle->dhdl, data_size, xrtHandle->bandwidth_kernel.group_id(0)); + auto output_buffer = xrt::bo(xrtHandle->dhdl, data_size, xrtHandle->bandwidth_kernel.group_id(1)); + auto input_host = input_buffer.map(); + auto output_host = output_buffer.map(); + + // Filling up memory with an incremental byte pattern + for (uint32_t j = 0; j < data_size; j++) { + input_host[j] = j % 256; + // Initializing output vectors to zero + output_host[j] = 0; + } + + auto time_start = std::chrono::high_resolution_clock::now(); + auto run = xrtHandle->bandwidth_kernel(input_buffer, output_buffer, data_size, reps); + run.wait(); + auto time_end = std::chrono::high_resolution_clock::now(); + + // check + for (uint32_t j = 0; j < data_size; j++) { + if (output_host[j] != input_host[j]) { + syslog(LOG_ERR,"%s: ERROR : kernel failed to copy entry %d, input %d output %d\n",__func__, j, input_host[j], output_host[j]); + return EXIT_FAILURE; + } + } + + double usduration = + (double)(std::chrono::duration_cast(time_end - time_start).count() / reps); + + double dnsduration = (double)usduration; + double dsduration = dnsduration / ((double)1000000000); // Convert the duration from nanoseconds to seconds + double bpersec = (data_size * 1) / dsduration; + double mbpersec = (2 * bpersec) / ((double)1024 * 1024); // Convert b/sec to mb/sec + + if (mbpersec > max_throughput_int) max_throughput_int = mbpersec; + syslog(LOG_INFO, "%s: Throughput : %f MB/s\n",__func__,mbpersec); + } + + syslog(LOG_INFO, "%s: Throughput (Type: DDR) = %f MB/s\n",__func__,max_throughput_int); + max_throughput[0] = max_throughput_int; + + return 0; +} + + +__attribute__((visibility("default"))) +int bandwidth_kernel_fini(xrtHandles *handles) { + delete handles; + return 0; +} + +#ifdef __cplusplus +} +#endif diff --git a/src/runtime_src/core/edge/ps_kernels/xrt/tests/validate/ps_validate_test/CMakeLists.txt b/src/runtime_src/core/edge/ps_kernels/xrt/tests/validate/ps_validate_test/CMakeLists.txt new file mode 100644 index 00000000000..9db1ea7d051 --- /dev/null +++ b/src/runtime_src/core/edge/ps_kernels/xrt/tests/validate/ps_validate_test/CMakeLists.txt @@ -0,0 +1,20 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (C) 2022-2023 Advanced Micro Devices, Inc. All rights reserved. +set(PS_KERNEL_NAME "ps_validate") + +add_library(${PS_KERNEL_NAME} SHARED + "ps_validate.cpp" + ) + +set_target_properties(${PS_KERNEL_NAME} PROPERTIES + VERSION ${XRT_VERSION_STRING} + SOVERSION ${XRT_SOVERSION} + ) + +target_link_libraries(${PS_KERNEL_NAME} + PRIVATE + xrt_coreutil + xrt_core +) + +generate_ps_kernel_xclbin(${PS_KERNEL_NAME}) diff --git a/src/runtime_src/core/edge/ps_kernels/xrt/tests/validate/ps_validate_test/ps_validate.cpp b/src/runtime_src/core/edge/ps_kernels/xrt/tests/validate/ps_validate_test/ps_validate.cpp new file mode 100644 index 00000000000..1c26c2621dc --- /dev/null +++ b/src/runtime_src/core/edge/ps_kernels/xrt/tests/validate/ps_validate_test/ps_validate.cpp @@ -0,0 +1,27 @@ +#include "xrt/xrt_kernel.h" + +#include + +#include "core/edge/include/sk_types.h" + +// User private data structure container (context object) definition +class xrtHandles : public pscontext +{ +public: +}; + +#ifdef __cplusplus +extern "C" { +#endif + +__attribute__((visibility("default"))) +int hello_world(int *input, int *output, int count, struct xrtHandles *xrtHandle) +{ + memcpy(output, input, count*sizeof(int)); + + return 0; +} + +#ifdef __cplusplus +} +#endif diff --git a/src/runtime_src/core/edge/ps_kernels/xrt/tests/validate/test_dependencies.json b/src/runtime_src/core/edge/ps_kernels/xrt/tests/validate/test_dependencies.json new file mode 100644 index 00000000000..65706b454b9 --- /dev/null +++ b/src/runtime_src/core/edge/ps_kernels/xrt/tests/validate/test_dependencies.json @@ -0,0 +1,16 @@ +{ + "ps_kernel_mappings": [ + { + "name": "ps_aie.xclbin", + "dependencies": ["ps_aie.xclbin"] + }, + { + "name": "ps_bandwidth.xclbin", + "dependencies": ["ps_bandwidth.xclbin"] + }, + { + "name": "ps_validate.xclbin", + "dependencies": [] + } + ] +} diff --git a/src/runtime_src/tools/scripts/pkgapu.sh b/src/runtime_src/tools/scripts/pkgapu.sh index d9fa4df226c..e412a5401e8 100755 --- a/src/runtime_src/tools/scripts/pkgapu.sh +++ b/src/runtime_src/tools/scripts/pkgapu.sh @@ -1,6 +1,7 @@ #!/bin/bash # # Copyright (C) 2021-2022 Xilinx, Inc. All rights reserved. +# Copyright (C) 2023 Advanced Micro Devices, Inc. All rights reserved. # # This script creates rpm and deb packages for Versal APU firmware and Built-in PS Kernels. @@ -166,6 +167,10 @@ FW_FILE="$BUILD_DIR/lib/firmware/xilinx/xrt-versal-apu.xsabin" INSTALL_ROOT="$BUILD_DIR/lib" SDK="$BUILD_DIR/lib/firmware/xilinx/sysroot/sdk.sh" +THIS_SCRIPT_DIR="$( cd "$( dirname "${THIS_SCRIPT}" )" >/dev/null 2>&1 && pwd )" +VITIS_FILE="${THIS_SCRIPT_DIR}/vitis.build" +source $VITIS_FILE + if [[ $clean == 1 ]]; then echo $PWD echo "/bin/rm -rf $BUILD_DIR" @@ -177,7 +182,6 @@ if [[ ! -d $IMAGES_DIR ]]; then error "Please specify the valid path of APU images by -images" fi IMAGES_DIR=`realpath $IMAGES_DIR` -source /proj/xbuilds/2022.2_released/installs/lin64/Vitis/2022.2/settings64.sh if [[ ! (`which mkimage` && `which bootgen` && `which xclbinutil`) ]]; then @@ -298,27 +302,29 @@ if [ -e $IMAGES_DIR/sdk.sh ]; then echo "sdk.sh exists copy it to apu package" cp $IMAGES_DIR/sdk.sh $SDK fi -# Generate PS Kernel xclbin -# Hardcoding the ps kernel xclbin name to ps_kernels.xclbin -# We can create one xclbin per PS Kernel also based on future requirements -PS_KERNELS_XCLBIN="$BUILD_DIR/lib/firmware/xilinx/ps_kernels/ps_kernels.xclbin" + +# Output the required xclbins for validating and operating the APU #Extract ps_kernels_lib from rootfs tar tar -C $BUILD_DIR -xf $IMAGES_DIR/rootfs.tar.gz ./usr/lib/ps_kernels_lib if [ $? -eq 0 ]; then + # Generate the output directory + PS_KERNELS_XCLBIN_PATH="$BUILD_DIR/lib/firmware/xilinx/ps_kernels" + mkdir -p $PS_KERNELS_XCLBIN_PATH + # Copy over PS kernel xclbins built by XRT PS_KERNEL_DIR=$BUILD_DIR/usr/lib/ps_kernels_lib - ps_kernel_command="xclbinutil --output $PS_KERNELS_XCLBIN " - pk_exists=0 - for entry in "$PS_KERNEL_DIR"/*.so + + # Extract generated xclbins here + # This handles the xclbins required for device validation + for entry in "$PS_KERNEL_DIR"/*.xclbin + do + cp $entry $PS_KERNELS_XCLBIN_PATH + done + + # Copy the configuration json files for PS kernels + for entry in "$PS_KERNEL_DIR"/*.json do - pk_exists=1 - ps_kernel_command+=" --add-pskernel $entry" + cp $entry $PS_KERNELS_XCLBIN_PATH done - # Generate xclbin if atleast one PS Kernel exists - if [ $pk_exists -eq 1 ]; then - # Run final xclbinutil command to generate the PS Kernels xclbin - echo "Running $ps_kernel_command" - $ps_kernel_command - fi fi dodeb $INSTALL_ROOT