Vitis 6327 Add PS kernel xclbins into APU Package

build/build_edge.sh

@@ -45,6 +45,7 @@ install_recipes()
     if [ $? != 0 ]; then
         echo "inherit externalsrc" > $XRT_BB
         echo "EXTERNALSRC = \"$XRT_REPO_DIR/src\"" >> $XRT_BB
+        echo "EXTRA_OECMAKE += \"-DMY_VITIS=$XILINX_VITIS\"" >> $XRT_BB
         echo 'EXTERNALSRC_BUILD = "${WORKDIR}/build"' >> $XRT_BB
         echo 'PACKAGE_CLASSES = "package_rpm"' >> $XRT_BB
         echo 'LICENSE = "GPLv2 & Apache-2.0"' >> $XRT_BB
@@ -307,7 +308,10 @@ fi
 if [ -f $SETTINGS_FILE ]; then
     source $SETTINGS_FILE
 fi
-source $PETALINUX/settings.sh 
+source $PETALINUX/settings.sh
+
+VITIS_FILE="${THIS_SCRIPT_DIR}/vitis.build"
+source $VITIS_FILE
 
 if [[ $AARCH = $aarch64_dir ]]; then
     if [[ -f $PETALINUX/../../bsp/release/zynqmp-common-v$PETALINUX_VER-final.bsp ]]; then

build/vitis.build

@@ -0,0 +1 @@
+source /proj/xbuilds/2023.2_daily_latest/installs/lin64/Vitis/2023.2/settings64.sh

src/runtime_src/core/edge/ps_kernels/CMakeLists.txt

@@ -1,9 +1,34 @@
 # SPDX-License-Identifier: Apache-2.0
 # Copyright (C) 2019-2022 Xilinx, Inc. All rights reserved.
-# Copyright (C) 2022 Advanced Micro Devices, Inc. All rights reserved.
-if (DEFINED XRT_AIE_BUILD)
-  add_subdirectory(sample)
-  add_subdirectory(profiling)
-endif()
+# Copyright (C) 2022-2023 Advanced Micro Devices, Inc. All rights reserved.
+set(PS_KERNEL_INSTALL_DIR "${XRT_INSTALL_LIB_DIR}/ps_kernels_lib")
+
+function(generate_ps_kernel_xclbin PS_KERNEL_NAME)
+  set(XCLBIN_NAME "${PS_KERNEL_NAME}.xclbin")
+  set(XCLBIN_TARGET "${PS_KERNEL_NAME}_xclbin")
+  set(PS_KERNEL_LOCATION "lib${PS_KERNEL_NAME}.so")
+  set(XCLBIN_LOCATION "${CMAKE_CURRENT_BINARY_DIR}/${XCLBIN_NAME}")
+
+  add_custom_command(
+    OUTPUT ${XCLBIN_LOCATION}
+    COMMAND "${MY_VITIS}/bin/xclbinutil" --output ${XCLBIN_LOCATION} --add-pskernel ${PS_KERNEL_LOCATION} --force
+    DEPENDS ${PS_KERNEL_NAME}
+    )
+
+  add_custom_target(${XCLBIN_TARGET} ALL
+    DEPENDS ${XCLBIN_LOCATION}
+    )
 
-add_subdirectory(xrt)
+  install (FILES ${XCLBIN_LOCATION}
+    DESTINATION ${PS_KERNEL_INSTALL_DIR}
+    )
+endfunction()
+
+if (DEFINED MY_VITIS)
+  if (DEFINED XRT_AIE_BUILD)
+    add_subdirectory(sample)
+    add_subdirectory(profiling)
+  endif()
+
+  add_subdirectory(xrt)
+endif()

src/runtime_src/core/edge/ps_kernels/xrt/CMakeLists.txt

@@ -1,4 +1,5 @@
 # SPDX-License-Identifier: Apache-2.0
-# Copyright (C) 2022 Advanced Micro Devices, Inc. All rights reserved.
+# Copyright (C) 2022-2023 Advanced Micro Devices, Inc. All rights reserved.
 #
 add_subdirectory(instance_query)
+add_subdirectory(tests)

src/runtime_src/core/edge/ps_kernels/xrt/instance_query/CMakeLists.txt

@@ -3,15 +3,20 @@
 #
 set(PS_KERNEL_INSTALL_DIR "${XRT_INSTALL_LIB_DIR}/ps_kernels_lib")
 
-add_library(instance_query OBJECT
-  instance_query.cpp
+add_library(instance_query SHARED
+  "instance_query.cpp"
   )
 
 set_target_properties(instance_query PROPERTIES 
   VERSION ${XRT_VERSION_STRING}
   SOVERSION ${XRT_SOVERSION}
   )
 
+target_link_libraries(instance_query
+  PRIVATE
+  ${Boost_FILESYSTEM_LIBRARY}
+  )
+
 install (TARGETS instance_query 
   EXPORT xrt-targets
   LIBRARY DESTINATION ${PS_KERNEL_INSTALL_DIR}

src/runtime_src/core/edge/ps_kernels/xrt/tests/CMakeLists.txt

@@ -0,0 +1,4 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (C) 2023 Advanced Micro Devices, Inc. All rights reserved.
+#
+add_subdirectory(validate)

src/runtime_src/core/edge/ps_kernels/xrt/tests/validate/CMakeLists.txt

@@ -0,0 +1,12 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (C) 2023 Advanced Micro Devices, Inc. All rights reserved.
+#
+set(PS_KERNEL_INSTALL_DIR "${XRT_INSTALL_LIB_DIR}/ps_kernels_lib")
+
+add_subdirectory(ps_aie_test)
+add_subdirectory(ps_bandwidth_test)
+add_subdirectory(ps_validate_test)
+
+install (FILES test_dependencies.json
+  DESTINATION ${PS_KERNEL_INSTALL_DIR}
+  )

src/runtime_src/core/edge/ps_kernels/xrt/tests/validate/ps_aie_test/CMakeLists.txt

@@ -0,0 +1,20 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (C) 2022-2023 Advanced Micro Devices, Inc. All rights reserved.
+set(PS_KERNEL_NAME "ps_aie")
+
+add_library(${PS_KERNEL_NAME} SHARED
+  "ps_aie.cpp"
+  )
+
+set_target_properties(${PS_KERNEL_NAME} PROPERTIES 
+  VERSION ${XRT_VERSION_STRING}
+  SOVERSION ${XRT_SOVERSION}
+  )
+
+target_link_libraries(${PS_KERNEL_NAME}
+  PRIVATE
+  xrt_coreutil
+  xrt_core
+)
+
+generate_ps_kernel_xclbin(${PS_KERNEL_NAME})

src/runtime_src/core/edge/ps_kernels/xrt/tests/validate/ps_aie_test/ps_aie.cpp

@@ -0,0 +1,80 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright (C) 2022 Xilinx, Inc
+// Copyright (C) 2023 Advanced Micro Devices, Inc. - All rights reserved
+
+#include "core/edge/include/sk_types.h"
+#include <cstring>
+#include <fstream>
+#include <iostream>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <vector>
+
+#include "xrt/xrt_aie.h"
+#include "xrt/xrt_bo.h"
+#include "xrt/xrt_graph.h"
+#include "xrt/xrt_kernel.h"
+
+#define SAMPLES 256
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+class xrtHandles : public pscontext {
+   public:
+    xrt::device dhdl;
+    xrt::graph graphhdl;
+    xrtHandles(xclDeviceHandle dhdl_in, const xuid_t xclbin_uuid) : dhdl(dhdl_in), graphhdl(dhdl, xclbin_uuid, "mm") {
+        graphhdl.run();
+    }
+};
+
+__attribute__((visibility("default"))) xrtHandles*
+aie_kernel_init(xclDeviceHandle dhdl, const xuid_t xclbin_uuid) {
+    xrtHandles* handles = new xrtHandles(dhdl, xclbin_uuid);
+
+    return handles;
+}
+
+__attribute__((visibility("default"))) int
+aie_kernel(
+    float* in_boA, float* in_boB, float* out_bo, int input_size, int output_size, xrtHandles* handles) {
+    auto out_bohdl = xrt::aie::bo(handles->dhdl, output_size, 0, 0);
+    auto in_bohdlA = xrt::aie::bo(handles->dhdl, input_size, 0, 0);
+    auto in_bohdlB = xrt::aie::bo(handles->dhdl, input_size, 0, 0);
+
+    auto inA_bo_map = in_bohdlA.map<float*>();
+    auto inB_bo_map = in_bohdlB.map<float*>();
+    auto out_bo_map = out_bohdl.map<float*>();
+
+    memcpy(inA_bo_map, in_boA, input_size);
+    memcpy(inB_bo_map, in_boB, input_size);
+
+    std::string gmioportA, gmioportB, gmioportC;
+    int gmio_id = 0;
+    gmioportA = std::to_string(gmio_id++);
+    gmioportB = std::to_string(gmio_id++);
+    in_bohdlB.sync("in_source2", XCL_BO_SYNC_BO_GMIO_TO_AIE, input_size, 0);
+    in_bohdlA.sync("in_source1", XCL_BO_SYNC_BO_GMIO_TO_AIE, input_size, 0);
+
+    gmioportC = std::to_string(gmio_id++);
+    out_bohdl.sync("out_sink", XCL_BO_SYNC_BO_AIE_TO_GMIO, output_size, 0);
+
+    memcpy(out_bo, out_bo_map, output_size);
+
+    return 0;
+}
+
+__attribute__((visibility("default"))) int
+aie_kernel_fini(xrtHandles* handles) {
+    std::cout << "Releasing remaining XRT objects...\n";
+    handles->graphhdl.end();
+    delete handles;
+    return 0;
+}
+
+#ifdef __cplusplus
+}
+#endif

src/runtime_src/core/edge/ps_kernels/xrt/tests/validate/ps_bandwidth_test/CMakeLists.txt

@@ -0,0 +1,20 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (C) 2022-2023 Advanced Micro Devices, Inc. All rights reserved.
+set(PS_KERNEL_NAME "ps_bandwidth")
+
+add_library(${PS_KERNEL_NAME} SHARED
+  "ps_bandwidth.cpp"
+  )
+
+set_target_properties(${PS_KERNEL_NAME} PROPERTIES 
+  VERSION ${XRT_VERSION_STRING}
+  SOVERSION ${XRT_SOVERSION}
+  )
+
+target_link_libraries(${PS_KERNEL_NAME}
+  PRIVATE
+  xrt_coreutil
+  xrt_core
+)
+
+generate_ps_kernel_xclbin(${PS_KERNEL_NAME})

src/runtime_src/core/edge/ps_kernels/xrt/tests/validate/ps_bandwidth_test/ps_bandwidth.cpp

@@ -0,0 +1,109 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+#include <sys/types.h>
+#include <syslog.h>
+#include <sys/mman.h>
+
+#include "xrt/xrt_kernel.h"
+
+constexpr unsigned long long int operator "" _Ki(unsigned long long int n) {
+  return n * 1024;
+}
+
+constexpr unsigned long long int operator "" _Mi(unsigned long long int n) {
+  return n * 1024 * 1024;
+}
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+#include "core/edge/include/sk_types.h"
+
+// User private data structure container (context object) definition
+class xrtHandles : public pscontext
+{
+public:
+  xrt::device dhdl;
+  xrt::kernel bandwidth_kernel;
+  xrtHandles(xclDeviceHandle dhdl_in, const xuid_t xclbin_uuid)
+    : dhdl(dhdl_in)
+    , bandwidth_kernel(dhdl,xclbin_uuid,"bandwidth")
+  {
+  }
+};
+
+  __attribute__((visibility("default")))
+  pscontext *bandwidth_kernel_init(xclDeviceHandle dhdl, const xuid_t xclbin_uuid) {
+    xrtHandles *handles = new xrtHandles(dhdl, xclbin_uuid);
+
+    return(handles);
+  }
+
+__attribute__((visibility("default")))
+int bandwidth_kernel(int reps, double *max_throughput, xrtHandles *xrtHandle)
+{
+  double max_throughput_int = 0;
+
+  // Starting at 4K and going up to 16M with increments of power of 2
+  for (uint32_t i = 4_Ki; i <= 16_Mi; i *= 2) {
+    unsigned int data_size = i;
+
+    // These commands will allocate memory on the FPGA. The xrt::bo objects
+    // can be used to reference the memory locations on the device.
+    // Creating Buffers
+    auto input_buffer = xrt::bo(xrtHandle->dhdl, data_size, xrtHandle->bandwidth_kernel.group_id(0));
+    auto output_buffer = xrt::bo(xrtHandle->dhdl, data_size, xrtHandle->bandwidth_kernel.group_id(1));
+    auto input_host = input_buffer.map<unsigned char*>();
+    auto output_host = output_buffer.map<unsigned char*>();
+
+    // Filling up memory with an incremental byte pattern
+    for (uint32_t j = 0; j < data_size; j++) {
+      input_host[j] = j % 256;
+      // Initializing output vectors to zero
+      output_host[j] = 0;
+    }
+
+    auto time_start = std::chrono::high_resolution_clock::now();
+    auto run = xrtHandle->bandwidth_kernel(input_buffer, output_buffer, data_size, reps);
+    run.wait();
+    auto time_end = std::chrono::high_resolution_clock::now();
+
+    // check
+    for (uint32_t j = 0; j < data_size; j++) {
+      if (output_host[j] != input_host[j]) {
+        syslog(LOG_ERR,"%s: ERROR : kernel failed to copy entry %d, input %d output %d\n",__func__, j, input_host[j], output_host[j]);
+        return EXIT_FAILURE;
+      }
+    }
+
+    double usduration =
+    (double)(std::chrono::duration_cast<std::chrono::nanoseconds>(time_end - time_start).count() / reps);
+
+    double dnsduration = (double)usduration;
+    double dsduration = dnsduration / ((double)1000000000); // Convert the duration from nanoseconds to seconds
+    double bpersec = (data_size * 1) / dsduration;
+    double mbpersec = (2 * bpersec) / ((double)1024 * 1024); // Convert b/sec to mb/sec
+
+    if (mbpersec > max_throughput_int) max_throughput_int = mbpersec;
+    syslog(LOG_INFO, "%s: Throughput : %f MB/s\n",__func__,mbpersec);
+  }
+
+  syslog(LOG_INFO, "%s: Throughput (Type: DDR) = %f MB/s\n",__func__,max_throughput_int);
+  max_throughput[0] = max_throughput_int;
+
+  return 0;
+}
+
+
+__attribute__((visibility("default")))
+int bandwidth_kernel_fini(xrtHandles *handles) {
+  delete handles;
+  return 0;
+}
+
+#ifdef __cplusplus
+}
+#endif

src/runtime_src/core/edge/ps_kernels/xrt/tests/validate/ps_validate_test/CMakeLists.txt

@@ -0,0 +1,20 @@
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (C) 2022-2023 Advanced Micro Devices, Inc. All rights reserved.
+set(PS_KERNEL_NAME "ps_validate")
+
+add_library(${PS_KERNEL_NAME} SHARED
+  "ps_validate.cpp"
+  )
+
+set_target_properties(${PS_KERNEL_NAME} PROPERTIES 
+  VERSION ${XRT_VERSION_STRING}
+  SOVERSION ${XRT_SOVERSION}
+  )
+
+target_link_libraries(${PS_KERNEL_NAME}
+  PRIVATE
+  xrt_coreutil
+  xrt_core
+)
+
+generate_ps_kernel_xclbin(${PS_KERNEL_NAME})

src/runtime_src/core/edge/ps_kernels/xrt/tests/validate/ps_validate_test/ps_validate.cpp

@@ -0,0 +1,27 @@
+#include "xrt/xrt_kernel.h"
+
+#include <cstring>
+
+#include "core/edge/include/sk_types.h"
+
+// User private data structure container (context object) definition
+class xrtHandles : public pscontext
+{
+public:
+};
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+__attribute__((visibility("default")))
+int hello_world(int *input, int *output, int count, struct xrtHandles *xrtHandle)
+{
+    memcpy(output, input, count*sizeof(int));
+
+    return 0;
+}
+
+#ifdef __cplusplus
+}
+#endif