Revert "squish"

This reverts commit c9f4c7a.

README.md

@@ -47,11 +47,11 @@ Build and install `xdna-driver`, use commit `929e8ab`:
 git clone [email protected]:amd/xdna-driver.git
 cd <root-of-source-tree>
 # get code for submodules
-git checkout 59f1d62
+git checkout 929e8ab
 git submodule update --init --recursive
 ```
 
-Follow the instructions to build and install the driver module: [xdna-driver](https://github.com/amd/xdna-driver/tree/59f1d6235334499b22dbd056a60ab00bfec142ee).
+Follow the instructions to build and install the driver module: [xdna-driver](https://github.com/amd/xdna-driver/tree/929e8ab459cab5915631849b9f1ef9a4982d1c11).
 
 ## Building (along with IREE)
 

build_tools/ci/amdxdna_driver_utils/amdxdna_accel.py

@@ -129,9 +129,9 @@ class Union(ctypes.Union, AsDictMixin):
     4: "DRM_AMDXDNA_GET_BO_INFO",
     5: "DRM_AMDXDNA_SYNC_BO",
     6: "DRM_AMDXDNA_EXEC_CMD",
-    7: "DRM_AMDXDNA_WAIT_CMD",
-    8: "DRM_AMDXDNA_GET_INFO",
-    9: "DRM_AMDXDNA_SET_STATE",
+    7: "DRM_AMDXDNA_GET_INFO",
+    8: "DRM_AMDXDNA_SET_STATE",
+    9: "DRM_AMDXDNA_WAIT_CMD",
     10: "DRM_AMDXDNA_SUBMIT_WAIT",
     11: "DRM_AMDXDNA_SUBMIT_SIGNAL",
     12: "DRM_AMDXDNA_NUM_IOCTLS",
@@ -143,9 +143,9 @@ class Union(ctypes.Union, AsDictMixin):
 DRM_AMDXDNA_GET_BO_INFO = 4
 DRM_AMDXDNA_SYNC_BO = 5
 DRM_AMDXDNA_EXEC_CMD = 6
-DRM_AMDXDNA_WAIT_CMD = 7
-DRM_AMDXDNA_GET_INFO = 8
-DRM_AMDXDNA_SET_STATE = 9
+DRM_AMDXDNA_GET_INFO = 7
+DRM_AMDXDNA_SET_STATE = 8
+DRM_AMDXDNA_WAIT_CMD = 9
 DRM_AMDXDNA_SUBMIT_WAIT = 10
 DRM_AMDXDNA_SUBMIT_SIGNAL = 11
 DRM_AMDXDNA_NUM_IOCTLS = 12

runtime/src/iree-amd-aie/driver/xrt-lite/shim/linux/kmq/amdxdna_accel.h

@@ -45,9 +45,9 @@ enum amdxdna_drm_ioctl_id {
   DRM_AMDXDNA_GET_BO_INFO,
   DRM_AMDXDNA_SYNC_BO,
   DRM_AMDXDNA_EXEC_CMD,
-  DRM_AMDXDNA_WAIT_CMD,
   DRM_AMDXDNA_GET_INFO,
   DRM_AMDXDNA_SET_STATE,
+  DRM_AMDXDNA_WAIT_CMD,
   DRM_AMDXDNA_NUM_IOCTLS
 };
 
@@ -57,6 +57,23 @@ enum amdxdna_device_type {
   AMDXDNA_DEV_TYPE_UMQ,
 };
 
+/*
+ * Enum for priority in application's QoS. Values copied from Window shim layer.
+ * AMDXDNA_QOS_DEFAULT_PRIORITY: Default priority.
+ * AMDXDNA_QOS_REALTIME_PRIORITY: Real time clients.
+ * AMDXDNA_QOS_HIGH_PRIORITY: Best effort foreground clients.
+ * AMDXDNA_QOS_NORMAL_PRIORITY: Best effort or background clients.
+ * AMDXDNA_QOS_LOW_PRIORITY: Clients that can wait indefinite amount of time for
+ *                           completion.
+ */
+enum amdxdna_qos_priority {
+  AMDXDNA_QOS_DEFAULT_PRIORITY = 0x0,
+  AMDXDNA_QOS_REALTIME_PRIORITY = 0x100,
+  AMDXDNA_QOS_HIGH_PRIORITY = 0x180,
+  AMDXDNA_QOS_NORMAL_PRIORITY = 0x200,
+  AMDXDNA_QOS_LOW_PRIORITY = 0x280
+};
+
 /**
  * struct qos_info - QoS information for driver.
  * @gops: Giga operations per second.
@@ -89,6 +106,8 @@ struct amdxdna_qos_info {
  * @mem_size: Size of AIE tile memory.
  * @umq_doorbell: Returned offset of doorbell associated with UMQ.
  * @handle: Returned hardware context handle.
+ * @syncobj_handle: The drm timeline syncobj handle for command completion
+ * notification.
  * @pad: Structure padding.
  */
 struct amdxdna_drm_create_hwctx {
@@ -102,6 +121,7 @@ struct amdxdna_drm_create_hwctx {
   __u32 mem_size;
   __u32 umq_doorbell;
   __u32 handle;
+  __u32 syncobj_handle;
   __u32 pad;
 };
 
@@ -156,7 +176,7 @@ enum amdxdna_drm_config_hwctx_param {
  * @param_val_size: Size of the parameter buffer pointed to by the param_val.
  *		    If param_val is not a pointer, driver can ignore this.
  * @pad: Structure padding.
- * 
+ *
  * Note: if the param_val is a pointer pointing to a buffer, the maximum size
  * of the buffer is 4KiB(PAGE_SIZE).
  */
@@ -493,6 +513,17 @@ struct amdxdna_drm_query_firmware_version {
   __u32 build; /* out */
 };
 
+/**
+ * struct amdxdna_drm_get_force_preempt_state - Get force preemption state.
+ * @force_preempt_state: 1 implies force preemption is enabled.
+ *                       0 implies disabled.
+ * @pad: MBZ.
+ */
+struct amdxdna_drm_get_force_preempt_state {
+  __u8 state;
+  __u8 pad[7];
+};
+
 enum amdxdna_drm_get_param {
   DRM_AMDXDNA_QUERY_AIE_STATUS,
   DRM_AMDXDNA_QUERY_AIE_METADATA,
@@ -505,6 +536,7 @@ enum amdxdna_drm_get_param {
   DRM_AMDXDNA_QUERY_FIRMWARE_VERSION,
   DRM_AMDXDNA_GET_POWER_MODE,
   DRM_AMDXDNA_QUERY_TELEMETRY,
+  DRM_AMDXDNA_GET_FORCE_PREEMPT_STATE,
   DRM_AMDXDNA_NUM_GET_PARAM,
 };
 
@@ -531,10 +563,22 @@ struct amdxdna_drm_set_power_mode {
   __u8 pad[7];
 };
 
+/**
+ * struct amdxdna_drm_set_force_preempt_state - set force preemption state
+ * @force_preempt_state: 1 implies force preemption is enabled.
+ *                       0 implies disabled
+ * @pad: MBZ.
+ */
+struct amdxdna_drm_set_force_preempt_state {
+  __u8 state;
+  __u8 pad[7];
+};
+
 enum amdxdna_drm_set_param {
   DRM_AMDXDNA_SET_POWER_MODE,
   DRM_AMDXDNA_WRITE_AIE_MEM,
   DRM_AMDXDNA_WRITE_AIE_REG,
+  DRM_AMDXDNA_SET_FORCE_PREEMPT,
   DRM_AMDXDNA_NUM_SET_PARAM,
 };
 

runtime/src/iree-amd-aie/driver/xrt-lite/shim/linux/kmq/bo.cpp

@@ -464,7 +464,7 @@ uint32_t bo::get_arg_bo_handles(uint32_t *handles, size_t num) const {
     shim_err(E2BIG, "There are %ld BO args, provided buffer can hold only %ld",
              sz, num);
 
-  for (auto m : m_args_map) *(handles++) = m.second;
+  for (auto &m : m_args_map) *(handles++) = m.second;
 
   return sz;
 }

runtime/src/iree-amd-aie/driver/xrt-lite/shim/linux/kmq/hwctx.cpp

@@ -21,6 +21,7 @@ hw_ctx::hw_ctx(device &dev, const std::map<std::string, uint32_t> &qos,
       m_num_rows(n_rows),
       m_num_cols(n_cols),
       m_doorbell(0),
+      m_syncobj(AMDXDNA_INVALID_FENCE_HANDLE),
       m_log_buf(nullptr) {
   SHIM_DEBUG("Creating HW context...");
 
@@ -92,6 +93,7 @@ hw_ctx::hw_ctx(device &device, const std::vector<uint8_t> &pdi,
 
 hw_ctx::~hw_ctx() {
   delete_ctx_on_device();
+  delete_syncobj();
   SHIM_DEBUG("Destroyed HW context (%d)...", m_handle);
   SHIM_DEBUG("Destroying KMQ HW context (%d)...", m_handle);
 }
@@ -134,6 +136,7 @@ void hw_ctx::create_ctx_on_device() {
 
   m_handle = arg.handle;
   m_doorbell = arg.umq_doorbell;
+  m_syncobj = arg.syncobj_handle;
 
   m_q->bind_hwctx(this);
 }
@@ -149,17 +152,40 @@ void hw_ctx::delete_ctx_on_device() const {
   fini_log_buf();
 }
 
+void hw_ctx::delete_syncobj() const {
+  if (m_syncobj == AMDXDNA_INVALID_FENCE_HANDLE) return;
+  drm_syncobj_destroy dsobj = {.handle = m_syncobj};
+  m_device.get_pdev().ioctl(DRM_IOCTL_SYNCOBJ_DESTROY, &dsobj);
+}
+
 void hw_ctx::init_log_buf() {
-  auto log_buf_size = m_num_cols * 1024;
+  size_t column_size = 1024;
+  auto log_buf_size = m_num_cols * column_size + sizeof(m_metadata);
   shim_xcl_bo_flags f;
   f.flags = XCL_BO_FLAGS_EXECBUF;
   m_log_bo = alloc_bo(log_buf_size, f);
   m_log_buf = m_log_bo->map();
+  uint64_t bo_paddr = m_log_bo->get_properties().paddr;
+  set_metadata(m_num_cols, column_size, bo_paddr, 1);
   std::memset(m_log_buf, 0, log_buf_size);
+  std::memcpy(m_log_buf, &m_metadata, sizeof(m_metadata));
 }
 
 void hw_ctx::fini_log_buf() const {
   if (m_log_bo) m_log_bo->unmap(m_log_buf);
 }
 
+void hw_ctx::set_metadata(int num_cols, size_t size, uint64_t bo_paddr,
+                          uint8_t flag) {
+  m_metadata.magic_no = CERT_MAGIC_NO;
+  m_metadata.major = 0;
+  m_metadata.minor = 1;
+  m_metadata.cert_log_flag = flag;
+  m_metadata.num_cols = num_cols;
+  for (int i = 0; i < num_cols; i++) {
+    m_metadata.col_paddr[i] = bo_paddr + size * i + sizeof(m_metadata);
+    m_metadata.col_size[i] = size;
+  }
+}
+
 }  // namespace shim_xdna

runtime/src/iree-amd-aie/driver/xrt-lite/shim/linux/kmq/hwctx.h

@@ -14,6 +14,19 @@ struct hw_q;
 struct bo;
 struct device;
 
+enum cert_log_flag { debug_buffer = 0, trace_buffer };
+
+struct cert_log_metadata {
+#define CERT_MAGIC_NO 0x43455254  // "CERT"
+  uint32_t magic_no;
+  uint8_t major;
+  uint8_t minor;
+  uint8_t cert_log_flag;
+  uint8_t num_cols;       // how many valid cols, up to 8 for now
+  uint64_t col_paddr[8];  // device accessible address array for each valid col
+  uint32_t col_size[8];   // bo size for each valid col
+};
+
 struct cu_info {
   std::string m_name;
   size_t m_func;
@@ -40,11 +53,13 @@ struct hw_ctx {
   uint32_t m_handle = AMDXDNA_INVALID_CTX_HANDLE;
   amdxdna_qos_info m_qos = {};
   std::vector<cu_info> m_cu_info;
+  cert_log_metadata m_metadata;
   std::unique_ptr<hw_q> m_q;
   uint32_t m_ops_per_cycle;
   uint32_t m_num_rows;
   uint32_t m_num_cols;
   uint32_t m_doorbell;
+  uint32_t m_syncobj;
   std::unique_ptr<bo> m_log_bo;
   void *m_log_buf;
   std::vector<std::unique_ptr<bo>> m_pdi_bos;
@@ -68,8 +83,11 @@ struct hw_ctx {
   void init_log_buf();
   void fini_log_buf() const;
   void delete_ctx_on_device() const;
+  void delete_syncobj() const;
 
   hw_q *get_hw_queue() const;
+
+  void set_metadata(int num_cols, size_t size, uint64_t bo_paddr, uint8_t flag);
 };
 
 }  // namespace shim_xdna

runtime/src/iree-amd-aie/driver/xrt-lite/shim/linux/kmq/hwq.cpp

@@ -12,6 +12,12 @@
 
 namespace {
 
+uint64_t abs_now_ns() {
+  auto now = std::chrono::high_resolution_clock::now();
+  auto now_ns = std::chrono::time_point_cast<std::chrono::nanoseconds>(now);
+  return now_ns.time_since_epoch().count();
+}
+
 ert_packet *get_chained_command_pkt(shim_xdna::bo *boh) {
   ert_packet *cmdpkt = reinterpret_cast<ert_packet *>(boh->map());
   return cmdpkt->opcode == ERT_CMD_CHAIN ? cmdpkt : nullptr;
@@ -21,22 +27,47 @@ int wait_cmd(const shim_xdna::pdev &pdev, const shim_xdna::hw_ctx *ctx,
              shim_xdna::bo *cmd, uint32_t timeout_ms) {
   int ret = 1;
   auto id = cmd->get_cmd_id();
+  uint32_t syncobj = ctx->m_syncobj;
 
   SHIM_DEBUG("Waiting for cmd (%ld)...", id);
 
-  amdxdna_drm_wait_cmd wcmd = {
-      .hwctx = ctx->m_handle,
-      .timeout = timeout_ms,
-      .seq = id,
-  };
-
-  if (::ioctl(pdev.m_dev_fd, DRM_IOCTL_AMDXDNA_WAIT_CMD, &wcmd) == -1) {
-    if (errno == ETIME) {
-      ret = 0;
-    } else {
-      shim_xdna::shim_err(errno, "DRM_IOCTL_AMDXDNA_WAIT_CMD IOCTL failed");
+  if (syncobj != AMDXDNA_INVALID_FENCE_HANDLE) {
+    int64_t timeout = std::numeric_limits<int64_t>::max();
+    if (timeout_ms) {
+      timeout = timeout_ms;
+      timeout *= 1000000;
+      timeout += abs_now_ns();
+    }
+    drm_syncobj_timeline_wait wsobj = {
+        .handles = reinterpret_cast<uintptr_t>(&syncobj),
+        .points = reinterpret_cast<uintptr_t>(&id),
+        .timeout_nsec = timeout,
+        .count_handles = 1,
+        .flags = 0,
+    };
+    if (::ioctl(pdev.m_dev_fd, DRM_IOCTL_SYNCOBJ_TIMELINE_WAIT, &wsobj) == -1) {
+      if (errno == ETIME) {
+        ret = 0;
+      } else {
+        shim_xdna::shim_err(errno,
+                            "DRM_IOCTL_SYNCOBJ_TIMELINE_WAIT IOCTL failed");
+      }
+    }
+  } else {
+    amdxdna_drm_wait_cmd wcmd = {
+        .hwctx = ctx->m_handle,
+        .timeout = timeout_ms,
+        .seq = id,
+    };
+    if (::ioctl(pdev.m_dev_fd, DRM_IOCTL_AMDXDNA_WAIT_CMD, &wcmd) == -1) {
+      if (errno == ETIME) {
+        ret = 0;
+      } else {
+        shim_xdna::shim_err(errno, "DRM_IOCTL_AMDXDNA_WAIT_CMD IOCTL failed");
+      }
     }
   }
+
   return ret;
 }