anv: Add INTEL_DEBUG for bvh dump and visualization tools

This commit allows you to dump different regions of memory related to
bvh building. An additional script to decode the memory dump is also
added, and you're able to view the built bvh in 3D view in html. See the
included README.md for usage.

Rework:
- you can now view the actual child_coord in internalNode in html
- change exponent to be int8_t in the interpreter
- fix the actual coordinates using an updated formula
- now you can have 3D view of the bvh
- blockIncr could be 2 and vk_aabb should be first
- Now, if any bvh dump is enabled, we will zero out tlas, to prevent gpu
  hang caused by incorrect tlas traversal
- rootNodeOffset is back to the beginning
- Add INTEL_DEBUG=bvh_no_build.
- Fix type of dump_size
- add assertion for a 4B alignment
- when clearing out bvh, only clear out everything after
  (header+bvh_offset)
- TODO: instead of dumping on destory, track in the command buffer

Acked-by: Lionel Landwerlin <[email protected]>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/31588>

Author: kaiwenjon

src/intel/dev/intel_debug.c

@@ -94,6 +94,13 @@ static const struct debug_control debug_control[] = {
                     DEBUG_TES | DEBUG_GS | DEBUG_CS |
                     DEBUG_RT | DEBUG_TASK | DEBUG_MESH },
    { "rt",          DEBUG_RT },
+   { "bvh_blas",    DEBUG_BVH_BLAS},
+   { "bvh_tlas",    DEBUG_BVH_TLAS},
+   { "bvh_blas_ir_hdr", DEBUG_BVH_BLAS_IR_HDR},
+   { "bvh_tlas_ir_hdr", DEBUG_BVH_TLAS_IR_HDR},
+   { "bvh_blas_ir_as",  DEBUG_BVH_BLAS_IR_AS},
+   { "bvh_tlas_ir_as",  DEBUG_BVH_TLAS_IR_AS},
+   { "bvh_no_build",    DEBUG_BVH_NO_BUILD},
    { "task",        DEBUG_TASK },
    { "mesh",        DEBUG_MESH },
    { "stall",       DEBUG_STALL },

src/intel/dev/intel_debug.h

@@ -99,6 +99,13 @@ extern uint64_t intel_debug;
 #define DEBUG_REG_PRESSURE        (1ull << 51)
 #define DEBUG_SHADER_PRINT        (1ull << 52)
 #define DEBUG_CL_QUIET            (1ull << 53)
+#define DEBUG_BVH_BLAS            (1ull << 54)
+#define DEBUG_BVH_TLAS            (1ull << 55)
+#define DEBUG_BVH_BLAS_IR_HDR     (1ull << 56)
+#define DEBUG_BVH_TLAS_IR_HDR     (1ull << 57)
+#define DEBUG_BVH_BLAS_IR_AS      (1ull << 58)
+#define DEBUG_BVH_TLAS_IR_AS      (1ull << 59)
+#define DEBUG_BVH_NO_BUILD        (1ull << 60)
 
 #define DEBUG_ANY                 (~0ull)
 
@@ -110,6 +117,12 @@ extern uint64_t intel_debug;
    (DEBUG_NO_DUAL_OBJECT_GS | DEBUG_SPILL_FS | \
    DEBUG_SPILL_VEC4 | DEBUG_NO_COMPACTION | DEBUG_DO32 | DEBUG_SOFT64)
 
+/* Flags to determine what bvh to dump out */
+#define DEBUG_BVH_ANV (DEBUG_BVH_BLAS | DEBUG_BVH_TLAS)
+#define DEBUG_BVH_IR_HDR (DEBUG_BVH_BLAS_IR_HDR | DEBUG_BVH_TLAS_IR_HDR)
+#define DEBUG_BVH_IR_AS (DEBUG_BVH_BLAS_IR_AS | DEBUG_BVH_TLAS_IR_AS)
+#define DEBUG_BVH_ANY (DEBUG_BVH_ANV | DEBUG_BVH_IR_HDR | DEBUG_BVH_IR_AS)
+
 extern uint64_t intel_simd;
 extern uint32_t intel_debug_bkp_before_draw_count;
 extern uint32_t intel_debug_bkp_after_draw_count;

src/intel/vulkan/anv_private.h

@@ -1834,6 +1834,20 @@ enum anv_rt_bvh_build_method {
  */
 #define ANV_RT_UUID_MACRO             "ANV_RT_BVH_0001"
 
+enum bvh_dump_type {
+   BVH_ANV,
+   BVH_IR_HDR,
+   BVH_IR_AS
+};
+
+struct bvh_dump_struct {
+   struct anv_bo *bo;
+   uint32_t bvh_id;
+   uint64_t dump_size;
+   VkGeometryTypeKHR geometry_type;
+   enum bvh_dump_type dump_type;
+};
+
 struct anv_device_astc_emu {
     struct vk_texcompress_astc_state           *texcompress;
 

src/intel/vulkan/bvh/README.md

@@ -0,0 +1,38 @@
+## BVH Debug
+
+1. `INTEL_DEBUG=bvh_tlas,bvh_blas` will generate `tlas_{id}.txt` or `blas_{id}.txt` in `bvh_dump/BVH_ANV` directory.
+2. `INTEL_DEBUG=bvh_tlas_ir_hdr,bvh_blas_ir_hdr` will generate `tlas_{id}.txt` or `blas_{id}.txt` in `bvh_dump/BVH_IR_HDR` directory.
+3. `INTEL_DEBUG=bvh_tlas_ir_as,bvh_blas_ir_as` will generate `tlas_{id}.txt` or `blas_{id}.txt` in `bvh_dump/BVH_IR_AS` directory.
+4. `INTEL_DEBUG=bvh_no_build` will skip the intel-specific-encoding part. If gpu hang is seen, this is the first step to isolate the problem. If toggled on and gpu doesn't hang anymore, that means it was either encode.comp was spinning, or the built bvh has issues so gpu hung during bvh traversal.
+
+The dumped text file contains memory dump, byte-by-byte in hex. The contents are contiguous memory of a certain region.
+1. The dump in `BVH_ANV` starts from the beginning of `anv_accel_struct_header` to the end of the bvh. Nodes/leaves are packed tightly after the header, encoded in a way that our HW expects.
+2. The dump in `BVH_IR_HDR` records the contents of `vk_ir_header` sitting at the beginning of ir bvh.
+3. The dump in `BVH_IR_AS` records all `vk_ir_{leaf_type}_node` and  `vk_ir_box_node` in ir bvh. The region starts from where leaves are encoded to the end of ir bvh.
+
+### Decode the dump
+
+We have a way to decode the dump in `BVH_ANV`.
+- To decode this memory dump, use a python script to parse the file and generate a human-readable json.
+- To further visualize the tree, there is a html that parses the json and draws the tree topology and 3D views of bounding boxes.
+
+```
+# Using blas_0 as an example
+xxd -r -p bvh_dump/BVH_ANV/blas_0.txt > input.bin
+
+# Use a python script to generate a human-readable json file
+cd mesa/src/intel/vulkan/bvh/
+python3 interpret.py <path/to/input.bin>
+
+# To further visualize the tree, the html parses the json and draws it in 3D.
+cd mesa/src/intel/vulkan/bvh/
+python3 -m http.server 8000
+# go to localhost:8000/visualize_json.html
+
+```
+
+### Note and Limitations:
+1. The python script use `ctypes` to interpret the memory dump, so the structure defined in the script should match the structure defined in the driver.
+2. The memory dump is a snapshot of a VkBuffer captured at the end of `CmdBuildAccelerationStructure` call. It won't capture any bvh obtained from `CmdCopy`.
+3. The memory dump of captured bvhs so far are saved to files at the moment when `DestroyAccelerationStructure` is called every time.
+4. If ANY dump is enabled, we will nullify anv tlas bvh and send all 0s to the gpu. Doing this can prevent gpu hang caused by incorrect bvh traversal. However, the actual contents are still saved to files for debugging.