Use python bindings for tests (nod-ai#795)

daveliddell · Oct 7, 2024 · 4191152 · 4191152
1 parent 0235806
commit 4191152
Show file tree

Hide file tree

Showing 35 changed files with 704 additions and 309 deletions.
diff --git a/.github/workflows/ci-linux.yml b/.github/workflows/ci-linux.yml
@@ -111,7 +111,6 @@ jobs:
           path: ${{ env.CACHE_DIR }}
           key: ${{ env.CACHE_KEY }}
 
-
   test_linux:
     name: E2E Test linux
     needs: build_and_ctest
@@ -127,7 +126,7 @@ jobs:
         uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
         with:
           submodules: false # not required for testbench
-
+          
       - name: Download artifacts
         uses: actions/download-artifact@v4
         with:
@@ -136,20 +135,26 @@ jobs:
       - name: Extract artifact
         run: |
           tar -xvf iree-dist-linux.tar
+          echo "IREE_INSTALL_DIR=$PWD/iree-install" >> $GITHUB_ENV
+          echo "PYTHONPATH=$PWD/iree-install/python_packages/iree_compiler:$PWD/iree-install/python_packages/iree_runtime" >> $GITHUB_ENV
           bash build_tools/download_peano.sh
+          echo "PEANO_INSTALL_DIR=$PWD/llvm-aie" >> $GITHUB_ENV
 
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
       - name: Create venv and install dependencies
         run: |
-          python3 -m venv .venv
+          python -m venv .venv
           source .venv/bin/activate
-          pip install -r build_tools/ci/requirements.txt
-
+          pip install -r tests/requirements.txt
 
       - name : E2E comparison of AIE to llvm-cpu
         run: |
           source .venv/bin/activate
           source /opt/xilinx/xrt/setup.sh
-          python3 build_tools/ci/cpu_comparison/run.py \
+          python build_tools/ci/cpu_comparison/run.py \
             test_aie_vs_cpu \
             $PWD/iree-install \
             $PWD/llvm-aie \
@@ -193,3 +198,12 @@ jobs:
             $PWD/llvm-aie \
             /opt/xilinx/xrt \
             /opt/Xilinx/Vitis/2024.2
+
+      - name: Python tests
+        run: |
+          source .venv/bin/activate
+          source /opt/xilinx/xrt/setup.sh
+          pytest -v tests \
+            --capture=tee-sys \
+            --iree-install-dir=$PWD/iree-install \
+            --peano-install-dir=$PWD/llvm-aie
diff --git a/.github/workflows/ci-windows.yml b/.github/workflows/ci-windows.yml
@@ -91,7 +91,6 @@ jobs:
           $env:cache_dir = "${{ env.CACHE_DIR }}"
           $env:CCACHE_COMPILERCHECK = "string:$(clang-cl.exe --version)"
           .\build_tools\build_llvm.ps1
-          ls
           # Remove-Item -Path "$pwd\llvm-build" -Force
           $env:llvm_install_dir = "$pwd\llvm-install"
           echo $env:llvm_install_dir
@@ -139,11 +138,8 @@ jobs:
     name: E2E Test windows
     runs-on: windows-phoenix
     needs: build_and_ctest
-    defaults:
-      run:
-        shell: bash
-    strategy:
-      fail-fast: true
+    env:
+      XILINX_XRT: "C:\\Xilinx\\XRT"
     steps:
       - name: "Checking out repository" # for test scripts
         uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
@@ -158,28 +154,41 @@ jobs:
       - name: Extract artifact
         run: |
           tar -xvf iree-dist-windows.tar
-          bash build_tools/download_peano.sh 
+          Add-Content -Path $env:GITHUB_ENV -Value "IREE_INSTALL_DIR=$PWD\iree-install"
+          Add-Content -Path $env:GITHUB_ENV -Value "PYTHONPATH=$PWD\iree-install\python_packages\iree_compiler;$PWD\iree-install\python_packages\iree_runtime"
+          .\build_tools\download_peano.ps1
+          Add-Content -Path $env:GITHUB_ENV -Value "PEANO_INSTALL_DIR=$PWD\llvm-aie"
 
       - name: Create venv and install dependencies
         run: |
           python -m venv .venv
-          source .venv/Scripts/activate
-          pip install -r build_tools/ci/requirements.txt
+          .\.venv\Scripts\Activate.ps1
+          pip install -r tests\requirements.txt
 
       - name: E2E correctness matmul test
+        shell: bash
         run: |
           source .venv/Scripts/activate
-          export XILINX_XRT=/c/Xilinx/XRT
           bash build_tools/ci/run_matmul_test.sh \
             /c/test_matmuls \
             $PWD/iree-install \
             $PWD/llvm-aie
 
       - name : E2E comparison of AIE to llvm-cpu
+        shell: bash
         run: |
           source .venv/Scripts/activate
-          export XILINX_XRT=/c/Xilinx/XRT
           python build_tools/ci/cpu_comparison/run.py \
             /c/test_aie_vs_cpu \
             $PWD/iree-install \
             $PWD/llvm-aie -v
+
+      - name: Python tests
+        run: |
+          ls $env:XILINX_XRT
+          .\.venv\Scripts\Activate.ps1
+          mkdir temp
+          pytest tests -sv `
+            --basetemp=$PWD\temp `
+            --iree-install-dir="$PWD/iree-install" `
+            --peano-install-dir="$PWD/llvm-aie"
diff --git a/build_tools/build_llvm.ps1 b/build_tools/build_llvm.ps1
@@ -55,10 +55,6 @@ $env:CCACHE_SLOPPINESS = 'include_file_ctime,include_file_mtime,time_macros'
 
 & ccache -z
 
-# on windows python bindings don't for split build because
-# i can't figure out MLIR_CAPI_EXPORTED and MLIR_CAPI_BUILDING_LIBRARY
-# which somehow disables exceptions (blocking bootgen and xrt)
-
 $CMAKE_ARGS = @(
   "-GNinja"
   "-DCMAKE_BUILD_TYPE=Release"
@@ -85,7 +81,7 @@ $CMAKE_ARGS = @(
   "-DLLVM_ENABLE_ZSTD=OFF"
   "-DLLVM_FORCE_ENABLE_STATS=ON"
   "-DLLVM_INSTALL_UTILS=ON"
-  "-DMLIR_ENABLE_BINDINGS_PYTHON=OFF"
+  "-DMLIR_ENABLE_BINDINGS_PYTHON=ON"
   "-DLLVM_ENABLE_PROJECTS=mlir;clang;lld"
   "-DLLVM_TARGET_ARCH=X86"
   "-DLLVM_TARGETS_TO_BUILD=X86"

diff --git a/build_tools/build_test_cpp.ps1 b/build_tools/build_test_cpp.ps1
@@ -61,9 +61,6 @@ $CMAKE_ARGS = @(
     "-DCMAKE_BUILD_TYPE=Release"
     "-DCMAKE_INSTALL_PREFIX=$install_dir"
     "-DCMAKE_INSTALL_LIBDIR=lib"
-    "-DCMAKE_EXE_LINKER_FLAGS_INIT=-fuse-ld=lld"
-    "-DCMAKE_SHARED_LINKER_FLAGS_INIT=-fuse-ld=lld"
-    "-DCMAKE_MODULE_LINKER_FLAGS_INIT=-fuse-ld=lld"
     "-DCMAKE_C_COMPILER=$env:CC"
     "-DCMAKE_CXX_COMPILER=$env:CXX"
     "-DLLVM_TARGET_ARCH=X86"
@@ -84,6 +81,7 @@ $CMAKE_ARGS = @(
     "-DIREE_TARGET_BACKEND_LLVM_CPU=ON"
     "-DIREE_CMAKE_PLUGIN_PATHS=$repo_root"
     "-DIREE_EXTERNAL_HAL_DRIVERS=xrt"
+    "-DIREE_BUILD_PYTHON_BINDINGS=ON"
 )
 
 if ($llvm_install_dir -and (Test-Path "$llvm_install_dir"))
@@ -92,26 +90,16 @@ if ($llvm_install_dir -and (Test-Path "$llvm_install_dir"))
     # TODO(max): send IREE a fix for this
     # target_compile_definitions may only set INTERFACE properties on IMPORTED
     $cmake_file = Resolve-Path -Path "$iree_dir/compiler/src/iree/compiler/API/CMakeLists.txt"
-    (Get-Content $cmake_file).Replace("`$`{_object_lib} PRIVATE", "`$`{_object_lib} INTERFACE") `
+    (Get-Content $cmake_file).Replace("if(MSVC)", "get_target_property(_imported `$`{_object_lib} IMPORTED)`n  if(MSVC AND NOT `$`{_imported})") `
         | Out-File -encoding ASCII $cmake_file
     $CMAKE_ARGS += @(
         "-DIREE_BUILD_BUNDLED_LLVM=OFF"
         "-DClang_DIR=$llvm_install_dir/lib/cmake/clang"
         "-DLLD_DIR=$llvm_install_dir/lib/cmake/lld"
         "-DMLIR_DIR=$llvm_install_dir/lib/cmake/mlir"
         "-DLLVM_DIR=$llvm_install_dir/lib/cmake/llvm"
-        # TODO(max)
-        # on windows python bindings don't for split build because
-        # i can't figure out MLIR_CAPI_EXPORTED and MLIR_CAPI_BUILDING_LIBRARY
-        # which somehow disables exceptions
-        "-DIREE_BUILD_PYTHON_BINDINGS=OFF"
     )
 }
-else
-{
-    echo "building bundled llvm"
-    $CMAKE_ARGS += @("-DIREE_BUILD_PYTHON_BINDINGS=ON")
-}
 
 & cmake $CMAKE_ARGS -S $iree_dir -B $build_dir
 
@@ -122,6 +110,7 @@ echo "------------"
 echo "Installing"
 echo "----------"
 echo "Install to: $install_dir"
+& cmake --build $build_dir --target install
 & cmake --build $build_dir --target iree-install-dist
 
 echo "CTest"
@@ -143,3 +132,4 @@ if ($llvm_install_dir -and (Test-Path "$llvm_install_dir"))
 }
 
 Copy-Item -Path "$build_dir/tools/testing/e2e/iree-e2e-matmul-test.exe" -Destination "$install_dir/bin" -Force
+Copy-Item -Path "$build_dir/tools/xrt_coreutil.dll" -Destination "$install_dir/python_packages/iree_runtime/iree/_runtime_libs" -Force
diff --git a/build_tools/build_test_cpp.sh b/build_tools/build_test_cpp.sh
@@ -116,6 +116,9 @@ cmake --build "$build_dir" -- -k 0
 echo "Installing"
 echo "----------"
 echo "Install to: $install_dir"
+cmake --build "$build_dir" --target install
+# TODO(max): there's no way to install the python runtime bindings
+# ninja iree-install-dist install-IREECompilerPythonModules install-IREEDialectsPythonModules
 cmake --build "$build_dir" --target iree-install-dist
 
 echo "CTest"

diff --git a/build_tools/ci/cpu_comparison/run.py b/build_tools/ci/cpu_comparison/run.py
@@ -2,6 +2,7 @@
 
 # Copyright 2024 The IREE Authors
 
+import sys
 import argparse
 import os
 import platform
@@ -97,15 +98,17 @@ def shell_out(cmd: list, workdir=None, verbose: int = 0, raise_on_error=True, en
             print("Standard output from script:")
             print(stdout_decode)
         if stderr_decode:
-            print("Standard error from script:")
-            print(stderr_decode)
+            print("Standard error from script:", file=sys.stderr)
+            print(stderr_decode, file=sys.stderr)
     if not raise_on_error and handle.returncode != 0:
         print(
-            f"Error executing script, error code was {handle.returncode}. Not raising an error."
+            f"Error executing script, error code was {handle.returncode}. Not raising an error.",
+            file=sys.stderr
         )
     if raise_on_error and handle.returncode != 0:
         raise RuntimeError(
-            f"Error executing script, error code was {handle.returncode}"
+            f"Error executing script, error code was {handle.returncode}",
+            file=sys.stderr
         )
     return stdout_decode, stderr_decode
 

diff --git a/build_tools/ci/run_matmul_test.sh b/build_tools/ci/run_matmul_test.sh
@@ -634,11 +634,6 @@ i32_shapes_small=(
   '128x256x128'
 )
 
-i32_shapes_medium=(
-  '1024x1024x1024' 
-  '1536x2048x1536'
-)
-
 run_matmul_test_on_shapes ${i32_shapes_small[@]} \
     --name_prefix "small_i32" \
     --lower_to_aie_pipeline "objectFifo" \
@@ -659,9 +654,10 @@ run_matmul_test_on_shapes ${i32_shapes_small[@]} \
 i32_shapes_medium=(
   '1024x1024x1024'
 )
-if [ "$OSTYPE" != "msys" ]; then
-  i32_shapes_medium+=('1536x2048x1536')
-fi
+# TODO(jornt): re-enable `1536x2048x1536`
+#if [ "$OSTYPE" != "msys" ]; then
+#  i32_shapes_medium+=('1536x2048x1536')
+#fi
 
 run_matmul_test_on_shapes ${i32_shapes_medium[@]} \
     --name_prefix "medium_i32" \

diff --git a/build_tools/download_peano.ps1 b/build_tools/download_peano.ps1
@@ -0,0 +1,12 @@
+# Copyright 2024 The IREE Authors
+#
+# Licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+$ErrorActionPreference = 'Stop'
+
+$this_dir = Split-Path -Path $MyInvocation.MyCommand.Path -Parent
+$RELEASE = (Get-Content -Path "$this_dir/peano_commit.txt")
+pip download llvm_aie==$RELEASE -f https://github.com/Xilinx/llvm-aie/releases/expanded_assets/nightly
+Expand-Archive (Get-ChildItem -Filter llvm*.whl).FullName -DestinationPath $PWD.Path
diff --git a/build_tools/download_peano.sh b/build_tools/download_peano.sh
@@ -6,6 +6,7 @@
 # See https://llvm.org/LICENSE.txt for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-RELEASE=19.0.0.2024092601+562ccea2
+this_dir="$(cd $(dirname $0) && pwd)"
+RELEASE=$(cat $this_dir/peano_commit.txt)
 pip download llvm_aie==$RELEASE -f https://github.com/Xilinx/llvm-aie/releases/expanded_assets/nightly
 unzip llvm_aie*whl
diff --git a/build_tools/peano_commit.txt b/build_tools/peano_commit.txt
@@ -0,0 +1 @@
+19.0.0.2024092601+562ccea2
diff --git a/cmake/iree_aie_bootgen.cmake b/cmake/iree_aie_bootgen.cmake
@@ -16,12 +16,10 @@ set(_BOOTGEN_SOURCE_DIR ${IREE_AMD_AIE_SOURCE_DIR}/third_party/bootgen)
 # https://stackoverflow.com/a/56463133 If you want to use malloc, then include stdlib.h
 replace_string_in_file(${_BOOTGEN_SOURCE_DIR}/cdo-npi.c "#include <malloc.h>" "#include <stdlib.h>")
 replace_string_in_file(${_BOOTGEN_SOURCE_DIR}/cdo-alloc.c "#include <malloc.h>" "#include <stdlib.h>")
-replace_string_in_file("${_BOOTGEN_SOURCE_DIR}/main.cpp"
-                       "#include \"openssl/ms/applink.c\"" "//#include \"openssl/ms/applink.c\"")
-replace_string_in_file("${_BOOTGEN_SOURCE_DIR}/main.cpp"
-                       "int main" "int iree_aie_bootgen_main")
-replace_string_in_file("${_BOOTGEN_SOURCE_DIR}/main.cpp"
-                       "DisplayBanner();" "//DisplayBanner();")
+replace_string_in_file("${_BOOTGEN_SOURCE_DIR}/main.cpp" "#include \"openssl/ms/applink.c\"" "")
+replace_string_in_file("${_BOOTGEN_SOURCE_DIR}/main.cpp" "int main" "int iree_aie_bootgen_main")
+replace_string_in_file("${_BOOTGEN_SOURCE_DIR}/main.cpp" "DisplayBanner();" "")
+replace_string_in_file("${_BOOTGEN_SOURCE_DIR}/main.cpp" "LOG_MSG(\"\\n[INFO]   : Bootimage generated successfully\\n\");" "")
 
 file(GLOB _bootgen_sources "${_BOOTGEN_SOURCE_DIR}/*.c" "${_BOOTGEN_SOURCE_DIR}/*.cpp")
 add_library(iree-aie-bootgen STATIC ${_bootgen_sources})

diff --git a/cmake/iree_aie_rt.cmake b/cmake/iree_aie_rt.cmake
@@ -14,16 +14,17 @@ endif()
 # cdo-drver
 # ##############################################################################
 
-set(_BOOTGEN_SOURCE_DIR ${IREE_AMD_AIE_SOURCE_DIR}/third_party/bootgen)
-replace_string_in_file(${_BOOTGEN_SOURCE_DIR}/cdo-driver/cdo_driver.c "\"(Write64)" "\"cdo-driver: (Write64)")
-replace_string_in_file(${_BOOTGEN_SOURCE_DIR}/cdo-driver/cdo_driver.c "\"(MaskWrite64)" "\"cdo-driver: (MaskWrite64)")
-replace_string_in_file(${_BOOTGEN_SOURCE_DIR}/cdo-driver/cdo_driver.c "\"(NOP Command)" "\"cdo-driver: (NOP Command)")
-replace_string_in_file(${_BOOTGEN_SOURCE_DIR}/cdo-driver/cdo_driver.c "\"(BlockWrite-DMAWriteCmd)" "\"cdo-driver: (BlockWrite-DMAWriteCmd)")
-replace_string_in_file(${_BOOTGEN_SOURCE_DIR}/cdo-driver/cdo_driver.c "Data@ 0x%\" PRIxPTR \"" "Data")
-replace_string_in_file(${_BOOTGEN_SOURCE_DIR}/cdo-driver/cdo_driver.c "(uintptr_t)(pData + i)," "")
-replace_string_in_file(${_BOOTGEN_SOURCE_DIR}/cdo-driver/cdo_driver.c "\"    Address:" "\"cdo-driver:     Address:")
-replace_string_in_file(${_BOOTGEN_SOURCE_DIR}/cdo-driver/cdo_driver.c "\"(BlockSet-DMAWriteCmd)" "\"cdo-driver: (BlockSet-DMAWriteCmd)")
-replace_string_in_file(${_BOOTGEN_SOURCE_DIR}/cdo-driver/cdo_driver.c "\"(MaskPoll64)" "\"cdo-driver: (MaskPoll64)")
+set(_BOOTGEN_SOURCE_DIR "${IREE_AMD_AIE_SOURCE_DIR}/third_party/bootgen")
+replace_string_in_file("${_BOOTGEN_SOURCE_DIR}/cdo-driver/cdo_driver.c" "\"(Write64)" "\"cdo-driver: (Write64)")
+replace_string_in_file("${_BOOTGEN_SOURCE_DIR}/cdo-driver/cdo_driver.c" "\"(MaskWrite64)" "\"cdo-driver: (MaskWrite64)")
+replace_string_in_file("${_BOOTGEN_SOURCE_DIR}/cdo-driver/cdo_driver.c" "\"(NOP Command)" "\"cdo-driver: (NOP Command)")
+replace_string_in_file("${_BOOTGEN_SOURCE_DIR}/cdo-driver/cdo_driver.c" "\"(BlockWrite-DMAWriteCmd)" "\"cdo-driver: (BlockWrite-DMAWriteCmd)")
+replace_string_in_file("${_BOOTGEN_SOURCE_DIR}/cdo-driver/cdo_driver.c" "Data@ 0x%\" PRIxPTR \"" "Data")
+replace_string_in_file("${_BOOTGEN_SOURCE_DIR}/cdo-driver/cdo_driver.c" "(uintptr_t)(pData + i)," "")
+replace_string_in_file("${_BOOTGEN_SOURCE_DIR}/cdo-driver/cdo_driver.c" "\"    Address:" "\"cdo-driver:     Address:")
+replace_string_in_file("${_BOOTGEN_SOURCE_DIR}/cdo-driver/cdo_driver.c" "\"(BlockSet-DMAWriteCmd)" "\"cdo-driver: (BlockSet-DMAWriteCmd)")
+replace_string_in_file("${_BOOTGEN_SOURCE_DIR}/cdo-driver/cdo_driver.c" "\"(MaskPoll64)" "\"cdo-driver: (MaskPoll64)")
+replace_string_in_file("${_BOOTGEN_SOURCE_DIR}/cdo-driver/cdo_driver.c" "printf(\"Generating: %s\\n\", cdoFileName);" "")
 
 add_library(cdo_driver STATIC ${_BOOTGEN_SOURCE_DIR}/cdo-driver/cdo_driver.c)
 target_include_directories(cdo_driver PUBLIC ${_BOOTGEN_SOURCE_DIR}/cdo-driver)

diff --git a/cmake/iree_aie_xrt.cmake b/cmake/iree_aie_xrt.cmake
@@ -34,7 +34,6 @@ set(IREE_AIE_BOOST_LIBS
     any
     algorithm
     asio
-    exception
     format
     functional
     lexical_cast
@@ -176,7 +175,10 @@ foreach(_core_lib IN LISTS _core_libs)
                              ${IREE_XRT_SOURCE_DIR}/runtime_src)
   target_include_directories(${_core_lib} SYSTEM PUBLIC
                              ${IREE_XRT_SOURCE_DIR}/runtime_src/core/common/elf)
-  target_compile_definitions(${_core_lib} PUBLIC -DBOOST_BIND_GLOBAL_PLACEHOLDERS)
+  target_compile_definitions(${_core_lib}
+                             PUBLIC
+                             -DBOOST_BIND_GLOBAL_PLACEHOLDERS
+                             $<$<CONFIG:Debug>:-DXRT_VERBOSE>)
   target_compile_options(${_core_lib}
                          PRIVATE
                          $<$<PLATFORM_ID:Linux>:-fexceptions -frtti -w>