Skip to content

Commit

Permalink
Use python bindings for tests (nod-ai#795)
Browse files Browse the repository at this point in the history
  • Loading branch information
makslevental authored and daveliddell committed Oct 7, 2024
1 parent 0235806 commit 4191152
Show file tree
Hide file tree
Showing 35 changed files with 704 additions and 309 deletions.
26 changes: 20 additions & 6 deletions .github/workflows/ci-linux.yml
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,6 @@ jobs:
path: ${{ env.CACHE_DIR }}
key: ${{ env.CACHE_KEY }}


test_linux:
name: E2E Test linux
needs: build_and_ctest
Expand All @@ -127,7 +126,7 @@ jobs:
uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
with:
submodules: false # not required for testbench

- name: Download artifacts
uses: actions/download-artifact@v4
with:
Expand All @@ -136,20 +135,26 @@ jobs:
- name: Extract artifact
run: |
tar -xvf iree-dist-linux.tar
echo "IREE_INSTALL_DIR=$PWD/iree-install" >> $GITHUB_ENV
echo "PYTHONPATH=$PWD/iree-install/python_packages/iree_compiler:$PWD/iree-install/python_packages/iree_runtime" >> $GITHUB_ENV
bash build_tools/download_peano.sh
echo "PEANO_INSTALL_DIR=$PWD/llvm-aie" >> $GITHUB_ENV
- uses: actions/setup-python@v4
with:
python-version: '3.11'

- name: Create venv and install dependencies
run: |
python3 -m venv .venv
python -m venv .venv
source .venv/bin/activate
pip install -r build_tools/ci/requirements.txt
pip install -r tests/requirements.txt
- name : E2E comparison of AIE to llvm-cpu
run: |
source .venv/bin/activate
source /opt/xilinx/xrt/setup.sh
python3 build_tools/ci/cpu_comparison/run.py \
python build_tools/ci/cpu_comparison/run.py \
test_aie_vs_cpu \
$PWD/iree-install \
$PWD/llvm-aie \
Expand Down Expand Up @@ -193,3 +198,12 @@ jobs:
$PWD/llvm-aie \
/opt/xilinx/xrt \
/opt/Xilinx/Vitis/2024.2
- name: Python tests
run: |
source .venv/bin/activate
source /opt/xilinx/xrt/setup.sh
pytest -v tests \
--capture=tee-sys \
--iree-install-dir=$PWD/iree-install \
--peano-install-dir=$PWD/llvm-aie
31 changes: 20 additions & 11 deletions .github/workflows/ci-windows.yml
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,6 @@ jobs:
$env:cache_dir = "${{ env.CACHE_DIR }}"
$env:CCACHE_COMPILERCHECK = "string:$(clang-cl.exe --version)"
.\build_tools\build_llvm.ps1
ls
# Remove-Item -Path "$pwd\llvm-build" -Force
$env:llvm_install_dir = "$pwd\llvm-install"
echo $env:llvm_install_dir
Expand Down Expand Up @@ -139,11 +138,8 @@ jobs:
name: E2E Test windows
runs-on: windows-phoenix
needs: build_and_ctest
defaults:
run:
shell: bash
strategy:
fail-fast: true
env:
XILINX_XRT: "C:\\Xilinx\\XRT"
steps:
- name: "Checking out repository" # for test scripts
uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 # v3.5.0
Expand All @@ -158,28 +154,41 @@ jobs:
- name: Extract artifact
run: |
tar -xvf iree-dist-windows.tar
bash build_tools/download_peano.sh
Add-Content -Path $env:GITHUB_ENV -Value "IREE_INSTALL_DIR=$PWD\iree-install"
Add-Content -Path $env:GITHUB_ENV -Value "PYTHONPATH=$PWD\iree-install\python_packages\iree_compiler;$PWD\iree-install\python_packages\iree_runtime"
.\build_tools\download_peano.ps1
Add-Content -Path $env:GITHUB_ENV -Value "PEANO_INSTALL_DIR=$PWD\llvm-aie"
- name: Create venv and install dependencies
run: |
python -m venv .venv
source .venv/Scripts/activate
pip install -r build_tools/ci/requirements.txt
.\.venv\Scripts\Activate.ps1
pip install -r tests\requirements.txt
- name: E2E correctness matmul test
shell: bash
run: |
source .venv/Scripts/activate
export XILINX_XRT=/c/Xilinx/XRT
bash build_tools/ci/run_matmul_test.sh \
/c/test_matmuls \
$PWD/iree-install \
$PWD/llvm-aie
- name : E2E comparison of AIE to llvm-cpu
shell: bash
run: |
source .venv/Scripts/activate
export XILINX_XRT=/c/Xilinx/XRT
python build_tools/ci/cpu_comparison/run.py \
/c/test_aie_vs_cpu \
$PWD/iree-install \
$PWD/llvm-aie -v
- name: Python tests
run: |
ls $env:XILINX_XRT
.\.venv\Scripts\Activate.ps1
mkdir temp
pytest tests -sv `
--basetemp=$PWD\temp `
--iree-install-dir="$PWD/iree-install" `
--peano-install-dir="$PWD/llvm-aie"
6 changes: 1 addition & 5 deletions build_tools/build_llvm.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -55,10 +55,6 @@ $env:CCACHE_SLOPPINESS = 'include_file_ctime,include_file_mtime,time_macros'

& ccache -z

# on windows python bindings don't for split build because
# i can't figure out MLIR_CAPI_EXPORTED and MLIR_CAPI_BUILDING_LIBRARY
# which somehow disables exceptions (blocking bootgen and xrt)

$CMAKE_ARGS = @(
"-GNinja"
"-DCMAKE_BUILD_TYPE=Release"
Expand All @@ -85,7 +81,7 @@ $CMAKE_ARGS = @(
"-DLLVM_ENABLE_ZSTD=OFF"
"-DLLVM_FORCE_ENABLE_STATS=ON"
"-DLLVM_INSTALL_UTILS=ON"
"-DMLIR_ENABLE_BINDINGS_PYTHON=OFF"
"-DMLIR_ENABLE_BINDINGS_PYTHON=ON"
"-DLLVM_ENABLE_PROJECTS=mlir;clang;lld"
"-DLLVM_TARGET_ARCH=X86"
"-DLLVM_TARGETS_TO_BUILD=X86"
Expand Down
18 changes: 4 additions & 14 deletions build_tools/build_test_cpp.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -61,9 +61,6 @@ $CMAKE_ARGS = @(
"-DCMAKE_BUILD_TYPE=Release"
"-DCMAKE_INSTALL_PREFIX=$install_dir"
"-DCMAKE_INSTALL_LIBDIR=lib"
"-DCMAKE_EXE_LINKER_FLAGS_INIT=-fuse-ld=lld"
"-DCMAKE_SHARED_LINKER_FLAGS_INIT=-fuse-ld=lld"
"-DCMAKE_MODULE_LINKER_FLAGS_INIT=-fuse-ld=lld"
"-DCMAKE_C_COMPILER=$env:CC"
"-DCMAKE_CXX_COMPILER=$env:CXX"
"-DLLVM_TARGET_ARCH=X86"
Expand All @@ -84,6 +81,7 @@ $CMAKE_ARGS = @(
"-DIREE_TARGET_BACKEND_LLVM_CPU=ON"
"-DIREE_CMAKE_PLUGIN_PATHS=$repo_root"
"-DIREE_EXTERNAL_HAL_DRIVERS=xrt"
"-DIREE_BUILD_PYTHON_BINDINGS=ON"
)

if ($llvm_install_dir -and (Test-Path "$llvm_install_dir"))
Expand All @@ -92,26 +90,16 @@ if ($llvm_install_dir -and (Test-Path "$llvm_install_dir"))
# TODO(max): send IREE a fix for this
# target_compile_definitions may only set INTERFACE properties on IMPORTED
$cmake_file = Resolve-Path -Path "$iree_dir/compiler/src/iree/compiler/API/CMakeLists.txt"
(Get-Content $cmake_file).Replace("`$`{_object_lib} PRIVATE", "`$`{_object_lib} INTERFACE") `
(Get-Content $cmake_file).Replace("if(MSVC)", "get_target_property(_imported `$`{_object_lib} IMPORTED)`n if(MSVC AND NOT `$`{_imported})") `
| Out-File -encoding ASCII $cmake_file
$CMAKE_ARGS += @(
"-DIREE_BUILD_BUNDLED_LLVM=OFF"
"-DClang_DIR=$llvm_install_dir/lib/cmake/clang"
"-DLLD_DIR=$llvm_install_dir/lib/cmake/lld"
"-DMLIR_DIR=$llvm_install_dir/lib/cmake/mlir"
"-DLLVM_DIR=$llvm_install_dir/lib/cmake/llvm"
# TODO(max)
# on windows python bindings don't for split build because
# i can't figure out MLIR_CAPI_EXPORTED and MLIR_CAPI_BUILDING_LIBRARY
# which somehow disables exceptions
"-DIREE_BUILD_PYTHON_BINDINGS=OFF"
)
}
else
{
echo "building bundled llvm"
$CMAKE_ARGS += @("-DIREE_BUILD_PYTHON_BINDINGS=ON")
}

& cmake $CMAKE_ARGS -S $iree_dir -B $build_dir

Expand All @@ -122,6 +110,7 @@ echo "------------"
echo "Installing"
echo "----------"
echo "Install to: $install_dir"
& cmake --build $build_dir --target install
& cmake --build $build_dir --target iree-install-dist

echo "CTest"
Expand All @@ -143,3 +132,4 @@ if ($llvm_install_dir -and (Test-Path "$llvm_install_dir"))
}

Copy-Item -Path "$build_dir/tools/testing/e2e/iree-e2e-matmul-test.exe" -Destination "$install_dir/bin" -Force
Copy-Item -Path "$build_dir/tools/xrt_coreutil.dll" -Destination "$install_dir/python_packages/iree_runtime/iree/_runtime_libs" -Force
3 changes: 3 additions & 0 deletions build_tools/build_test_cpp.sh
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,9 @@ cmake --build "$build_dir" -- -k 0
echo "Installing"
echo "----------"
echo "Install to: $install_dir"
cmake --build "$build_dir" --target install
# TODO(max): there's no way to install the python runtime bindings
# ninja iree-install-dist install-IREECompilerPythonModules install-IREEDialectsPythonModules
cmake --build "$build_dir" --target iree-install-dist

echo "CTest"
Expand Down
11 changes: 7 additions & 4 deletions build_tools/ci/cpu_comparison/run.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

# Copyright 2024 The IREE Authors

import sys
import argparse
import os
import platform
Expand Down Expand Up @@ -97,15 +98,17 @@ def shell_out(cmd: list, workdir=None, verbose: int = 0, raise_on_error=True, en
print("Standard output from script:")
print(stdout_decode)
if stderr_decode:
print("Standard error from script:")
print(stderr_decode)
print("Standard error from script:", file=sys.stderr)
print(stderr_decode, file=sys.stderr)
if not raise_on_error and handle.returncode != 0:
print(
f"Error executing script, error code was {handle.returncode}. Not raising an error."
f"Error executing script, error code was {handle.returncode}. Not raising an error.",
file=sys.stderr
)
if raise_on_error and handle.returncode != 0:
raise RuntimeError(
f"Error executing script, error code was {handle.returncode}"
f"Error executing script, error code was {handle.returncode}",
file=sys.stderr
)
return stdout_decode, stderr_decode

Expand Down
12 changes: 4 additions & 8 deletions build_tools/ci/run_matmul_test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -634,11 +634,6 @@ i32_shapes_small=(
'128x256x128'
)

i32_shapes_medium=(
'1024x1024x1024'
'1536x2048x1536'
)

run_matmul_test_on_shapes ${i32_shapes_small[@]} \
--name_prefix "small_i32" \
--lower_to_aie_pipeline "objectFifo" \
Expand All @@ -659,9 +654,10 @@ run_matmul_test_on_shapes ${i32_shapes_small[@]} \
i32_shapes_medium=(
'1024x1024x1024'
)
if [ "$OSTYPE" != "msys" ]; then
i32_shapes_medium+=('1536x2048x1536')
fi
# TODO(jornt): re-enable `1536x2048x1536`
#if [ "$OSTYPE" != "msys" ]; then
# i32_shapes_medium+=('1536x2048x1536')
#fi

run_matmul_test_on_shapes ${i32_shapes_medium[@]} \
--name_prefix "medium_i32" \
Expand Down
12 changes: 12 additions & 0 deletions build_tools/download_peano.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
# Copyright 2024 The IREE Authors
#
# Licensed under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

$ErrorActionPreference = 'Stop'

$this_dir = Split-Path -Path $MyInvocation.MyCommand.Path -Parent
$RELEASE = (Get-Content -Path "$this_dir/peano_commit.txt")
pip download llvm_aie==$RELEASE -f https://github.com/Xilinx/llvm-aie/releases/expanded_assets/nightly
Expand-Archive (Get-ChildItem -Filter llvm*.whl).FullName -DestinationPath $PWD.Path
3 changes: 2 additions & 1 deletion build_tools/download_peano.sh
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

RELEASE=19.0.0.2024092601+562ccea2
this_dir="$(cd $(dirname $0) && pwd)"
RELEASE=$(cat $this_dir/peano_commit.txt)
pip download llvm_aie==$RELEASE -f https://github.com/Xilinx/llvm-aie/releases/expanded_assets/nightly
unzip llvm_aie*whl
1 change: 1 addition & 0 deletions build_tools/peano_commit.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
19.0.0.2024092601+562ccea2
10 changes: 4 additions & 6 deletions cmake/iree_aie_bootgen.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,10 @@ set(_BOOTGEN_SOURCE_DIR ${IREE_AMD_AIE_SOURCE_DIR}/third_party/bootgen)
# https://stackoverflow.com/a/56463133 If you want to use malloc, then include stdlib.h
replace_string_in_file(${_BOOTGEN_SOURCE_DIR}/cdo-npi.c "#include <malloc.h>" "#include <stdlib.h>")
replace_string_in_file(${_BOOTGEN_SOURCE_DIR}/cdo-alloc.c "#include <malloc.h>" "#include <stdlib.h>")
replace_string_in_file("${_BOOTGEN_SOURCE_DIR}/main.cpp"
"#include \"openssl/ms/applink.c\"" "//#include \"openssl/ms/applink.c\"")
replace_string_in_file("${_BOOTGEN_SOURCE_DIR}/main.cpp"
"int main" "int iree_aie_bootgen_main")
replace_string_in_file("${_BOOTGEN_SOURCE_DIR}/main.cpp"
"DisplayBanner();" "//DisplayBanner();")
replace_string_in_file("${_BOOTGEN_SOURCE_DIR}/main.cpp" "#include \"openssl/ms/applink.c\"" "")
replace_string_in_file("${_BOOTGEN_SOURCE_DIR}/main.cpp" "int main" "int iree_aie_bootgen_main")
replace_string_in_file("${_BOOTGEN_SOURCE_DIR}/main.cpp" "DisplayBanner();" "")
replace_string_in_file("${_BOOTGEN_SOURCE_DIR}/main.cpp" "LOG_MSG(\"\\n[INFO] : Bootimage generated successfully\\n\");" "")

file(GLOB _bootgen_sources "${_BOOTGEN_SOURCE_DIR}/*.c" "${_BOOTGEN_SOURCE_DIR}/*.cpp")
add_library(iree-aie-bootgen STATIC ${_bootgen_sources})
Expand Down
21 changes: 11 additions & 10 deletions cmake/iree_aie_rt.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -14,16 +14,17 @@ endif()
# cdo-drver
# ##############################################################################

set(_BOOTGEN_SOURCE_DIR ${IREE_AMD_AIE_SOURCE_DIR}/third_party/bootgen)
replace_string_in_file(${_BOOTGEN_SOURCE_DIR}/cdo-driver/cdo_driver.c "\"(Write64)" "\"cdo-driver: (Write64)")
replace_string_in_file(${_BOOTGEN_SOURCE_DIR}/cdo-driver/cdo_driver.c "\"(MaskWrite64)" "\"cdo-driver: (MaskWrite64)")
replace_string_in_file(${_BOOTGEN_SOURCE_DIR}/cdo-driver/cdo_driver.c "\"(NOP Command)" "\"cdo-driver: (NOP Command)")
replace_string_in_file(${_BOOTGEN_SOURCE_DIR}/cdo-driver/cdo_driver.c "\"(BlockWrite-DMAWriteCmd)" "\"cdo-driver: (BlockWrite-DMAWriteCmd)")
replace_string_in_file(${_BOOTGEN_SOURCE_DIR}/cdo-driver/cdo_driver.c "Data@ 0x%\" PRIxPTR \"" "Data")
replace_string_in_file(${_BOOTGEN_SOURCE_DIR}/cdo-driver/cdo_driver.c "(uintptr_t)(pData + i)," "")
replace_string_in_file(${_BOOTGEN_SOURCE_DIR}/cdo-driver/cdo_driver.c "\" Address:" "\"cdo-driver: Address:")
replace_string_in_file(${_BOOTGEN_SOURCE_DIR}/cdo-driver/cdo_driver.c "\"(BlockSet-DMAWriteCmd)" "\"cdo-driver: (BlockSet-DMAWriteCmd)")
replace_string_in_file(${_BOOTGEN_SOURCE_DIR}/cdo-driver/cdo_driver.c "\"(MaskPoll64)" "\"cdo-driver: (MaskPoll64)")
set(_BOOTGEN_SOURCE_DIR "${IREE_AMD_AIE_SOURCE_DIR}/third_party/bootgen")
replace_string_in_file("${_BOOTGEN_SOURCE_DIR}/cdo-driver/cdo_driver.c" "\"(Write64)" "\"cdo-driver: (Write64)")
replace_string_in_file("${_BOOTGEN_SOURCE_DIR}/cdo-driver/cdo_driver.c" "\"(MaskWrite64)" "\"cdo-driver: (MaskWrite64)")
replace_string_in_file("${_BOOTGEN_SOURCE_DIR}/cdo-driver/cdo_driver.c" "\"(NOP Command)" "\"cdo-driver: (NOP Command)")
replace_string_in_file("${_BOOTGEN_SOURCE_DIR}/cdo-driver/cdo_driver.c" "\"(BlockWrite-DMAWriteCmd)" "\"cdo-driver: (BlockWrite-DMAWriteCmd)")
replace_string_in_file("${_BOOTGEN_SOURCE_DIR}/cdo-driver/cdo_driver.c" "Data@ 0x%\" PRIxPTR \"" "Data")
replace_string_in_file("${_BOOTGEN_SOURCE_DIR}/cdo-driver/cdo_driver.c" "(uintptr_t)(pData + i)," "")
replace_string_in_file("${_BOOTGEN_SOURCE_DIR}/cdo-driver/cdo_driver.c" "\" Address:" "\"cdo-driver: Address:")
replace_string_in_file("${_BOOTGEN_SOURCE_DIR}/cdo-driver/cdo_driver.c" "\"(BlockSet-DMAWriteCmd)" "\"cdo-driver: (BlockSet-DMAWriteCmd)")
replace_string_in_file("${_BOOTGEN_SOURCE_DIR}/cdo-driver/cdo_driver.c" "\"(MaskPoll64)" "\"cdo-driver: (MaskPoll64)")
replace_string_in_file("${_BOOTGEN_SOURCE_DIR}/cdo-driver/cdo_driver.c" "printf(\"Generating: %s\\n\", cdoFileName);" "")

add_library(cdo_driver STATIC ${_BOOTGEN_SOURCE_DIR}/cdo-driver/cdo_driver.c)
target_include_directories(cdo_driver PUBLIC ${_BOOTGEN_SOURCE_DIR}/cdo-driver)
Expand Down
6 changes: 4 additions & 2 deletions cmake/iree_aie_xrt.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@ set(IREE_AIE_BOOST_LIBS
any
algorithm
asio
exception
format
functional
lexical_cast
Expand Down Expand Up @@ -176,7 +175,10 @@ foreach(_core_lib IN LISTS _core_libs)
${IREE_XRT_SOURCE_DIR}/runtime_src)
target_include_directories(${_core_lib} SYSTEM PUBLIC
${IREE_XRT_SOURCE_DIR}/runtime_src/core/common/elf)
target_compile_definitions(${_core_lib} PUBLIC -DBOOST_BIND_GLOBAL_PLACEHOLDERS)
target_compile_definitions(${_core_lib}
PUBLIC
-DBOOST_BIND_GLOBAL_PLACEHOLDERS
$<$<CONFIG:Debug>:-DXRT_VERBOSE>)
target_compile_options(${_core_lib}
PRIVATE
$<$<PLATFORM_ID:Linux>:-fexceptions -frtti -w>
Expand Down
Loading

0 comments on commit 4191152

Please sign in to comment.