Move llvm-external-projects/iree-compiler-api -> iree/compiler/API. (#…

…8911)

* Move llvm-external-projects/iree-compiler-api -> iree/compiler/Api.

Non-functional changes:

* Relayers the build so that we can simply delegate to the LLVM macros for C/Python API building.
* Various escalation of include hack battle.
* Brings the compile_sample_module.py test from a sample which did not run to a real test.
* Reworks tests to be based on IREE macros (as possible). No more magic.
* Retired the old build/install locations for this piece: PYTHONPATH=${iree_build}/bindings/python gets everything now.

Functional changes:

* Since this is "in-tree" now, includes are laid out per the rest of the project. So:
  * `#include <iree-compiler-c/Compiler.h>` -> `#include <iree/compiler/Api/Compiler.h>
* Adds `iree.compiler.transforms.ireec.register_all_dialects(context)`, needed to make the `compile_sample_module.py` test work (Fixes #8604).
* iree/compiler/setup.py should now work from both the source and build dir. If using from the latter, it will use in-situ artifacts (will not rebuild).

In a subsequent change, I will do a similar refactoring for the Python runtime API and update user docs. For now the iree/compiler/README.md has instructions for building/installing a Python package from source (we've never had documentation for this, so something > nothing and we can move it around later).

For integration into internal Google, this shouldn't require more than some path changes from `llvm-external-projects/iree-compiler-api` -> `iree/compiler/Api` in BUILD files.

Fixes #8597
Progress on #6958
Progress on #3817

* Silence pytype for setup.py and remove obsolte path from CI.

* Re-enable namespace package auto-detection.

* Fix typo in .env file (should fix integrations tests)

* Rename Api -> API

* Fix one more Api -> API rename

* Address comments

.github/workflows/build_package.yml

@@ -194,7 +194,7 @@ jobs:
           # In pip 21.3, in-tree builds became the default and only way to
           # build. We require that and make sure to be past that threshold.
           export CIBW_BEFORE_BUILD="python -m pip install --upgrade pip>=21.3"
-          python -m cibuildwheel --output-dir bindist ./main_checkout/llvm-external-projects/iree-compiler-api
+          python -m cibuildwheel --output-dir bindist ./main_checkout/iree/compiler
 
       # Compiler tools wheels are not python version specific, so just build
       # for one examplar python version.

.gitignore

@@ -49,6 +49,9 @@ compile_commands.json
 *.venv
 .venv
 
+# Python deployment artifacts
+*.whl
+
 # Android Studio / Gradle files
 .cxx
 .gradle

CMakeLists.txt

@@ -476,12 +476,23 @@ else()
     ${CMAKE_CURRENT_BINARY_DIR}/third_party/llvm-project/llvm/tools/mlir/include
   )
 
+  # TODO: It should be possible to fix upstream targets so as to not require
+  # any of these. Remove as they become unnecessary.
   function(_hack_llvm_include_paths)
     set(_common_include_dirs
+      # LLVM
       $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/third_party/llvm-project/llvm/include>
       $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/third_party/llvm-project/llvm/include>
+
+      # MLIR
       $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/third_party/llvm-project/mlir/include>
       $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/third_party/llvm-project/llvm/tools/mlir/include>
+
+      # LLD
+      $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/third_party/llvm-project/lld/include>
+      $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/third_party/llvm-project/llvm/tools/lld/include>
+
+      # MHLO
       $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/third_party/llvm-project/llvm/tools/mlir-hlo/include>
     )
     # Avoid globally modifying paths by instead adding the include paths to the
@@ -645,6 +656,16 @@ endif()
 # and tracy, and must come after it.
 if(${IREE_BUILD_PYTHON_BINDINGS})
   add_subdirectory(bindings/python)
+  # Write out a .env file to make IDEs and developers happy.
+  # Yes, we are writing this to the source dir. It is only for IDEs and if
+  # it gets clobbered, it is fine.
+  set(_pythonpath_env "PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR}/bindings/python\n")
+  file(GENERATE OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/.env"
+    CONTENT "${_pythonpath_env}"
+  )
+  file(GENERATE OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/.env"
+    CONTENT "${_pythonpath_env}"
+  )
 endif()
 
 if(${IREE_BUILD_BINDINGS_TFLITE})
@@ -673,36 +694,6 @@ endif()
 set(IREE_PUBLIC_INCLUDE_DIRS "${IREE_COMMON_INCLUDE_DIRS}"
     CACHE INTERNAL "IREE: Include Directories" FORCE)
 
-# Include the iree-compiler-api sub-project. We do this for development
-# and CI so that developers have access to the API and tools it provides
-# (otherwise, they would need to build multiple top-level projects).
-# However, logically, iree-compiler-api depends on iree, and for deployment
-# is always built standalone, taking responsibility to include iree and LLVM
-# as sub-projects.
-# The dependency mode is controlled by the variables
-# IREE_COMPILER_API_STANDALONE, which will be set if iree-compiler-api is
-# top-level. Otherwise, we set IREE_COMPILER_API_SUB_PROJECT, indicating it
-# is being embedded as a sub project.
-if(IREE_BUILD_COMPILER AND IREE_BUILD_PYTHON_BINDINGS)
-  if(NOT IREE_COMPILER_API_STANDALONE)
-    message(STATUS "Including iree-compiler-api as a sub-project")
-    set(IREE_COMPILER_API_SUB_PROJECT ON)
-    add_subdirectory(llvm-external-projects/iree-compiler-api
-        "${CMAKE_CURRENT_BINARY_DIR}/compiler-api")
-
-    # Write out a .env file to make IDEs and developers happy.
-    # Yes, we are writing this to the source dir. It is only for IDEs and if
-    # it gets clobbered, it is fine.
-    set(_pythonpath_env "PYTHONPATH=$<SHELL_PATH:${CMAKE_CURRENT_BINARY_DIR}/compiler-api/python_package;${CMAKE_CURRENT_BINARY_DIR}/bindings/python>\n")
-    file(GENERATE OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/.env"
-      CONTENT "${_pythonpath_env}"
-    )
-    file(GENERATE OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/.env"
-      CONTENT "${_pythonpath_env}"
-    )
-  endif()
-endif()
-
 #-------------------------------------------------------------------------------
 # IREE benchmark tools
 #-------------------------------------------------------------------------------

build_tools/bazel/build_core.sh

@@ -84,7 +84,6 @@ bazel \
   query \
     --config=non_darwin \
     //iree/... + //build_tools/... + \
-    //llvm-external-projects/iree-compiler-api/... + \
     //llvm-external-projects/iree-dialects/... | \
       xargs --max-args 1000000 --max-chars 1000000 --exit \
         bazel \

build_tools/cmake/iree_python.cmake

@@ -336,7 +336,7 @@ function(iree_py_test)
 
   set_property(TEST ${_NAME_PATH} PROPERTY LABELS "${_RULE_LABELS}")
   set_property(TEST ${_NAME_PATH} PROPERTY ENVIRONMENT
-      "PYTHONPATH=${IREE_BINARY_DIR}/compiler-api/python_package:${IREE_BINARY_DIR}/bindings/python:$ENV{PYTHONPATH}"
+      "PYTHONPATH=${IREE_BINARY_DIR}/bindings/python:$ENV{PYTHONPATH}"
       "TEST_TMPDIR=${IREE_BINARY_DIR}/tmp/${_NAME}_test_tmpdir"
   )
   iree_add_test_environment_properties(${_NAME_PATH})

build_tools/pytype/check_diff.sh

@@ -26,7 +26,9 @@ fi
 
 # We seperate the python files into multiple pytype calls because otherwise
 # Ninja gets confused. See https://github.com/google/pytype/issues/198
-BASE=$(echo "${FILES?}" | grep -vP '^(\./)?integrations/*$')
+BASE=$(echo "${FILES?}" | \
+       grep -vP '^(\./)?integrations/*$' | \
+       grep -vP '(\./)?setup\.py$')
 IREE_TF=$(echo "${FILES?}" | \
           grep -P '^(\./)?integrations/tensorflow/bindings/python/iree/tf/.*')
 IREE_XLA=$(echo "${FILES?}" | \

llvm-external-projects/iree-compiler-api/BUILD.bazel → iree/compiler/API/BUILD

@@ -37,13 +37,13 @@ filegroup(
 
 filegroup(
     name = "python_compiler_api",
-    srcs = glob(["python/iree/compiler/api/*.py"]),
+    srcs = glob(["python/iree/compiler/transforms/*.py"]),
 )
 
 filegroup(
     name = "unittests_tools_testdata",
     srcs = glob([
-        "unittests/tools/testdata/*",
+        "python/test/tools/testdata/*",
     ]),
 )
 
@@ -54,15 +54,14 @@ filegroup(
 cc_library(
     name = "CAPI",
     srcs = [
-        "lib/CAPI/Compiler.cpp",
-        "lib/CAPI/Lld.cpp",
-        "lib/CAPI/Tools.cpp",
+        "Compiler.cpp",
+        "Lld.cpp",
+        "Tools.cpp",
     ],
     hdrs = [
-        "include/iree-compiler-c/Compiler.h",
-        "include/iree-compiler-c/Tools.h",
+        "Compiler.h",
+        "Tools.h",
     ],
-    includes = ["include"],
     deps = [
         "//iree/compiler/ConstEval",
         "//iree/compiler/Dialect/VM/IR",

llvm-external-projects/iree-compiler-api/lib/CAPI/CMakeLists.txt → iree/compiler/API/CMakeLists.txt

@@ -1,3 +1,22 @@
+# Copyright 2022 The IREE Authors
+#
+# Licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+################################################################################
+# CAPI library using the LLVM build system.
+#
+# WARNING WILL ROBINSON!
+# This does not look like the rest of IREE. It is directly using the upstream
+# LLVM build system in order to create bundled compiler API binaries that
+# are consistent with LLVM. Consult upstream CMake macros if you don't
+# understand what this does.
+################################################################################
+
+include(AddLLVM)
+include(AddMLIR)
+
 # LLD may not have been enabled at the global level (typically because we
 # are not building for a CPU backend). If not, don't add API support for it.
 set(_OPTIONAL_SOURCES)
@@ -14,7 +33,7 @@ if(TARGET lld)
   )
 endif()
 
-add_mlir_public_c_api_library(IREECompilerAPICompilerCAPI
+add_mlir_public_c_api_library(IREECompilerCAPILib
   Compiler.cpp
   Tools.cpp
   ${_OPTIONAL_SOURCES}
@@ -46,12 +65,19 @@ add_mlir_public_c_api_library(IREECompilerAPICompilerCAPI
 )
 
 if(IREE_BUILD_TORCH_MLIR_SUPPORT)
-  target_compile_definitions(obj.IREECompilerAPICompilerCAPI
+  target_compile_definitions(obj.IREECompilerCAPILib
     PUBLIC
     IREE_HAVE_TORCH_MLIR_DIALECTS
   )
 endif()
 
-# TODO: Fix upstream so there is a way to know what the actual compile target
-# is (versus prefixing with "obj." which is conditional).
-iree_compiler_target_includes(obj.IREECompilerAPICompilerCAPI)
+add_subdirectory(test)
+
+################################################################################
+# Language specific bindings.
+# These have an ordering dependency on the backing CAPI being defined.
+################################################################################
+
+if(IREE_BUILD_PYTHON_BINDINGS)
+  add_subdirectory(python)
+endif()

llvm-external-projects/iree-compiler-api/lib/CAPI/Compiler.cpp → iree/compiler/API/Compiler.cpp

@@ -4,7 +4,7 @@
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-#include "iree-compiler-c/Compiler.h"
+#include "iree/compiler/API/Compiler.h"
 
 #include "iree/compiler/ConstEval/Passes.h"
 #include "iree/compiler/Dialect/VM/IR/VMOps.h"

llvm-external-projects/iree-compiler-api/lib/CAPI/Lld.cpp → iree/compiler/API/Lld.cpp

@@ -12,7 +12,7 @@
 #include <cstdlib>
 #include <vector>
 
-#include "iree-compiler-c/Tools.h"
+#include "iree/compiler/API/Tools.h"
 #include "lld/Common/Driver.h"
 #include "lld/Common/ErrorHandler.h"
 #include "lld/Common/Memory.h"

llvm-external-projects/iree-compiler-api/lib/CAPI/Tools.cpp → iree/compiler/API/Tools.cpp

@@ -4,7 +4,7 @@
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-#include "iree-compiler-c/Tools.h"
+#include "iree/compiler/API/Tools.h"
 
 #include "iree/tools/iree_translate_lib.h"
 

llvm-external-projects/iree-compiler-api/python/CMakeLists.txt → iree/compiler/API/python/CMakeLists.txt

@@ -1,9 +1,37 @@
+# Copyright 2022 The IREE Authors
+#
+# Licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
 include(AddMLIRPython)
 
 # Specifies that all MLIR packages are co-located under npcomp.
 # TODO: Add an upstream cmake param for this vs having a global here.
 add_compile_definitions("MLIR_PYTHON_PACKAGE_PREFIX=iree.compiler.")
 
+set(_PYTHON_BUILD_PREFIX "${IREE_BINARY_DIR}/bindings/python")
+set(_PYTHON_INSTALL_PREFIX "python_packages/iree-compiler")
+
+# HACK: This should not be necessary, but add_mlir_python_extension is
+# accidentally closing over an errant include_directories from up-tree, so
+# when built in-tree it is somehow working based on that. This will need
+# to be fixed to capture the correct include directories in that macro.
+include_directories(
+  "${IREE_SOURCE_DIR}"
+  "${IREE_SOURCE_DIR}/llvm-external-projects/iree-dialects/include"
+  "${IREE_SOURCE_DIR}/third_party/llvm-project/mlir/include"
+  "${IREE_SOURCE_DIR}/third_party/mlir-hlo/include"
+)
+
+# On Unixes, disable the creation of versioned/symlinked `.so` files. With
+# this set, we just generate libIREECompilerAggregateCAPI.so vs making that
+# a symlink to a versioned file right next to it. When packaging for Python,
+# symlinks are duplicated, so this is pretty important. It is usually set at
+# a toolchain level for dedicated Python builds, but is nice to override since
+# it makes the build directory suitable for directly packaging.
+set(CMAKE_PLATFORM_NO_VERSIONED_SONAME 1)
+
 ################################################################################
 # Sources
 ################################################################################
@@ -36,7 +64,7 @@ declare_mlir_python_extension(IREECompilerAPIPythonExtensions.IREECTransforms
   SOURCES
     IREECTransforms.cpp
   EMBED_CAPI_LINK_LIBS
-    IREECompilerAPICompilerCAPI
+    IREECompilerCAPILib
   PRIVATE_LINK_LIBS
     LLVMSupport
 )
@@ -82,15 +110,15 @@ set(_source_components
 
 add_mlir_python_common_capi_library(IREECompilerAggregateCAPI
   INSTALL_COMPONENT IREECompilerPythonModules
-  INSTALL_DESTINATION python_package/iree/compiler/_mlir_libs
-  OUTPUT_DIRECTORY "${IREE_COMPILER_API_BINARY_DIR}/python_package/iree/compiler/_mlir_libs"
+  INSTALL_DESTINATION ${_PYTHON_INSTALL_PREFIX}/iree/compiler/_mlir_libs
+  OUTPUT_DIRECTORY "${_PYTHON_BUILD_PREFIX}/iree/compiler/_mlir_libs"
   RELATIVE_INSTALL_ROOT "../../../.."
   DECLARED_SOURCES ${_source_components}
 )
 
 add_mlir_python_modules(IREECompilerPythonModules
-  ROOT_PREFIX "${IREE_COMPILER_API_BINARY_DIR}/python_package/iree/compiler"
-  INSTALL_PREFIX "python_package/iree/compiler"
+  ROOT_PREFIX "${_PYTHON_BUILD_PREFIX}/iree/compiler"
+  INSTALL_PREFIX "${_PYTHON_INSTALL_PREFIX}/iree/compiler"
   DECLARED_SOURCES ${_source_components}
   COMMON_CAPI_LINK_LIBS
     IREECompilerAggregateCAPI
@@ -116,12 +144,12 @@ function(add_iree_compiler_busybox_tool target)
   set_target_properties(${target}
     PROPERTIES
       OUTPUT_NAME "${ARG_OUTPUT_NAME}"
-      RUNTIME_OUTPUT_DIRECTORY "${IREE_COMPILER_API_BINARY_DIR}/python_package/iree/compiler/_mlir_libs"
+      RUNTIME_OUTPUT_DIRECTORY "${_PYTHON_BUILD_PREFIX}/iree/compiler/_mlir_libs"
   )
   mlir_python_setup_extension_rpath(${target})
   add_dependencies(IREECompilerPythonModules ${target})
   install(TARGETS ${target}
-    DESTINATION "python_package/iree/compiler/_mlir_libs"
+    DESTINATION "${_PYTHON_INSTALL_PREFIX}/iree/compiler/_mlir_libs"
   )
 endfunction()
 
@@ -140,3 +168,9 @@ if(TARGET lld)
       LldTool.c
   )
 endif()
+
+################################################################################
+# Subdirectories
+################################################################################
+
+add_subdirectory(test)

llvm-external-projects/iree-compiler-api/python/IREECTransforms.cpp → iree/compiler/API/python/IREECTransforms.cpp

@@ -4,7 +4,7 @@
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-#include "iree-compiler-c/Compiler.h"
+#include "iree/compiler/API/Compiler.h"
 #include "mlir-c/Bindings/Python/Interop.h"
 #include "mlir/Bindings/Python/PybindAdaptors.h"
 
@@ -142,6 +142,9 @@ PYBIND11_MODULE(_ireecTransforms, m) {
   m.doc() = "ireec transforms API";
   ireeCompilerRegisterTargetBackends();
 
+  m.def("register_all_dialects",
+        [](MlirContext context) { ireeCompilerRegisterAllDialects(context); });
+
   py::class_<PyCompilerOptions>(m, "CompilerOptions",
                                 "Options for the IREE backend compiler.")
       .def(py::init([](py::args args) {

llvm-external-projects/iree-compiler-api/python/IREECompileTool.c → iree/compiler/API/python/IREECompileTool.c

@@ -4,6 +4,6 @@
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-#include "iree-compiler-c/Tools.h"
+#include "iree/compiler/API/Tools.h"
 
 int main(int argc, char **argv) { return ireeCompilerRunMain(argc, argv); }

llvm-external-projects/iree-compiler-api/python/LldTool.c → iree/compiler/API/python/LldTool.c

@@ -4,6 +4,6 @@
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 
-#include "iree-compiler-c/Tools.h"
+#include "iree/compiler/API/Tools.h"
 
 int main(int argc, char **argv) { return ireeCompilerRunLldMain(argc, argv); }

iree/compiler/API/python/test/CMakeLists.txt

@@ -0,0 +1,8 @@
+# Copyright 2022 The IREE Authors
+#
+# Licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+add_subdirectory(tools)
+add_subdirectory(transforms)