Skip to content

Commit

Permalink
Rework the build_package pipeline to use the new onestop scripts. (#8935
Browse files Browse the repository at this point in the history 
)

* Temporarily removes the windows build (because I haven't written the scripts for it yet).
* Should be ~the same on Linux. On MacOS, this *should* yield us shiny new universal wheels.
* MacOS compiler build time should increase to ~4h, but this is probably still less than long tail Linux TF build time.
* Use 'override_python_versions' vs 'python_versions' to avoid conflict with the latter in MacOS GA env.
* Be more verbose about deletions and contents.
  • Loading branch information
@stellaraccident
stellaraccident authored Apr 20, 2022
1 parent 341150b commit 6e95378
Show file tree
Hide file tree
Showing 3 changed files with 93 additions and 105 deletions.
170 changes: 71 additions & 99 deletions .github/workflows/build_package.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,91 +32,56 @@ jobs:
matrix:
include:
# Ubuntu packages.
- os: ubuntu-18.04
- os: ubuntu-20.04
build_family: linux
build_package: main-dist-linux
experimental: false
- os: ubuntu-18.04
- os: ubuntu-20.04
build_family: linux
build_package: py-compiler-pkg
experimental: false
- os: ubuntu-18.04
- os: ubuntu-20.04
build_family: linux
build_package: py-runtime-pkg
experimental: false
- os: ubuntu-18.04
build_package: instrumented-py-runtime-pkg
experimental: true
- os: ubuntu-18.04
- os: ubuntu-20.04
build_family: linux
build_package: py-tf-compiler-tools-pkg
experimental: false

# Windows packages.
- os: windows-2019
build_package: py-compiler-pkg
experimental: true
- os: windows-2019
build_package: py-runtime-pkg
experimental: true

# Macos packages.
- os: macos-latest
build_family: macos
build_package: py-compiler-pkg
experimental: true
- os: macos-latest
build_family: macos
build_package: py-runtime-pkg
experimental: true
env:
CIBW_BUILD_VERBOSITY: 1

# Note that on Linux, we run under docker with an altered path.
# Note that on Windows, we need to configure the compiler api project to
# but its CMake build directory on a short path to avoid path overflow.
CIBW_ENVIRONMENT_LINUX: "REPO_DIR=/project/main_checkout BINDIST_DIR=/output CMAKE_GENERATOR=Ninja IREE_TARGET_BACKEND_CUDA=ON IREE_HAL_DRIVER_CUDA=ON"
CIBW_ENVIRONMENT_MACOS: "REPO_DIR=${{ github.workspace }}/main_checkout CMAKE_GENERATOR=Ninja"
CIBW_ENVIRONMENT_WINDOWS: "REPO_DIR='${{ github.workspace }}\\main_checkout' CMAKE_GENERATOR=Ninja IREE_COMPILER_API_CMAKE_BUILD_DIR=D:/b"

# Needs the bazel manylinux image.
# TODO: Move this to our repo and pin.
# TODO: update to Bazel 5.1.0
CIBW_MANYLINUX_X86_64_IMAGE: stellaraccident/manylinux2014_x86_64-bazel-5.1.0:latest
# CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014

CIBW_BUILD: "cp37-* cp38-* cp39-* cp310-*"
CIBW_SKIP: "*-win32 *-manylinux_i686"
MANYLINUX_X86_64_IMAGE: stellaraccident/manylinux2014_x86_64-bazel-5.1.0:latest

steps:
- uses: actions/checkout@v2
with:
path: 'main_checkout'
submodules: true

# Make sure we have a system python before setting up the dev path.
- uses: actions/setup-python@v2
name: Install Python
with:
python-version: '3.8'

# It is 2021. And the Windows Path is still a mess.
- name: Set up visual studio environment
if: "contains(matrix.os, 'windows')"
shell: powershell
run: |
${{ github.workspace }}\main_checkout\build_tools\github_actions\configure_dev_environment.ps1 -bashExePath C:\msys64\usr\bin\bash.exe
- name: Report windows environment
if: "contains(matrix.os, 'windows')"
##########################################################################
# OS specific setup
##########################################################################
- name: Install MacOS Deps
if: "matrix.build_family == 'macos'"
shell: bash
run: |
# Should display either an msys mount table or an "install from the windows store" banner
echo "--- System path:"
echo "$PATH"
echo "--- Result of asking bash to run 'mount' (should show msys mounts):"
mount
sudo ./main_checkout/build_tools/python_deploy/install_macos_deps.sh
- name: Install cibuildwheel
shell: bash
run: |
python -m pip install cibuildwheel==1.7.2
##########################################################################
# Write version_info.json
# Various tools will read this in order to embed release information.
##########################################################################
- name: Write version info (release)
if: "!startsWith(matrix.build_package, 'instrumented-')"
shell: bash
run: |
cat << EOF > ./main_checkout/version_info.json
Expand All @@ -128,19 +93,8 @@ jobs:
EOF
cat ./main_checkout/version_info.json
- name: Write version info (instrumented)
if: "startsWith(matrix.build_package, 'instrumented-')"
shell: bash
run: |
cat << EOF > ./main_checkout/version_info.json
{
"package-suffix": "-instrumented${{ github.event.inputs.package_suffix }}",
"package-version": "${{ github.event.inputs.package_version }}",
"iree-revision": "$(cd ./main_checkout && git rev-parse HEAD)"
}
EOF
cat ./main_checkout/version_info.json
##########################################################################
# Build the main distribution tarball.
# The main distribution consists of the project being built, installed
# and archived. We have to split it per operating system, and Linux
# is special because we build under a manylinux container which gives
Expand All @@ -150,62 +104,79 @@ jobs:
# python versions under /opt/python (need to add one to the path). It
# is not enough to just invoke it directly because then pip managed
# console scripts (like cmake, ninja) will not be on the path.
##########################################################################
- name: Main distribution (Linux)
if: "matrix.build_package == 'main-dist-linux'"
shell: bash
run: |
docker run --rm -w=/work \
-v $PWD:/work \
"${CIBW_MANYLINUX_X86_64_IMAGE}" \
"${MANYLINUX_X86_64_IMAGE}" \
bash -c 'export PATH=/opt/python/cp39-cp39/bin:$PATH; python ./main_checkout/build_tools/github_actions/build_dist.py main-dist'
# Runtime wheels are version specific, so we build one for each python
# version. We do this serially by deleting the CMakeCache and install
# directory between runs. Most of the build will be incremental.
# We save a lot of time by disabling the compiler build.
- name: Build runtime wheels
if: "matrix.build_package == 'py-runtime-pkg'"
##########################################################################
# py-runtime-pkg
# Builds the iree-runtime and iree-runtime-instrumented wheels.
# One step per OS.
##########################################################################

- name: Build runtime wheels (Linux)
if: "matrix.build_package == 'py-runtime-pkg' && matrix.build_family == 'linux'"
shell: bash
run: |
export CIBW_BEFORE_BUILD="python -m pip install --upgrade pip>=21.3"
python -m cibuildwheel --output-dir bindist ./runtime/
packages="iree-runtime iree-runtime-instrumented" \
output_dir="$PWD/bindist" \
./main_checkout/build_tools/python_deploy/build_linux_packages.sh
- name: Build runtime wheels (instrumented)
if: "matrix.build_package == 'instrumented-py-runtime-pkg'"
- name: Build runtime wheels (MacOS)
if: "matrix.build_package == 'py-runtime-pkg' && matrix.build_family == 'macos'"
shell: bash
run: |
export CIBW_BEFORE_ALL_LINUX="./main_checkout/build_tools/github_actions/install_tracy_cli_deps_manylinux2014.sh"
export CIBW_BEFORE_BUILD="python -m pip install --upgrade pip>=21.3"
# I really dislike how cibuildwheel and GitHub actions make tunneling
# environment variables so obtuse. Burn this all with fire.
export CIBW_ENVIRONMENT_LINUX="$CIBW_ENVIRONMENT_LINUX IREE_BUILD_TRACY=ON IREE_ENABLE_RUNTIME_TRACING=ON"
export CIBW_ENVIRONMENT_MACOS="$CIBW_ENVIRONMENT_LINUX IREE_BUILD_TRACY=ON IREE_ENABLE_RUNTIME_TRACING=ON"
export CIBW_ENVIRONMENT_WINDOWS="$CIBW_ENVIRONMENT_LINUX IREE_BUILD_TRACY=ON IREE_ENABLE_RUNTIME_TRACING=ON"
python -m cibuildwheel --output-dir bindist ./runtime/
# Experimental iree.compiler package.
- name: Build compiler wheels
if: "matrix.build_package == 'py-compiler-pkg'"
packages="iree-runtime iree-runtime-instrumented" \
output_dir="$PWD/bindist" \
./main_checkout/build_tools/python_deploy/build_macos_packages.sh
##########################################################################
# py-compiler-pkg
# Builds the iree-compielr wheel.
# One step per OS.
##########################################################################
- name: Build compiler wheels (Linux)
if: "matrix.build_package == 'py-compiler-pkg' && matrix.build_family == 'linux'"
shell: bash
run: |
# In pip 21.3, in-tree builds became the default and only way to
# build. We require that and make sure to be past that threshold.
export CIBW_BEFORE_BUILD="python -m pip install --upgrade pip>=21.3"
python -m cibuildwheel --output-dir bindist ./main_checkout/iree/compiler
packages="iree-compiler" \
output_dir="$PWD/bindist" \
./main_checkout/build_tools/python_deploy/build_linux_packages.sh
- name: Build compiler wheels (MacOS)
if: "matrix.build_package == 'py-compiler-pkg' && matrix.build_family == 'macos'"
shell: bash
run: |
packages="iree-compiler" \
output_dir="$PWD/bindist" \
./main_checkout/build_tools/python_deploy/build_macos_packages.sh
##########################################################################
# TF Compiler Tools
# Compiler tools wheels are not python version specific, so just build
# for one examplar python version.
# TODO: This currently only builds on Linux and is hardcoded to do so.
##########################################################################

- name: Build TF Compiler Tools wheels
if: "matrix.build_package == 'py-tf-compiler-tools-pkg'"
shell: bash
run: |
docker run --rm -w=/work \
-v $PWD:/work \
"${CIBW_MANYLINUX_X86_64_IMAGE}" \
"${MANYLINUX_X86_64_IMAGE}" \
bash -c 'export PATH=/opt/python/cp39-cp39/bin:$PATH; python ./main_checkout/build_tools/github_actions/build_dist.py py-tf-compiler-tools-pkg'
- uses: actions/upload-artifact@v2
with:
# We upload all wheels (which includes deps so that subsequent
# steps can run without further fetching).
path: ./bindist/*
retention-days: 5

Expand All @@ -218,12 +189,13 @@ jobs:
GITHUB_TOKEN: ${{ secrets.WRITE_ACCESS_TOKEN }}
with:
release_id: ${{ github.event.inputs.release_id }}
assets_path: ./bindist/*
# Only upload iree wheels.
assets_path: ./bindist/iree*.whl

validate_and_publish:
name: "Invoke workflow to validate and publish release"
needs: build_core
runs-on: ubuntu-18.04
runs-on: ubuntu-20.04
steps:
- name: "Invoke workflow :: Validate and Publish Release"
uses: benc-uk/workflow-dispatch@v1
Expand Down
16 changes: 12 additions & 4 deletions build_tools/python_deploy/build_linux_packages.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
# ./build_tools/python_deploy/build_linux_packages.sh
#
# Build specific Python versions and packages to custom directory:
# python_versions="cp38-cp38 cp39-cp39" \
# override_python_versions="cp38-cp38 cp39-cp39" \
# packages="iree-runtime iree-runtime-instrumented" \
# output_dir="/tmp/wheelhouse" \
# ./build_tools/python_deploy/build_linux_packages.sh
Expand Down Expand Up @@ -44,13 +44,17 @@ script_name="$(basename $0)"
repo_root="$(cd $this_dir/../../ && pwd)"
script_name="$(basename $0)"
manylinux_docker_image="${manylinux_docker_image:-stellaraccident/manylinux2014_x86_64-bazel-5.1.0:latest}"
python_versions="${python_versions:-cp37-cp37m cp38-cp38 cp39-cp39 cp310-cp310}"
python_versions="${override_python_versions:-cp37-cp37m cp38-cp38 cp39-cp39 cp310-cp310}"
output_dir="${output_dir:-${this_dir}/wheelhouse}"
packages="${packages:-iree-runtime iree-runtime-instrumented iree-compiler}"

function run_on_host() {
echo "Running on host"
echo "Launching docker image ${manylinux_docker_image}"

# Canonicalize paths.
mkdir -p "$output_dir"
output_dir="$(cd $output_dir && pwd)"
echo "Outputting to ${output_dir}"
mkdir -p "${output_dir}"
docker run --rm \
Expand All @@ -61,6 +65,10 @@ function run_on_host() {
-e "packages=${packages}" \
${manylinux_docker_image} \
-- bash /main_checkout/iree/build_tools/python_deploy/build_linux_packages.sh

echo "******************** BUILD COMPLETE ********************"
echo "Generated binaries:"
ls -l $output_dir
}

function run_in_docker() {
Expand Down Expand Up @@ -136,14 +144,14 @@ function run_audit_wheel() {
generic_wheel="/wheelhouse/${wheel_basename}-*-${python_version}-linux_x86_64.whl"
echo ":::: Auditwheel $generic_wheel"
auditwheel repair -w /wheelhouse $generic_wheel
rm $generic_wheel
rm -v $generic_wheel
}

function clean_wheels() {
local wheel_basename="$1"
local python_version="$2"
echo ":::: Clean wheels $wheel_basename $python_version"
rm -f /wheelhouse/${wheel_basename}-*-${python_version}-*.whl
rm -f -v /wheelhouse/${wheel_basename}-*-${python_version}-*.whl
}

# Trampoline to the docker container if running on the host.
Expand Down
12 changes: 10 additions & 2 deletions build_tools/python_deploy/build_macos_packages.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ set -eu -o errtrace

this_dir="$(cd $(dirname $0) && pwd)"
repo_root="$(cd $this_dir/../../ && pwd)"
python_versions="${python_versions:3.9 3.10}"
python_versions="${override_python_versions:-3.9 3.10}"
output_dir="${output_dir:-${this_dir}/wheelhouse}"
packages="${packages:-iree-runtime iree-runtime-instrumented iree-compiler}"

Expand All @@ -34,6 +34,10 @@ export CMAKE_OSX_ARCHITECTURES="arm64;x86_64"
# cpuinfo is incompatible with universal builds.
export IREE_ENABLE_CPUINFO=OFF

# Canonicalize paths.
mkdir -p "$output_dir"
output_dir="$(cd $output_dir && pwd)"

function run() {
echo "Using python versions: ${python_versions}"

Expand Down Expand Up @@ -70,6 +74,10 @@ function run() {
esac
done
done

echo "******************** BUILD COMPLETE ********************"
echo "Generated binaries:"
ls -l $output_dir
}

function build_iree_runtime() {
Expand All @@ -93,7 +101,7 @@ function clean_wheels() {
local wheel_basename="$1"
local python_version="$2"
echo ":::: Clean wheels $wheel_basename $python_version"
rm -f /wheelhouse/${wheel_basename}-*-${python_version}-*.whl
rm -f -v /wheelhouse/${wheel_basename}-*-${python_version}-*.whl
}

run

0 comments on commit 6e95378

Please sign in to comment.