From 46884d5e108cb7f2294734944e44007060336aca Mon Sep 17 00:00:00 2001 From: Pablo Alonso Date: Tue, 24 Oct 2023 18:12:41 +0200 Subject: [PATCH 01/45] Add changes to support building arm64 macos wheels In theory, Apple Silicon wheels can be cross-compiled from the intel mac machines (however the can't be tested): https://cibuildwheel.readthedocs.io/en/stable/faq/#apple-silicon Additionally, we replace `os.system` calls by `subprocess.run` so that we can notice when one of these subprocesses fail. --- .github/workflows/build-wheels-cibuildwheel.yml | 2 ++ pyproject-tensorflow.toml | 13 +++++++++++++ pyproject.toml | 11 +++++++++++ setup.py | 15 ++++++++------- wscript | 5 +++++ 5 files changed, 39 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-wheels-cibuildwheel.yml b/.github/workflows/build-wheels-cibuildwheel.yml index 35517f121..3b840e899 100644 --- a/.github/workflows/build-wheels-cibuildwheel.yml +++ b/.github/workflows/build-wheels-cibuildwheel.yml @@ -22,6 +22,8 @@ jobs: uses: pypa/cibuildwheel@v2.11.4 with: config-file: ${{ matrix.config }} + env: + CIBW_ARCHS_MACOS: x86_64 arm64 - uses: actions/upload-artifact@v3 with: diff --git a/pyproject-tensorflow.toml b/pyproject-tensorflow.toml index 112da2fde..d6336321f 100644 --- a/pyproject-tensorflow.toml +++ b/pyproject-tensorflow.toml @@ -39,6 +39,19 @@ before-all = [ test-command = "python -c 'import essentia; import essentia.standard; import essentia.streaming; from essentia.standard import MonoLoader, MetadataReader, YamlInput, Chromaprinter, TensorflowPredict'" +[[tool.cibuildwheel.overrides]] +select = "*arm64*" + +before-all = [ + "brew install pkg-config gcc readline sqlite gdbm freetype libpng", + "brew install eigen libyaml fftw ffmpeg@2.8 libsamplerate libtag chromaprint", + "brew install tensorflow", + "brew link --force ffmpeg@2.8", + "python waf configure --with-tensorflow --pkg-config-path=\"${PKG_CONFIG_PATH}\" --arch arm64 --no-msse", + "python waf", + "python waf install" +] + [build-system] requires = ["wheel", "setuptools", "oldest-supported-numpy"] diff --git a/pyproject.toml b/pyproject.toml index a943f3371..23e6fd4b4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,17 @@ before-all = [ test-command = "python -c 'import essentia; import essentia.standard; import essentia.streaming; from essentia.standard import MonoLoader, MetadataReader, YamlInput, Chromaprinter'" +[[tool.cibuildwheel.overrides]] +select = "*arm64*" + +before-all = [ + "brew install pkg-config gcc readline sqlite gdbm freetype libpng", + "brew install eigen libyaml fftw ffmpeg@2.8 libsamplerate libtag chromaprint", + "brew link --force ffmpeg@2.8", + "python waf configure --pkg-config-path=\"${PKG_CONFIG_PATH}\" --arch arm64 --no-msse", + "python waf", + "python waf install" +] [build-system] diff --git a/setup.py b/setup.py index 7bb2eab13..edc4a5326 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,7 @@ import shutil import os import glob +import subprocess import sys from setuptools import setup, Extension from setuptools.command.build_ext import build_ext @@ -39,17 +40,17 @@ def run(self): if var_skip_3rdparty in os.environ and os.environ[var_skip_3rdparty]=='1': print('Skipping building static 3rdparty dependencies (%s=1)' % var_skip_3rdparty) else: - os.system('./packaging/build_3rdparty_static_debian.sh') + subprocess.run('./packaging/build_3rdparty_static_debian.sh', check=True) if var_only_python in os.environ and os.environ[var_only_python]=='1': print('Skipping building the core libessentia library (%s=1)' % var_only_python) - os.system('%s waf configure --only-python --static-dependencies ' - '--prefix=tmp' % PYTHON) + subprocess.run([PYTHON, 'waf', 'configure', '--only-python', '--static-dependencies', + '--prefix=tmp'], check=True) else: - os.system('%s waf configure --build-static --static-dependencies ' - '--with-python --prefix=tmp' % PYTHON) - os.system('%s waf' % PYTHON) - os.system('%s waf install' % PYTHON) + subprocess.run([PYTHON, 'waf', 'configure', '--build-static', '--static-dependencies' + '--with-python --prefix=tmp'], check=True) + subprocess.run([PYTHON, 'waf'], check=True) + subprocess.run([PYTHON, 'waf', 'install'], check=True) library = glob.glob('tmp/lib/python*/*-packages/essentia')[0] diff --git a/wscript b/wscript index fd9c5d01c..7970f4aa6 100644 --- a/wscript +++ b/wscript @@ -194,6 +194,11 @@ def configure(ctx): ctx.env.LINKFLAGS += ['-arch', 'i386', '-arch', 'x86_64'] ctx.env.LDFLAGS = ['-arch', 'i386', '-arch', 'x86_64'] + if ctx.options.ARCH == 'arm64': + ctx.env.CXXFLAGS += ['-arch', 'arm64'] + ctx.env.LINKFLAGS += ['-arch', 'arm64'] + ctx.env.LDFLAGS += ['-arch', 'arm64'] + elif sys.platform.startswith('linux'): # include -pthread flag because not all versions of gcc provide it automatically ctx.env.CXXFLAGS += ['-pthread'] From 8fe741c0dc9237681bb735bfb5731cd7800229c0 Mon Sep 17 00:00:00 2001 From: Pablo Alonso Date: Tue, 24 Oct 2023 18:41:20 +0200 Subject: [PATCH 02/45] Replace before-all by before-build --- pyproject-tensorflow.toml | 8 ++++---- pyproject.toml | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pyproject-tensorflow.toml b/pyproject-tensorflow.toml index d6336321f..83684f38f 100644 --- a/pyproject-tensorflow.toml +++ b/pyproject-tensorflow.toml @@ -8,7 +8,7 @@ skip = ["pp*", "*-musllinux*", "*i686"] environment = { PROJECT_NAME="essentia-tensorflow", ESSENTIA_PROJECT_NAME="${PROJECT_NAME}", ESSENTIA_WHEEL_SKIP_3RDPARTY=1, ESSENTIA_WHEEL_ONLY_PYTHON=1 } -before-all = [ +before-build = [ "PYBIN=/opt/python/cp36-cp36m/bin/", "\"${PYBIN}/python\" waf configure --with-gaia --with-tensorflow --build-static --static-dependencies --pkg-config-path=\"${PKG_CONFIG_PATH}\"", "\"${PYBIN}/python\" waf", @@ -24,7 +24,7 @@ skip = ["pp*"] environment = { PROJECT_NAME="essentia-tensorflow", ESSENTIA_PROJECT_NAME="${PROJECT_NAME}", ESSENTIA_WHEEL_SKIP_3RDPARTY=1, ESSENTIA_WHEEL_ONLY_PYTHON=1 } -before-all = [ +before-build = [ "brew install pkg-config gcc readline sqlite gdbm freetype libpng", "brew install eigen libyaml fftw ffmpeg@2.8 libsamplerate libtag chromaprint", "brew install tensorflow", @@ -40,9 +40,9 @@ test-command = "python -c 'import essentia; import essentia.standard; import ess [[tool.cibuildwheel.overrides]] -select = "*arm64*" +select = "*macosx_arm64*" -before-all = [ +before-build = [ "brew install pkg-config gcc readline sqlite gdbm freetype libpng", "brew install eigen libyaml fftw ffmpeg@2.8 libsamplerate libtag chromaprint", "brew install tensorflow", diff --git a/pyproject.toml b/pyproject.toml index 23e6fd4b4..b9c9e27ab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ skip = ["pp*", "*-musllinux*", "*i686"] environment = { PROJECT_NAME="essentia", ESSENTIA_PROJECT_NAME="${PROJECT_NAME}", ESSENTIA_WHEEL_SKIP_3RDPARTY=1, ESSENTIA_WHEEL_ONLY_PYTHON=1 } -before-all = [ +before-build = [ "PYBIN=/opt/python/cp36-cp36m/bin/", "\"${PYBIN}/python\" waf configure --with-gaia --build-static --static-dependencies --pkg-config-path=\"${PKG_CONFIG_PATH}\"", "\"${PYBIN}/python\" waf", @@ -24,7 +24,7 @@ skip = ["pp*"] environment = { PROJECT_NAME="essentia", ESSENTIA_PROJECT_NAME="${PROJECT_NAME}", ESSENTIA_WHEEL_SKIP_3RDPARTY=1, ESSENTIA_WHEEL_ONLY_PYTHON=1 } -before-all = [ +before-build = [ "brew install pkg-config gcc readline sqlite gdbm freetype libpng", "brew install eigen libyaml fftw ffmpeg@2.8 libsamplerate libtag chromaprint", "brew link --force ffmpeg@2.8", @@ -38,9 +38,9 @@ before-all = [ test-command = "python -c 'import essentia; import essentia.standard; import essentia.streaming; from essentia.standard import MonoLoader, MetadataReader, YamlInput, Chromaprinter'" [[tool.cibuildwheel.overrides]] -select = "*arm64*" +select = "*macosx_arm64*" -before-all = [ +before-build = [ "brew install pkg-config gcc readline sqlite gdbm freetype libpng", "brew install eigen libyaml fftw ffmpeg@2.8 libsamplerate libtag chromaprint", "brew link --force ffmpeg@2.8", From b2afb51f2e3ccd800094ba07b7899543839e52f5 Mon Sep 17 00:00:00 2001 From: Pablo Alonso Date: Tue, 24 Oct 2023 20:48:22 +0200 Subject: [PATCH 03/45] Force installing arm64 version for cross-build --- .github/workflows/build-wheels-cibuildwheel.yml | 2 +- pyproject-tensorflow.toml | 7 ++++--- pyproject.toml | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-wheels-cibuildwheel.yml b/.github/workflows/build-wheels-cibuildwheel.yml index 3b840e899..7914d7195 100644 --- a/.github/workflows/build-wheels-cibuildwheel.yml +++ b/.github/workflows/build-wheels-cibuildwheel.yml @@ -23,7 +23,7 @@ jobs: with: config-file: ${{ matrix.config }} env: - CIBW_ARCHS_MACOS: x86_64 arm64 + CIBW_ARCHS_MACOS: arm64 x86_64 - uses: actions/upload-artifact@v3 with: diff --git a/pyproject-tensorflow.toml b/pyproject-tensorflow.toml index 83684f38f..e3240e823 100644 --- a/pyproject-tensorflow.toml +++ b/pyproject-tensorflow.toml @@ -43,9 +43,10 @@ test-command = "python -c 'import essentia; import essentia.standard; import ess select = "*macosx_arm64*" before-build = [ - "brew install pkg-config gcc readline sqlite gdbm freetype libpng", - "brew install eigen libyaml fftw ffmpeg@2.8 libsamplerate libtag chromaprint", - "brew install tensorflow", + "brew install ", + "PACKAGES=(eigen libyaml fftw ffmpeg@2.8 libsamplerate libtag chromaprint); for PACKAGE in \"${PACKAGES[@]}\"; do brew fetch --force --bottle-tag=arm64_monterrey $PACKAGE; result=$(brew --cache --bottle-tag=arm64_monterrey $PACKAGE) brew install $result; done" + "PACKAGES=(pkg-config gcc readline sqlite gdbm freetype libpng); for PACKAGE in \"${PACKAGES[@]}\"; do brew fetch --force --bottle-tag=arm64_monterrey $PACKAGE; result=$(brew --cache --bottle-tag=arm64_monterrey $PACKAGE) brew install $result; done" + "PACKAGES=(tensorflow); for PACKAGE in \"${PACKAGES[@]}\"; do brew fetch --force --bottle-tag=arm64_monterrey $PACKAGE; result=$(brew --cache --bottle-tag=arm64_monterrey $PACKAGE) brew install $result; done" "brew link --force ffmpeg@2.8", "python waf configure --with-tensorflow --pkg-config-path=\"${PKG_CONFIG_PATH}\" --arch arm64 --no-msse", "python waf", diff --git a/pyproject.toml b/pyproject.toml index b9c9e27ab..630fb2b42 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,8 +41,8 @@ test-command = "python -c 'import essentia; import essentia.standard; import ess select = "*macosx_arm64*" before-build = [ - "brew install pkg-config gcc readline sqlite gdbm freetype libpng", - "brew install eigen libyaml fftw ffmpeg@2.8 libsamplerate libtag chromaprint", + "PACKAGES=(eigen libyaml fftw ffmpeg@2.8 libsamplerate libtag chromaprint); for PACKAGE in \"${PACKAGES[@]}\"; do brew fetch --force --bottle-tag=arm64_monterrey $PACKAGE; result=$(brew --cache --bottle-tag=arm64_monterrey $PACKAGE) brew install $result; done" + "PACKAGES=(pkg-config gcc readline sqlite gdbm freetype libpng); for PACKAGE in \"${PACKAGES[@]}\"; do brew fetch --force --bottle-tag=arm64_monterrey $PACKAGE; result=$(brew --cache --bottle-tag=arm64_monterrey $PACKAGE) brew install $result; done" "brew link --force ffmpeg@2.8", "python waf configure --pkg-config-path=\"${PKG_CONFIG_PATH}\" --arch arm64 --no-msse", "python waf", From e50b4652907b788c4a9f98b6f9d5e8ba40efc231 Mon Sep 17 00:00:00 2001 From: Pablo Alonso Date: Tue, 24 Oct 2023 20:51:13 +0200 Subject: [PATCH 04/45] Fix toml format --- pyproject-tensorflow.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject-tensorflow.toml b/pyproject-tensorflow.toml index e3240e823..884858a61 100644 --- a/pyproject-tensorflow.toml +++ b/pyproject-tensorflow.toml @@ -44,8 +44,8 @@ select = "*macosx_arm64*" before-build = [ "brew install ", - "PACKAGES=(eigen libyaml fftw ffmpeg@2.8 libsamplerate libtag chromaprint); for PACKAGE in \"${PACKAGES[@]}\"; do brew fetch --force --bottle-tag=arm64_monterrey $PACKAGE; result=$(brew --cache --bottle-tag=arm64_monterrey $PACKAGE) brew install $result; done" - "PACKAGES=(pkg-config gcc readline sqlite gdbm freetype libpng); for PACKAGE in \"${PACKAGES[@]}\"; do brew fetch --force --bottle-tag=arm64_monterrey $PACKAGE; result=$(brew --cache --bottle-tag=arm64_monterrey $PACKAGE) brew install $result; done" + "PACKAGES=(eigen libyaml fftw ffmpeg@2.8 libsamplerate libtag chromaprint); for PACKAGE in \"${PACKAGES[@]}\"; do brew fetch --force --bottle-tag=arm64_monterrey $PACKAGE; result=$(brew --cache --bottle-tag=arm64_monterrey $PACKAGE) brew install $result; done", + "PACKAGES=(pkg-config gcc readline sqlite gdbm freetype libpng); for PACKAGE in \"${PACKAGES[@]}\"; do brew fetch --force --bottle-tag=arm64_monterrey $PACKAGE; result=$(brew --cache --bottle-tag=arm64_monterrey $PACKAGE) brew install $result; done", "PACKAGES=(tensorflow); for PACKAGE in \"${PACKAGES[@]}\"; do brew fetch --force --bottle-tag=arm64_monterrey $PACKAGE; result=$(brew --cache --bottle-tag=arm64_monterrey $PACKAGE) brew install $result; done" "brew link --force ffmpeg@2.8", "python waf configure --with-tensorflow --pkg-config-path=\"${PKG_CONFIG_PATH}\" --arch arm64 --no-msse", From 52250b7a84c19340340b70e3c534e7ece490484b Mon Sep 17 00:00:00 2001 From: Pablo Alonso Date: Tue, 24 Oct 2023 20:54:40 +0200 Subject: [PATCH 05/45] Fix toml format --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 630fb2b42..6b3911ba1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,8 +41,8 @@ test-command = "python -c 'import essentia; import essentia.standard; import ess select = "*macosx_arm64*" before-build = [ - "PACKAGES=(eigen libyaml fftw ffmpeg@2.8 libsamplerate libtag chromaprint); for PACKAGE in \"${PACKAGES[@]}\"; do brew fetch --force --bottle-tag=arm64_monterrey $PACKAGE; result=$(brew --cache --bottle-tag=arm64_monterrey $PACKAGE) brew install $result; done" - "PACKAGES=(pkg-config gcc readline sqlite gdbm freetype libpng); for PACKAGE in \"${PACKAGES[@]}\"; do brew fetch --force --bottle-tag=arm64_monterrey $PACKAGE; result=$(brew --cache --bottle-tag=arm64_monterrey $PACKAGE) brew install $result; done" + "PACKAGES=(eigen libyaml fftw ffmpeg@2.8 libsamplerate libtag chromaprint); for PACKAGE in \"${PACKAGES[@]}\"; do brew fetch --force --bottle-tag=arm64_monterrey $PACKAGE; result=$(brew --cache --bottle-tag=arm64_monterrey $PACKAGE) brew install $result; done", + "PACKAGES=(pkg-config gcc readline sqlite gdbm freetype libpng); for PACKAGE in \"${PACKAGES[@]}\"; do brew fetch --force --bottle-tag=arm64_monterrey $PACKAGE; result=$(brew --cache --bottle-tag=arm64_monterrey $PACKAGE) brew install $result; done", "brew link --force ffmpeg@2.8", "python waf configure --pkg-config-path=\"${PKG_CONFIG_PATH}\" --arch arm64 --no-msse", "python waf", From 12e5895170421427209aeb4f4aef80a81221ff98 Mon Sep 17 00:00:00 2001 From: Pablo Alonso Date: Tue, 24 Oct 2023 21:00:45 +0200 Subject: [PATCH 06/45] Fix toml format --- pyproject-tensorflow.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject-tensorflow.toml b/pyproject-tensorflow.toml index 884858a61..8729bfaec 100644 --- a/pyproject-tensorflow.toml +++ b/pyproject-tensorflow.toml @@ -46,7 +46,7 @@ before-build = [ "brew install ", "PACKAGES=(eigen libyaml fftw ffmpeg@2.8 libsamplerate libtag chromaprint); for PACKAGE in \"${PACKAGES[@]}\"; do brew fetch --force --bottle-tag=arm64_monterrey $PACKAGE; result=$(brew --cache --bottle-tag=arm64_monterrey $PACKAGE) brew install $result; done", "PACKAGES=(pkg-config gcc readline sqlite gdbm freetype libpng); for PACKAGE in \"${PACKAGES[@]}\"; do brew fetch --force --bottle-tag=arm64_monterrey $PACKAGE; result=$(brew --cache --bottle-tag=arm64_monterrey $PACKAGE) brew install $result; done", - "PACKAGES=(tensorflow); for PACKAGE in \"${PACKAGES[@]}\"; do brew fetch --force --bottle-tag=arm64_monterrey $PACKAGE; result=$(brew --cache --bottle-tag=arm64_monterrey $PACKAGE) brew install $result; done" + "PACKAGES=(tensorflow); for PACKAGE in \"${PACKAGES[@]}\"; do brew fetch --force --bottle-tag=arm64_monterrey $PACKAGE; result=$(brew --cache --bottle-tag=arm64_monterrey $PACKAGE) brew install $result; done", "brew link --force ffmpeg@2.8", "python waf configure --with-tensorflow --pkg-config-path=\"${PKG_CONFIG_PATH}\" --arch arm64 --no-msse", "python waf", From 095b7abc27961acb78a23d18491da49b2560d22f Mon Sep 17 00:00:00 2001 From: Pablo Alonso Date: Fri, 27 Oct 2023 14:03:55 +0200 Subject: [PATCH 07/45] Fix homebrew installation step --- pyproject-tensorflow.toml | 6 +++--- pyproject.toml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pyproject-tensorflow.toml b/pyproject-tensorflow.toml index 8729bfaec..eb2ab730a 100644 --- a/pyproject-tensorflow.toml +++ b/pyproject-tensorflow.toml @@ -44,9 +44,9 @@ select = "*macosx_arm64*" before-build = [ "brew install ", - "PACKAGES=(eigen libyaml fftw ffmpeg@2.8 libsamplerate libtag chromaprint); for PACKAGE in \"${PACKAGES[@]}\"; do brew fetch --force --bottle-tag=arm64_monterrey $PACKAGE; result=$(brew --cache --bottle-tag=arm64_monterrey $PACKAGE) brew install $result; done", - "PACKAGES=(pkg-config gcc readline sqlite gdbm freetype libpng); for PACKAGE in \"${PACKAGES[@]}\"; do brew fetch --force --bottle-tag=arm64_monterrey $PACKAGE; result=$(brew --cache --bottle-tag=arm64_monterrey $PACKAGE) brew install $result; done", - "PACKAGES=(tensorflow); for PACKAGE in \"${PACKAGES[@]}\"; do brew fetch --force --bottle-tag=arm64_monterrey $PACKAGE; result=$(brew --cache --bottle-tag=arm64_monterrey $PACKAGE) brew install $result; done", + "PACKAGES=(eigen libyaml fftw ffmpeg@2.8 libsamplerate libtag chromaprint); for PACKAGE in ${PACKAGES[@]}; do brew fetch --force --bottle-tag=arm64_monterey $PACKAGE; result=$(brew --cache --bottle-tag=arm64_monterey $PACKAGE); brew install $result; done", + "PACKAGES=(pkg-config gcc readline sqlite gdbm freetype libpng); for PACKAGE in ${PACKAGES[@]}; do brew fetch --force --bottle-tag=arm64_monterey $PACKAGE; result=$(brew --cache --bottle-tag=arm64_monterey $PACKAGE); brew install $result; done", + "PACKAGES=(tensorflow); for PACKAGE in ${PACKAGES[@]}; do brew fetch --force --bottle-tag=arm64_monterey $PACKAGE; result=$(brew --cache --bottle-tag=arm64_monterey $PACKAGE); brew install $result; done", "brew link --force ffmpeg@2.8", "python waf configure --with-tensorflow --pkg-config-path=\"${PKG_CONFIG_PATH}\" --arch arm64 --no-msse", "python waf", diff --git a/pyproject.toml b/pyproject.toml index 6b3911ba1..43ab0e172 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,8 +41,8 @@ test-command = "python -c 'import essentia; import essentia.standard; import ess select = "*macosx_arm64*" before-build = [ - "PACKAGES=(eigen libyaml fftw ffmpeg@2.8 libsamplerate libtag chromaprint); for PACKAGE in \"${PACKAGES[@]}\"; do brew fetch --force --bottle-tag=arm64_monterrey $PACKAGE; result=$(brew --cache --bottle-tag=arm64_monterrey $PACKAGE) brew install $result; done", - "PACKAGES=(pkg-config gcc readline sqlite gdbm freetype libpng); for PACKAGE in \"${PACKAGES[@]}\"; do brew fetch --force --bottle-tag=arm64_monterrey $PACKAGE; result=$(brew --cache --bottle-tag=arm64_monterrey $PACKAGE) brew install $result; done", + "PACKAGES=(eigen libyaml fftw ffmpeg@2.8 libsamplerate libtag chromaprint); for PACKAGE in ${PACKAGES[@]}; do brew fetch --force --bottle-tag=arm64_monterey $PACKAGE; result=$(brew --cache --bottle-tag=arm64_monterey $PACKAGE); brew install $result; done", + "PACKAGES=(pkg-config gcc readline sqlite gdbm freetype libpng); for PACKAGE in ${PACKAGES[@]}; do brew fetch --force --bottle-tag=arm64_monterey $PACKAGE; result=$(brew --cache --bottle-tag=arm64_monterey $PACKAGE); brew install $result; done", "brew link --force ffmpeg@2.8", "python waf configure --pkg-config-path=\"${PKG_CONFIG_PATH}\" --arch arm64 --no-msse", "python waf", From cb30bfe33132eefe0cdeaa9d438be4b8503aea48 Mon Sep 17 00:00:00 2001 From: Pablo Alonso Date: Sat, 4 Nov 2023 14:53:49 +0100 Subject: [PATCH 08/45] Fix pyproject-tensorflow - Use brew reinstall instead of brew install. - Remove incorrect brew install command. --- pyproject-tensorflow.toml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pyproject-tensorflow.toml b/pyproject-tensorflow.toml index eb2ab730a..e972e70f1 100644 --- a/pyproject-tensorflow.toml +++ b/pyproject-tensorflow.toml @@ -43,10 +43,9 @@ test-command = "python -c 'import essentia; import essentia.standard; import ess select = "*macosx_arm64*" before-build = [ - "brew install ", - "PACKAGES=(eigen libyaml fftw ffmpeg@2.8 libsamplerate libtag chromaprint); for PACKAGE in ${PACKAGES[@]}; do brew fetch --force --bottle-tag=arm64_monterey $PACKAGE; result=$(brew --cache --bottle-tag=arm64_monterey $PACKAGE); brew install $result; done", - "PACKAGES=(pkg-config gcc readline sqlite gdbm freetype libpng); for PACKAGE in ${PACKAGES[@]}; do brew fetch --force --bottle-tag=arm64_monterey $PACKAGE; result=$(brew --cache --bottle-tag=arm64_monterey $PACKAGE); brew install $result; done", - "PACKAGES=(tensorflow); for PACKAGE in ${PACKAGES[@]}; do brew fetch --force --bottle-tag=arm64_monterey $PACKAGE; result=$(brew --cache --bottle-tag=arm64_monterey $PACKAGE); brew install $result; done", + "PACKAGES=(eigen libyaml fftw ffmpeg@2.8 libsamplerate libtag chromaprint); for PACKAGE in ${PACKAGES[@]}; do brew fetch --force --bottle-tag=arm64_monterey $PACKAGE; result=$(brew --cache --bottle-tag=arm64_monterey $PACKAGE); brew reinstall $result; done", + "PACKAGES=(pkg-config gcc readline sqlite gdbm freetype libpng); for PACKAGE in ${PACKAGES[@]}; do brew fetch --force --bottle-tag=arm64_monterey $PACKAGE; result=$(brew --cache --bottle-tag=arm64_monterey $PACKAGE); brew reinstall $result; done", + "PACKAGES=(tensorflow); for PACKAGE in ${PACKAGES[@]}; do brew fetch --force --bottle-tag=arm64_monterey $PACKAGE; result=$(brew --cache --bottle-tag=arm64_monterey $PACKAGE); brew reinstall $result; done", "brew link --force ffmpeg@2.8", "python waf configure --with-tensorflow --pkg-config-path=\"${PKG_CONFIG_PATH}\" --arch arm64 --no-msse", "python waf", From deb7a6ac64aede43aff2e475d6d0e9a7a32df75b Mon Sep 17 00:00:00 2001 From: Pablo Alonso Date: Sat, 4 Nov 2023 17:58:34 +0100 Subject: [PATCH 09/45] Build only arm64 wheels for testing (WIP) --- .github/workflows/build-wheels-cibuildwheel.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-wheels-cibuildwheel.yml b/.github/workflows/build-wheels-cibuildwheel.yml index 7914d7195..62845b9f0 100644 --- a/.github/workflows/build-wheels-cibuildwheel.yml +++ b/.github/workflows/build-wheels-cibuildwheel.yml @@ -8,8 +8,8 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-20.04, macos-11] - config: [pyproject-tensorflow.toml, pyproject.toml] + os: [macos-11] + config: [pyproject.toml] steps: - uses: actions/checkout@v3 @@ -23,7 +23,7 @@ jobs: with: config-file: ${{ matrix.config }} env: - CIBW_ARCHS_MACOS: arm64 x86_64 + CIBW_ARCHS_MACOS: arm64 - uses: actions/upload-artifact@v3 with: From 3dcd419e15712a5656499e1f6b76af34339770de Mon Sep 17 00:00:00 2001 From: Pablo Alonso Date: Sat, 4 Nov 2023 18:13:01 +0100 Subject: [PATCH 10/45] Reverse dependancy installation order --- pyproject-tensorflow.toml | 2 +- pyproject.toml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject-tensorflow.toml b/pyproject-tensorflow.toml index e972e70f1..48bb34e66 100644 --- a/pyproject-tensorflow.toml +++ b/pyproject-tensorflow.toml @@ -43,8 +43,8 @@ test-command = "python -c 'import essentia; import essentia.standard; import ess select = "*macosx_arm64*" before-build = [ - "PACKAGES=(eigen libyaml fftw ffmpeg@2.8 libsamplerate libtag chromaprint); for PACKAGE in ${PACKAGES[@]}; do brew fetch --force --bottle-tag=arm64_monterey $PACKAGE; result=$(brew --cache --bottle-tag=arm64_monterey $PACKAGE); brew reinstall $result; done", "PACKAGES=(pkg-config gcc readline sqlite gdbm freetype libpng); for PACKAGE in ${PACKAGES[@]}; do brew fetch --force --bottle-tag=arm64_monterey $PACKAGE; result=$(brew --cache --bottle-tag=arm64_monterey $PACKAGE); brew reinstall $result; done", + "PACKAGES=(eigen libyaml fftw ffmpeg@2.8 libsamplerate libtag chromaprint); for PACKAGE in ${PACKAGES[@]}; do brew fetch --force --bottle-tag=arm64_monterey $PACKAGE; result=$(brew --cache --bottle-tag=arm64_monterey $PACKAGE); brew reinstall $result; done", "PACKAGES=(tensorflow); for PACKAGE in ${PACKAGES[@]}; do brew fetch --force --bottle-tag=arm64_monterey $PACKAGE; result=$(brew --cache --bottle-tag=arm64_monterey $PACKAGE); brew reinstall $result; done", "brew link --force ffmpeg@2.8", "python waf configure --with-tensorflow --pkg-config-path=\"${PKG_CONFIG_PATH}\" --arch arm64 --no-msse", diff --git a/pyproject.toml b/pyproject.toml index 43ab0e172..b19d37437 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,8 +41,8 @@ test-command = "python -c 'import essentia; import essentia.standard; import ess select = "*macosx_arm64*" before-build = [ - "PACKAGES=(eigen libyaml fftw ffmpeg@2.8 libsamplerate libtag chromaprint); for PACKAGE in ${PACKAGES[@]}; do brew fetch --force --bottle-tag=arm64_monterey $PACKAGE; result=$(brew --cache --bottle-tag=arm64_monterey $PACKAGE); brew install $result; done", - "PACKAGES=(pkg-config gcc readline sqlite gdbm freetype libpng); for PACKAGE in ${PACKAGES[@]}; do brew fetch --force --bottle-tag=arm64_monterey $PACKAGE; result=$(brew --cache --bottle-tag=arm64_monterey $PACKAGE); brew install $result; done", + "PACKAGES=(pkg-config gcc readline sqlite gdbm freetype libpng); for PACKAGE in ${PACKAGES[@]}; do brew fetch --force --bottle-tag=arm64_monterey $PACKAGE; result=$(brew --cache --bottle-tag=arm64_monterey $PACKAGE); brew reinstall $result; done", + "PACKAGES=(eigen libyaml fftw ffmpeg@2.8 libsamplerate libtag chromaprint); for PACKAGE in ${PACKAGES[@]}; do brew fetch --force --bottle-tag=arm64_monterey $PACKAGE; result=$(brew --cache --bottle-tag=arm64_monterey $PACKAGE); brew reinstall $result; done", "brew link --force ffmpeg@2.8", "python waf configure --pkg-config-path=\"${PKG_CONFIG_PATH}\" --arch arm64 --no-msse", "python waf", From 7b68a74d87576705b4e14a6d504fc435c180f1ff Mon Sep 17 00:00:00 2001 From: Pablo Alonso Date: Sat, 4 Nov 2023 18:18:36 +0100 Subject: [PATCH 11/45] Use big_sur version instead of monterey --- pyproject-tensorflow.toml | 6 +++--- pyproject.toml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pyproject-tensorflow.toml b/pyproject-tensorflow.toml index 48bb34e66..5616efb22 100644 --- a/pyproject-tensorflow.toml +++ b/pyproject-tensorflow.toml @@ -43,9 +43,9 @@ test-command = "python -c 'import essentia; import essentia.standard; import ess select = "*macosx_arm64*" before-build = [ - "PACKAGES=(pkg-config gcc readline sqlite gdbm freetype libpng); for PACKAGE in ${PACKAGES[@]}; do brew fetch --force --bottle-tag=arm64_monterey $PACKAGE; result=$(brew --cache --bottle-tag=arm64_monterey $PACKAGE); brew reinstall $result; done", - "PACKAGES=(eigen libyaml fftw ffmpeg@2.8 libsamplerate libtag chromaprint); for PACKAGE in ${PACKAGES[@]}; do brew fetch --force --bottle-tag=arm64_monterey $PACKAGE; result=$(brew --cache --bottle-tag=arm64_monterey $PACKAGE); brew reinstall $result; done", - "PACKAGES=(tensorflow); for PACKAGE in ${PACKAGES[@]}; do brew fetch --force --bottle-tag=arm64_monterey $PACKAGE; result=$(brew --cache --bottle-tag=arm64_monterey $PACKAGE); brew reinstall $result; done", + "PACKAGES=(pkg-config gcc readline sqlite gdbm freetype libpng); for PACKAGE in ${PACKAGES[@]}; do brew fetch --force --bottle-tag=arm64_big_sur $PACKAGE; result=$(brew --cache --bottle-tag=arm64_big_sur $PACKAGE); brew reinstall $result; done", + "PACKAGES=(eigen libyaml fftw ffmpeg@2.8 libsamplerate libtag chromaprint); for PACKAGE in ${PACKAGES[@]}; do brew fetch --force --bottle-tag=arm64_big_sur $PACKAGE; result=$(brew --cache --bottle-tag=arm64_big_sur $PACKAGE); brew reinstall $result; done", + "PACKAGES=(tensorflow); for PACKAGE in ${PACKAGES[@]}; do brew fetch --force --bottle-tag=arm64_big_sur $PACKAGE; result=$(brew --cache --bottle-tag=arm64_big_sur $PACKAGE); brew reinstall $result; done", "brew link --force ffmpeg@2.8", "python waf configure --with-tensorflow --pkg-config-path=\"${PKG_CONFIG_PATH}\" --arch arm64 --no-msse", "python waf", diff --git a/pyproject.toml b/pyproject.toml index b19d37437..3075b9cf4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,8 +41,8 @@ test-command = "python -c 'import essentia; import essentia.standard; import ess select = "*macosx_arm64*" before-build = [ - "PACKAGES=(pkg-config gcc readline sqlite gdbm freetype libpng); for PACKAGE in ${PACKAGES[@]}; do brew fetch --force --bottle-tag=arm64_monterey $PACKAGE; result=$(brew --cache --bottle-tag=arm64_monterey $PACKAGE); brew reinstall $result; done", - "PACKAGES=(eigen libyaml fftw ffmpeg@2.8 libsamplerate libtag chromaprint); for PACKAGE in ${PACKAGES[@]}; do brew fetch --force --bottle-tag=arm64_monterey $PACKAGE; result=$(brew --cache --bottle-tag=arm64_monterey $PACKAGE); brew reinstall $result; done", + "PACKAGES=(pkg-config gcc readline sqlite gdbm freetype libpng); for PACKAGE in ${PACKAGES[@]}; do brew fetch --force --bottle-tag=arm64_big_sur $PACKAGE; result=$(brew --cache --bottle-tag=arm64_big_sur $PACKAGE); brew reinstall $result; done", + "PACKAGES=(eigen libyaml fftw ffmpeg@2.8 libsamplerate libtag chromaprint); for PACKAGE in ${PACKAGES[@]}; do brew fetch --force --bottle-tag=arm64_big_sur $PACKAGE; result=$(brew --cache --bottle-tag=arm64_big_sur $PACKAGE); brew reinstall $result; done", "brew link --force ffmpeg@2.8", "python waf configure --pkg-config-path=\"${PKG_CONFIG_PATH}\" --arch arm64 --no-msse", "python waf", From 57d19a2383655b050b8c54d6e2c5756220dd514a Mon Sep 17 00:00:00 2001 From: Pablo Alonso Date: Sat, 4 Nov 2023 18:46:45 +0100 Subject: [PATCH 12/45] Try ventura tag --- pyproject-tensorflow.toml | 6 +++--- pyproject.toml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pyproject-tensorflow.toml b/pyproject-tensorflow.toml index 5616efb22..2c413fc60 100644 --- a/pyproject-tensorflow.toml +++ b/pyproject-tensorflow.toml @@ -43,9 +43,9 @@ test-command = "python -c 'import essentia; import essentia.standard; import ess select = "*macosx_arm64*" before-build = [ - "PACKAGES=(pkg-config gcc readline sqlite gdbm freetype libpng); for PACKAGE in ${PACKAGES[@]}; do brew fetch --force --bottle-tag=arm64_big_sur $PACKAGE; result=$(brew --cache --bottle-tag=arm64_big_sur $PACKAGE); brew reinstall $result; done", - "PACKAGES=(eigen libyaml fftw ffmpeg@2.8 libsamplerate libtag chromaprint); for PACKAGE in ${PACKAGES[@]}; do brew fetch --force --bottle-tag=arm64_big_sur $PACKAGE; result=$(brew --cache --bottle-tag=arm64_big_sur $PACKAGE); brew reinstall $result; done", - "PACKAGES=(tensorflow); for PACKAGE in ${PACKAGES[@]}; do brew fetch --force --bottle-tag=arm64_big_sur $PACKAGE; result=$(brew --cache --bottle-tag=arm64_big_sur $PACKAGE); brew reinstall $result; done", + "brew install pkg-config gcc readline sqlite gdbm freetype libpng", + "PACKAGES=(eigen libyaml fftw ffmpeg@2.8 libsamplerate libtag chromaprint); for PACKAGE in ${PACKAGES[@]}; do brew fetch --force --bottle-tag=arm64_ventura $PACKAGE; result=$(brew --cache --bottle-tag=arm64_ventura $PACKAGE); brew reinstall $result; done", + "PACKAGES=(tensorflow); for PACKAGE in ${PACKAGES[@]}; do brew fetch --force --bottle-tag=arm64_ventura $PACKAGE; result=$(brew --cache --bottle-tag=arm64_ventura $PACKAGE); brew reinstall $result; done", "brew link --force ffmpeg@2.8", "python waf configure --with-tensorflow --pkg-config-path=\"${PKG_CONFIG_PATH}\" --arch arm64 --no-msse", "python waf", diff --git a/pyproject.toml b/pyproject.toml index 3075b9cf4..e833b4e8d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,8 +41,8 @@ test-command = "python -c 'import essentia; import essentia.standard; import ess select = "*macosx_arm64*" before-build = [ - "PACKAGES=(pkg-config gcc readline sqlite gdbm freetype libpng); for PACKAGE in ${PACKAGES[@]}; do brew fetch --force --bottle-tag=arm64_big_sur $PACKAGE; result=$(brew --cache --bottle-tag=arm64_big_sur $PACKAGE); brew reinstall $result; done", - "PACKAGES=(eigen libyaml fftw ffmpeg@2.8 libsamplerate libtag chromaprint); for PACKAGE in ${PACKAGES[@]}; do brew fetch --force --bottle-tag=arm64_big_sur $PACKAGE; result=$(brew --cache --bottle-tag=arm64_big_sur $PACKAGE); brew reinstall $result; done", + "brew install pkg-config gcc readline sqlite gdbm freetype libpng", + "PACKAGES=(eigen libyaml fftw ffmpeg@2.8 libsamplerate libtag chromaprint); for PACKAGE in ${PACKAGES[@]}; do brew fetch --force --bottle-tag=arm64_ventura $PACKAGE; result=$(brew --cache --bottle-tag=arm64_ventura $PACKAGE); brew reinstall $result; done", "brew link --force ffmpeg@2.8", "python waf configure --pkg-config-path=\"${PKG_CONFIG_PATH}\" --arch arm64 --no-msse", "python waf", From 2d2a2dc55e484b06c0d94384f46365695cb05742 Mon Sep 17 00:00:00 2001 From: Pablo Alonso Date: Sun, 5 Nov 2023 09:44:33 +0100 Subject: [PATCH 13/45] Do not build Python 3.8 wheel --- .github/workflows/build-wheels-cibuildwheel.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-wheels-cibuildwheel.yml b/.github/workflows/build-wheels-cibuildwheel.yml index 62845b9f0..9325a4ff6 100644 --- a/.github/workflows/build-wheels-cibuildwheel.yml +++ b/.github/workflows/build-wheels-cibuildwheel.yml @@ -23,7 +23,8 @@ jobs: with: config-file: ${{ matrix.config }} env: - CIBW_ARCHS_MACOS: arm64 + - CIBW_ARCHS_MACOS: arm64 + - CIBW_SKIP: cp38-macosx_* - uses: actions/upload-artifact@v3 with: From 31e9de9cbecd4ae87d502875d1dfc8099067139b Mon Sep 17 00:00:00 2001 From: Pablo Alonso Date: Sun, 5 Nov 2023 09:46:47 +0100 Subject: [PATCH 14/45] Fix workflow file --- .github/workflows/build-wheels-cibuildwheel.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-wheels-cibuildwheel.yml b/.github/workflows/build-wheels-cibuildwheel.yml index 9325a4ff6..41bd981a8 100644 --- a/.github/workflows/build-wheels-cibuildwheel.yml +++ b/.github/workflows/build-wheels-cibuildwheel.yml @@ -23,8 +23,8 @@ jobs: with: config-file: ${{ matrix.config }} env: - - CIBW_ARCHS_MACOS: arm64 - - CIBW_SKIP: cp38-macosx_* + CIBW_ARCHS_MACOS: arm64 + CIBW_SKIP: cp38-macosx_* - uses: actions/upload-artifact@v3 with: From 5cfff93656c87e4ab10f9520ce5ff899909db82e Mon Sep 17 00:00:00 2001 From: Pablo Alonso Date: Sun, 5 Nov 2023 11:49:10 +0100 Subject: [PATCH 15/45] Add arch arm64 to CFLAGS src/3rdparty/nnls/nnls.c was not cross-built for arm64 because the arch flag was not set for .c files. --- wscript | 1 + 1 file changed, 1 insertion(+) diff --git a/wscript b/wscript index 7970f4aa6..a48d304c8 100644 --- a/wscript +++ b/wscript @@ -195,6 +195,7 @@ def configure(ctx): ctx.env.LDFLAGS = ['-arch', 'i386', '-arch', 'x86_64'] if ctx.options.ARCH == 'arm64': + ctx.env.CFLAGS += ['-arch', 'arm64'] ctx.env.CXXFLAGS += ['-arch', 'arm64'] ctx.env.LINKFLAGS += ['-arch', 'arm64'] ctx.env.LDFLAGS += ['-arch', 'arm64'] From b99d3163ac21b7904ab869ed381197b8e965445a Mon Sep 17 00:00:00 2001 From: Pablo Alonso Date: Sun, 5 Nov 2023 17:27:42 +0100 Subject: [PATCH 16/45] Add arm64 flags to setup.py The wheel building process should be informed of the arch we are building for. If this operation works we should implement a mechanism to tell setup.py when we are cross-compiling. Maybe via env variable. --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index edc4a5326..0401b3e1f 100644 --- a/setup.py +++ b/setup.py @@ -45,10 +45,10 @@ def run(self): if var_only_python in os.environ and os.environ[var_only_python]=='1': print('Skipping building the core libessentia library (%s=1)' % var_only_python) subprocess.run([PYTHON, 'waf', 'configure', '--only-python', '--static-dependencies', - '--prefix=tmp'], check=True) + '--prefix=tmp','--arch=arm64', '--no-msse'], check=True) else: subprocess.run([PYTHON, 'waf', 'configure', '--build-static', '--static-dependencies' - '--with-python --prefix=tmp'], check=True) + '--with-python --prefix=tmp', '--arch=arm64', '--no-msse'], check=True) subprocess.run([PYTHON, 'waf'], check=True) subprocess.run([PYTHON, 'waf', 'install'], check=True) From 27c2f48dd4de5652b1b175648fdba636b187e523 Mon Sep 17 00:00:00 2001 From: Pablo Alonso Date: Mon, 6 Nov 2023 10:42:53 +0100 Subject: [PATCH 17/45] Install arm64 version of the missing libs --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e833b4e8d..dda8b5ef9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,7 +41,8 @@ test-command = "python -c 'import essentia; import essentia.standard; import ess select = "*macosx_arm64*" before-build = [ - "brew install pkg-config gcc readline sqlite gdbm freetype libpng", + "brew install pkg-config gcc readline sqlite gdbm libpng", + "PACKAGES=(freetype); for PACKAGE in ${PACKAGES[@]}; do brew fetch --force --bottle-tag=arm64_ventura $PACKAGE; result=$(brew --cache --bottle-tag=arm64_ventura $PACKAGE); brew reinstall $result; done", "PACKAGES=(eigen libyaml fftw ffmpeg@2.8 libsamplerate libtag chromaprint); for PACKAGE in ${PACKAGES[@]}; do brew fetch --force --bottle-tag=arm64_ventura $PACKAGE; result=$(brew --cache --bottle-tag=arm64_ventura $PACKAGE); brew reinstall $result; done", "brew link --force ffmpeg@2.8", "python waf configure --pkg-config-path=\"${PKG_CONFIG_PATH}\" --arch arm64 --no-msse", From fee770ec98680cce44d25e3092146970f13811d3 Mon Sep 17 00:00:00 2001 From: Pablo Alonso Date: Wed, 31 Jan 2024 11:04:32 +0100 Subject: [PATCH 18/45] Copy SDL2 libs into the Python wheel for Mac arm64 --- pyproject.toml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index dda8b5ef9..33781a91d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,6 +50,21 @@ before-build = [ "python waf install" ] +# On Mac arm64, libavcodec.56.60.100, libavformat.56.40.101 and +# libavutil.54.31.100, depend on libSDL1.2-compat, which is a compatibility +# layer for SDL2. libSDL1.2-compat expects SDL2 to be installed in the default +# brew location (i.e., /opt/homebrew/opt/sdl2/lib), so the user would need to +# install it via brew manually. Alternativelly, we can manualy copy the SDL2 +# libs into the wheel. This is a temporary solution, and in the long term we +# should move to FFmpeg > 2.X. +repair-wheel-command = [ + "delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}", + "mkdir -p {dest_dir}/essentia/.dylibs", + "cp /opt/homebrew/opt/sdl2/lib/libSDL2*.dylib {dest_dir}/essentia/.dylibs", + "wheel_rel=$(echo {wheel} | grep -o '[^/]*$')", + "cd {dest_dir} && zip -u {dest_dir}/$wheel_rel essentia/.dylibs/*" +] + [build-system] requires = ["wheel", "setuptools", "oldest-supported-numpy"] From 7fbfc8cbf909b7c780c4e4b0e092a774753fc145 Mon Sep 17 00:00:00 2001 From: Pablo Alonso Date: Wed, 31 Jan 2024 12:47:59 +0100 Subject: [PATCH 19/45] Use native's M1 MacOS runner according to GHs' blog, setting `runs-on` to `macos-14` automatically provides an M2 machine. Changes: - discard setting `CIBW_ARCHS_MACOS: arm64` to cross-compile - set `os` to `macos-14`. TODO: recover `macos-11` when this works. - discard forcing arm64 brew bottles since it shoudn't be needed anymore. (https://github.blog/changelog/2024-01-30-github-actions-introducing-the-new-m1-macos-runner-available-to-open-source/) --- .github/workflows/build-wheels-cibuildwheel.yml | 3 +-- pyproject.toml | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-wheels-cibuildwheel.yml b/.github/workflows/build-wheels-cibuildwheel.yml index 41bd981a8..556e8bfff 100644 --- a/.github/workflows/build-wheels-cibuildwheel.yml +++ b/.github/workflows/build-wheels-cibuildwheel.yml @@ -8,7 +8,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [macos-11] + os: [macos-14] config: [pyproject.toml] steps: @@ -23,7 +23,6 @@ jobs: with: config-file: ${{ matrix.config }} env: - CIBW_ARCHS_MACOS: arm64 CIBW_SKIP: cp38-macosx_* - uses: actions/upload-artifact@v3 diff --git a/pyproject.toml b/pyproject.toml index 33781a91d..ee66b2fb2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,8 +42,7 @@ select = "*macosx_arm64*" before-build = [ "brew install pkg-config gcc readline sqlite gdbm libpng", - "PACKAGES=(freetype); for PACKAGE in ${PACKAGES[@]}; do brew fetch --force --bottle-tag=arm64_ventura $PACKAGE; result=$(brew --cache --bottle-tag=arm64_ventura $PACKAGE); brew reinstall $result; done", - "PACKAGES=(eigen libyaml fftw ffmpeg@2.8 libsamplerate libtag chromaprint); for PACKAGE in ${PACKAGES[@]}; do brew fetch --force --bottle-tag=arm64_ventura $PACKAGE; result=$(brew --cache --bottle-tag=arm64_ventura $PACKAGE); brew reinstall $result; done", + "brew install eigen libyaml fftw ffmpeg@2.8 libsamplerate libtag chromaprint", "brew link --force ffmpeg@2.8", "python waf configure --pkg-config-path=\"${PKG_CONFIG_PATH}\" --arch arm64 --no-msse", "python waf", From 69c4cda89529e443383ef8f33f843f04aa713471 Mon Sep 17 00:00:00 2001 From: Pablo Alonso Date: Wed, 31 Jan 2024 13:07:49 +0100 Subject: [PATCH 20/45] Install pipx before building wheels According to GH's doc MacOS-14 arm64 machines comes without pipx installed -> https://github.com/actions/runner-images/blob/main/images/macos/macos-14-arm64-Readme.md --- .github/workflows/build-wheels-cibuildwheel.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build-wheels-cibuildwheel.yml b/.github/workflows/build-wheels-cibuildwheel.yml index 556e8bfff..533e89b8b 100644 --- a/.github/workflows/build-wheels-cibuildwheel.yml +++ b/.github/workflows/build-wheels-cibuildwheel.yml @@ -15,6 +15,8 @@ jobs: - uses: actions/checkout@v3 with: fetch-depth: 0 + - name: Install pipx + run: brew install pipx - name: Fetch release tags from GitHub # Workaround for https://github.com/actions/checkout/issues/290 run: git fetch --tags --force From 24b670bcf532f551667f7c0940763975c44eb087 Mon Sep 17 00:00:00 2001 From: Pablo Alonso Date: Wed, 31 Jan 2024 13:21:02 +0100 Subject: [PATCH 21/45] link --force ffmpeg before installing chromaprint --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ee66b2fb2..31ce67bf1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,8 +42,9 @@ select = "*macosx_arm64*" before-build = [ "brew install pkg-config gcc readline sqlite gdbm libpng", - "brew install eigen libyaml fftw ffmpeg@2.8 libsamplerate libtag chromaprint", + "brew install eigen libyaml fftw ffmpeg@2.8 libsamplerate libtag", "brew link --force ffmpeg@2.8", + "brew install chromaprint", "python waf configure --pkg-config-path=\"${PKG_CONFIG_PATH}\" --arch arm64 --no-msse", "python waf", "python waf install" From 9fc80f2337dabc668202a501868b1b19b7c97671 Mon Sep 17 00:00:00 2001 From: Pablo Alonso Date: Wed, 31 Jan 2024 13:53:27 +0100 Subject: [PATCH 22/45] link --force ffmpeg again after installing chromaprint --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 31ce67bf1..8d557749a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,6 +45,7 @@ before-build = [ "brew install eigen libyaml fftw ffmpeg@2.8 libsamplerate libtag", "brew link --force ffmpeg@2.8", "brew install chromaprint", + "brew link --force ffmpeg@2.8", "python waf configure --pkg-config-path=\"${PKG_CONFIG_PATH}\" --arch arm64 --no-msse", "python waf", "python waf install" From 08400b69bfa1d26b12756d5db9b8db75d14ec7da Mon Sep 17 00:00:00 2001 From: Pablo Alonso Date: Wed, 31 Jan 2024 15:00:14 +0100 Subject: [PATCH 23/45] Replace --force by --overwrite to force the link and overwrite all conflicting files --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8d557749a..3b9eb74b8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,7 @@ before-build = [ "brew install eigen libyaml fftw ffmpeg@2.8 libsamplerate libtag", "brew link --force ffmpeg@2.8", "brew install chromaprint", - "brew link --force ffmpeg@2.8", + "brew link --overwrite ffmpeg@2.8", "python waf configure --pkg-config-path=\"${PKG_CONFIG_PATH}\" --arch arm64 --no-msse", "python waf", "python waf install" From d9c2ed88ae1907c5b17d859cfd5fa888c46369ae Mon Sep 17 00:00:00 2001 From: Pablo Alonso Date: Wed, 31 Jan 2024 15:26:02 +0100 Subject: [PATCH 24/45] Create target contxt folder --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 3b9eb74b8..e6f9e6d36 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,7 @@ before-build = [ "brew link --force ffmpeg@2.8", "brew install chromaprint", "brew link --overwrite ffmpeg@2.8", + "mkdir -p /usr/local/lib", "python waf configure --pkg-config-path=\"${PKG_CONFIG_PATH}\" --arch arm64 --no-msse", "python waf", "python waf install" From 574aefe5892ada57019a402300f5835b9a00bbc5 Mon Sep 17 00:00:00 2001 From: Pablo Alonso Date: Wed, 31 Jan 2024 15:31:17 +0100 Subject: [PATCH 25/45] install essentia as root --- pyproject.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e6f9e6d36..c1acad9e1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,10 +46,9 @@ before-build = [ "brew link --force ffmpeg@2.8", "brew install chromaprint", "brew link --overwrite ffmpeg@2.8", - "mkdir -p /usr/local/lib", "python waf configure --pkg-config-path=\"${PKG_CONFIG_PATH}\" --arch arm64 --no-msse", "python waf", - "python waf install" + "sudo python waf install" ] # On Mac arm64, libavcodec.56.60.100, libavformat.56.40.101 and From 85f7b10a523b083fd0b8e3398ce2a87385224fac Mon Sep 17 00:00:00 2001 From: Pablo Alonso Date: Wed, 31 Jan 2024 16:22:39 +0100 Subject: [PATCH 26/45] Control arm64-dependent flags with env var --- pyproject.toml | 3 ++- setup.py | 8 ++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c1acad9e1..a5ffa4e1a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,7 +48,8 @@ before-build = [ "brew link --overwrite ffmpeg@2.8", "python waf configure --pkg-config-path=\"${PKG_CONFIG_PATH}\" --arch arm64 --no-msse", "python waf", - "sudo python waf install" + "sudo python waf install", + "ESSENTIA_MACOSX_ARM64=1", ] # On Mac arm64, libavcodec.56.60.100, libavformat.56.40.101 and diff --git a/setup.py b/setup.py index 0401b3e1f..bd15f14ad 100644 --- a/setup.py +++ b/setup.py @@ -37,6 +37,10 @@ def run(self): var_skip_3rdparty = 'ESSENTIA_WHEEL_SKIP_3RDPARTY' var_only_python = 'ESSENTIA_WHEEL_ONLY_PYTHON' + var_macos_arm64 = os.getenv('ESSENTIA_MACOSX_ARM64') + if var_macos_arm64 == '1': + macos_arm64_flags = ['--arch=arm64', '--no-msse'] + if var_skip_3rdparty in os.environ and os.environ[var_skip_3rdparty]=='1': print('Skipping building static 3rdparty dependencies (%s=1)' % var_skip_3rdparty) else: @@ -45,10 +49,10 @@ def run(self): if var_only_python in os.environ and os.environ[var_only_python]=='1': print('Skipping building the core libessentia library (%s=1)' % var_only_python) subprocess.run([PYTHON, 'waf', 'configure', '--only-python', '--static-dependencies', - '--prefix=tmp','--arch=arm64', '--no-msse'], check=True) + '--prefix=tmp'] + macos_arm64_flags, check=True) else: subprocess.run([PYTHON, 'waf', 'configure', '--build-static', '--static-dependencies' - '--with-python --prefix=tmp', '--arch=arm64', '--no-msse'], check=True) + '--with-python --prefix=tmp'] + macos_arm64_flags, check=True) subprocess.run([PYTHON, 'waf'], check=True) subprocess.run([PYTHON, 'waf', 'install'], check=True) From 7b6b4f5af6907d9810a85d61d11eabb2c062493d Mon Sep 17 00:00:00 2001 From: Pablo Alonso Date: Wed, 31 Jan 2024 16:55:20 +0100 Subject: [PATCH 27/45] Fix handling of env vars and resore linux and macos intel workflows --- .../workflows/build-wheels-cibuildwheel.yml | 7 ++--- pyproject-tensorflow.toml | 29 ++++++++++++++++--- pyproject.toml | 5 +++- 3 files changed, 31 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build-wheels-cibuildwheel.yml b/.github/workflows/build-wheels-cibuildwheel.yml index 533e89b8b..9947450d4 100644 --- a/.github/workflows/build-wheels-cibuildwheel.yml +++ b/.github/workflows/build-wheels-cibuildwheel.yml @@ -8,8 +8,8 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [macos-14] - config: [pyproject.toml] + os: [ubuntu-20.04, macos-12, macos-14] + config: [pyproject.toml, pyproject-tensorflow.toml] steps: - uses: actions/checkout@v3 @@ -24,9 +24,6 @@ jobs: uses: pypa/cibuildwheel@v2.11.4 with: config-file: ${{ matrix.config }} - env: - CIBW_SKIP: cp38-macosx_* - - uses: actions/upload-artifact@v3 with: path: ./wheelhouse/*.whl diff --git a/pyproject-tensorflow.toml b/pyproject-tensorflow.toml index 2c413fc60..b335ee7d3 100644 --- a/pyproject-tensorflow.toml +++ b/pyproject-tensorflow.toml @@ -42,14 +42,35 @@ test-command = "python -c 'import essentia; import essentia.standard; import ess [[tool.cibuildwheel.overrides]] select = "*macosx_arm64*" +skip = ["pp*", "cp38-macosx_*"] + +environment = { PROJECT_NAME="essentia", ESSENTIA_PROJECT_NAME="${PROJECT_NAME}", ESSENTIA_WHEEL_SKIP_3RDPARTY=1, ESSENTIA_WHEEL_ONLY_PYTHON=1, ESSENTIA_MACOSX_ARM64=1 } + before-build = [ - "brew install pkg-config gcc readline sqlite gdbm freetype libpng", - "PACKAGES=(eigen libyaml fftw ffmpeg@2.8 libsamplerate libtag chromaprint); for PACKAGE in ${PACKAGES[@]}; do brew fetch --force --bottle-tag=arm64_ventura $PACKAGE; result=$(brew --cache --bottle-tag=arm64_ventura $PACKAGE); brew reinstall $result; done", - "PACKAGES=(tensorflow); for PACKAGE in ${PACKAGES[@]}; do brew fetch --force --bottle-tag=arm64_ventura $PACKAGE; result=$(brew --cache --bottle-tag=arm64_ventura $PACKAGE); brew reinstall $result; done", + "brew install pkg-config gcc readline sqlite gdbm libpng", + "brew install eigen libyaml fftw ffmpeg@2.8 libsamplerate libtag", "brew link --force ffmpeg@2.8", + "brew install chromaprint", + "brew link --overwrite ffmpeg@2.8", + "brew install tensorflow", "python waf configure --with-tensorflow --pkg-config-path=\"${PKG_CONFIG_PATH}\" --arch arm64 --no-msse", "python waf", - "python waf install" + "sudo python waf install", +] + +# On Mac arm64, libavcodec.56.60.100, libavformat.56.40.101 and +# libavutil.54.31.100, depend on libSDL1.2-compat, which is a compatibility +# layer for SDL2. libSDL1.2-compat expects SDL2 to be installed in the default +# brew location (i.e., /opt/homebrew/opt/sdl2/lib), so the user would need to +# install it via brew manually. Alternativelly, we can manualy copy the SDL2 +# libs into the wheel. This is a temporary solution, and in the long term we +# should move to FFmpeg > 2.X. +repair-wheel-command = [ + "delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}", + "mkdir -p {dest_dir}/essentia/.dylibs", + "cp /opt/homebrew/opt/sdl2/lib/libSDL2*.dylib {dest_dir}/essentia/.dylibs", + "wheel_rel=$(echo {wheel} | grep -o '[^/]*$')", + "cd {dest_dir} && zip -u {dest_dir}/$wheel_rel essentia/.dylibs/*" ] [build-system] diff --git a/pyproject.toml b/pyproject.toml index a5ffa4e1a..d0f6a2646 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,10 @@ test-command = "python -c 'import essentia; import essentia.standard; import ess [[tool.cibuildwheel.overrides]] select = "*macosx_arm64*" +skip = ["pp*", "cp38-macosx_*"] + +environment = { PROJECT_NAME="essentia", ESSENTIA_PROJECT_NAME="${PROJECT_NAME}", ESSENTIA_WHEEL_SKIP_3RDPARTY=1, ESSENTIA_WHEEL_ONLY_PYTHON=1, ESSENTIA_MACOSX_ARM64=1 } + before-build = [ "brew install pkg-config gcc readline sqlite gdbm libpng", "brew install eigen libyaml fftw ffmpeg@2.8 libsamplerate libtag", @@ -49,7 +53,6 @@ before-build = [ "python waf configure --pkg-config-path=\"${PKG_CONFIG_PATH}\" --arch arm64 --no-msse", "python waf", "sudo python waf install", - "ESSENTIA_MACOSX_ARM64=1", ] # On Mac arm64, libavcodec.56.60.100, libavformat.56.40.101 and From ae4b0e6704cb005824b820e34fb3b5dc1cb96440 Mon Sep 17 00:00:00 2001 From: Pablo Alonso Date: Wed, 31 Jan 2024 16:56:07 +0100 Subject: [PATCH 28/45] Fix uninitialized variable --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index bd15f14ad..1bd9c3e72 100644 --- a/setup.py +++ b/setup.py @@ -38,6 +38,7 @@ def run(self): var_only_python = 'ESSENTIA_WHEEL_ONLY_PYTHON' var_macos_arm64 = os.getenv('ESSENTIA_MACOSX_ARM64') + macos_arm64_flags = [] if var_macos_arm64 == '1': macos_arm64_flags = ['--arch=arm64', '--no-msse'] From f31324f9fa31ba83061b39b2dd3430634736961b Mon Sep 17 00:00:00 2001 From: Pablo Alonso Date: Wed, 31 Jan 2024 17:22:15 +0100 Subject: [PATCH 29/45] Install pipx only in macos-14 --- .github/workflows/build-wheels-cibuildwheel.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-wheels-cibuildwheel.yml b/.github/workflows/build-wheels-cibuildwheel.yml index 9947450d4..d4ec2ff5b 100644 --- a/.github/workflows/build-wheels-cibuildwheel.yml +++ b/.github/workflows/build-wheels-cibuildwheel.yml @@ -16,6 +16,7 @@ jobs: with: fetch-depth: 0 - name: Install pipx + if: ${{ matrix.os == macos-14 }} run: brew install pipx - name: Fetch release tags from GitHub # Workaround for https://github.com/actions/checkout/issues/290 From 491b2f5b488ab8b3acc926d542de466c28fbdd30 Mon Sep 17 00:00:00 2001 From: Pablo Alonso Date: Wed, 31 Jan 2024 17:25:00 +0100 Subject: [PATCH 30/45] Fix if statement --- .github/workflows/build-wheels-cibuildwheel.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-wheels-cibuildwheel.yml b/.github/workflows/build-wheels-cibuildwheel.yml index d4ec2ff5b..81860bd24 100644 --- a/.github/workflows/build-wheels-cibuildwheel.yml +++ b/.github/workflows/build-wheels-cibuildwheel.yml @@ -16,7 +16,7 @@ jobs: with: fetch-depth: 0 - name: Install pipx - if: ${{ matrix.os == macos-14 }} + if: ${{ matrix.os == "macos-14" }} run: brew install pipx - name: Fetch release tags from GitHub # Workaround for https://github.com/actions/checkout/issues/290 From c2cf05aeac8455de585df366a05265b70b151f43 Mon Sep 17 00:00:00 2001 From: Pablo Alonso Date: Wed, 31 Jan 2024 17:26:55 +0100 Subject: [PATCH 31/45] Fix GH Actions syntax --- .github/workflows/build-wheels-cibuildwheel.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-wheels-cibuildwheel.yml b/.github/workflows/build-wheels-cibuildwheel.yml index 81860bd24..99a0f25bc 100644 --- a/.github/workflows/build-wheels-cibuildwheel.yml +++ b/.github/workflows/build-wheels-cibuildwheel.yml @@ -16,7 +16,7 @@ jobs: with: fetch-depth: 0 - name: Install pipx - if: ${{ matrix.os == "macos-14" }} + if: ${{ matrix.os == 'macos-14' }} run: brew install pipx - name: Fetch release tags from GitHub # Workaround for https://github.com/actions/checkout/issues/290 From 927f3979a84e75b050660113dafb5c9b0374afc1 Mon Sep 17 00:00:00 2001 From: Pablo Alonso Date: Wed, 31 Jan 2024 17:39:40 +0100 Subject: [PATCH 32/45] Repeat same brew steps in macos-12 as in macos-14 --- pyproject-tensorflow.toml | 6 ++++-- pyproject.toml | 4 +++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/pyproject-tensorflow.toml b/pyproject-tensorflow.toml index b335ee7d3..cfdb11726 100644 --- a/pyproject-tensorflow.toml +++ b/pyproject-tensorflow.toml @@ -26,9 +26,11 @@ environment = { PROJECT_NAME="essentia-tensorflow", ESSENTIA_PROJECT_NAME="${PRO before-build = [ "brew install pkg-config gcc readline sqlite gdbm freetype libpng", - "brew install eigen libyaml fftw ffmpeg@2.8 libsamplerate libtag chromaprint", - "brew install tensorflow", + "brew install eigen libyaml fftw ffmpeg@2.8 libsamplerate libtag", "brew link --force ffmpeg@2.8", + "brew install chromaprint", + "brew link --overwrite ffmpeg@2.8", + "brew install tensorflow", #"brew tap MTG/essentia", #"brew install gaia --HEAD", "python waf configure --with-tensorflow --pkg-config-path=\"${PKG_CONFIG_PATH}\"", diff --git a/pyproject.toml b/pyproject.toml index d0f6a2646..e0e4467a1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,8 +26,10 @@ environment = { PROJECT_NAME="essentia", ESSENTIA_PROJECT_NAME="${PROJECT_NAME}" before-build = [ "brew install pkg-config gcc readline sqlite gdbm freetype libpng", - "brew install eigen libyaml fftw ffmpeg@2.8 libsamplerate libtag chromaprint", + "brew install eigen libyaml fftw ffmpeg@2.8 libsamplerate libtag", "brew link --force ffmpeg@2.8", + "brew install chromaprint", + "brew link --overwrite ffmpeg@2.8", #"brew tap MTG/essentia", #"brew install gaia --HEAD", "python waf configure --pkg-config-path=\"${PKG_CONFIG_PATH}\"", From 6b11900c43ff2a2b637c5f29aa4ef61352596aff Mon Sep 17 00:00:00 2001 From: Pablo Alonso Date: Thu, 1 Feb 2024 09:24:08 +0100 Subject: [PATCH 33/45] Update Node.js 16 -> 20 Node.js 16 actions are deprecated -> github.blog/changelog/2023-09-22-github-actions-transitioning-from-node-16-to-node-20/ --- .github/workflows/build-wheels-cibuildwheel.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/build-wheels-cibuildwheel.yml b/.github/workflows/build-wheels-cibuildwheel.yml index 99a0f25bc..ac8968949 100644 --- a/.github/workflows/build-wheels-cibuildwheel.yml +++ b/.github/workflows/build-wheels-cibuildwheel.yml @@ -14,17 +14,24 @@ jobs: steps: - uses: actions/checkout@v3 with: + using: 'node20' fetch-depth: 0 + - name: Install pipx if: ${{ matrix.os == 'macos-14' }} run: brew install pipx + - name: Fetch release tags from GitHub # Workaround for https://github.com/actions/checkout/issues/290 run: git fetch --tags --force + - name: Build wheels uses: pypa/cibuildwheel@v2.11.4 with: + using: 'node20' config-file: ${{ matrix.config }} + - uses: actions/upload-artifact@v3 with: + using: 'node20' path: ./wheelhouse/*.whl From 228661e52916aa13cce59fc6e4f574d1bebc3f69 Mon Sep 17 00:00:00 2001 From: Pablo Alonso Date: Thu, 1 Feb 2024 09:26:20 +0100 Subject: [PATCH 34/45] Build wheels for Python 3.8 This version was previously skipped because we found errors when building wheels locally. We accidentally missed a leading * and found that the wheels could be created flawlessly in Actions. Restoring these wheels for now. --- pyproject-tensorflow.toml | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject-tensorflow.toml b/pyproject-tensorflow.toml index cfdb11726..bae8f19c3 100644 --- a/pyproject-tensorflow.toml +++ b/pyproject-tensorflow.toml @@ -44,7 +44,7 @@ test-command = "python -c 'import essentia; import essentia.standard; import ess [[tool.cibuildwheel.overrides]] select = "*macosx_arm64*" -skip = ["pp*", "cp38-macosx_*"] +skip = ["pp*"] environment = { PROJECT_NAME="essentia", ESSENTIA_PROJECT_NAME="${PROJECT_NAME}", ESSENTIA_WHEEL_SKIP_3RDPARTY=1, ESSENTIA_WHEEL_ONLY_PYTHON=1, ESSENTIA_MACOSX_ARM64=1 } diff --git a/pyproject.toml b/pyproject.toml index e0e4467a1..6313d0365 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,7 @@ test-command = "python -c 'import essentia; import essentia.standard; import ess [[tool.cibuildwheel.overrides]] select = "*macosx_arm64*" -skip = ["pp*", "cp38-macosx_*"] +skip = ["pp*"] environment = { PROJECT_NAME="essentia", ESSENTIA_PROJECT_NAME="${PROJECT_NAME}", ESSENTIA_WHEEL_SKIP_3RDPARTY=1, ESSENTIA_WHEEL_ONLY_PYTHON=1, ESSENTIA_MACOSX_ARM64=1 } From b31399be30054249919f4b18509feeec33779cbd Mon Sep 17 00:00:00 2001 From: Pablo Alonso Date: Thu, 1 Feb 2024 10:01:16 +0100 Subject: [PATCH 35/45] Revert #6b11900 --- .github/workflows/build-wheels-cibuildwheel.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/build-wheels-cibuildwheel.yml b/.github/workflows/build-wheels-cibuildwheel.yml index ac8968949..fc809dc5c 100644 --- a/.github/workflows/build-wheels-cibuildwheel.yml +++ b/.github/workflows/build-wheels-cibuildwheel.yml @@ -14,7 +14,6 @@ jobs: steps: - uses: actions/checkout@v3 with: - using: 'node20' fetch-depth: 0 - name: Install pipx @@ -28,10 +27,8 @@ jobs: - name: Build wheels uses: pypa/cibuildwheel@v2.11.4 with: - using: 'node20' config-file: ${{ matrix.config }} - uses: actions/upload-artifact@v3 with: - using: 'node20' path: ./wheelhouse/*.whl From 7e7bd585014345a2e4326f05129de6bc76d02c91 Mon Sep 17 00:00:00 2001 From: Pablo Alonso Date: Thu, 1 Feb 2024 10:02:13 +0100 Subject: [PATCH 36/45] Update checkout action to use node 20 --- .github/workflows/build-wheels-cibuildwheel.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-wheels-cibuildwheel.yml b/.github/workflows/build-wheels-cibuildwheel.yml index fc809dc5c..5ec852153 100644 --- a/.github/workflows/build-wheels-cibuildwheel.yml +++ b/.github/workflows/build-wheels-cibuildwheel.yml @@ -12,7 +12,7 @@ jobs: config: [pyproject.toml, pyproject-tensorflow.toml] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 From 0505073467e588ae4c57812e81d3d0b0c1c72550 Mon Sep 17 00:00:00 2001 From: Pablo Alonso Date: Thu, 1 Feb 2024 10:02:57 +0100 Subject: [PATCH 37/45] Update cibuildwheel cibuildwheel 2.16.5 takes care of pipx in macos-14 so that we don't have to manually install it. https://github.com/pypa/cibuildwheel/releases/tag/v2.16.5 --- .github/workflows/build-wheels-cibuildwheel.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/build-wheels-cibuildwheel.yml b/.github/workflows/build-wheels-cibuildwheel.yml index 5ec852153..ed032b9c2 100644 --- a/.github/workflows/build-wheels-cibuildwheel.yml +++ b/.github/workflows/build-wheels-cibuildwheel.yml @@ -16,16 +16,12 @@ jobs: with: fetch-depth: 0 - - name: Install pipx - if: ${{ matrix.os == 'macos-14' }} - run: brew install pipx - - name: Fetch release tags from GitHub # Workaround for https://github.com/actions/checkout/issues/290 run: git fetch --tags --force - name: Build wheels - uses: pypa/cibuildwheel@v2.11.4 + uses: pypa/cibuildwheel@v2.16.5 with: config-file: ${{ matrix.config }} From 4f6fb57319eb79988ab10af22bc5fe8ab9c9c899 Mon Sep 17 00:00:00 2001 From: Pablo Alonso Date: Thu, 1 Feb 2024 10:24:04 +0100 Subject: [PATCH 38/45] Skip Python 3.12 wheels for linux for now --- pyproject-tensorflow.toml | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject-tensorflow.toml b/pyproject-tensorflow.toml index bae8f19c3..870a56006 100644 --- a/pyproject-tensorflow.toml +++ b/pyproject-tensorflow.toml @@ -4,7 +4,7 @@ manylinux-x86_64-image = "mtgupf/essentia-builds:manylinux2014_x86_64" # Only support x86_64 for essentia-tensorflow build = "cp**-manylinux_x86_64" -skip = ["pp*", "*-musllinux*", "*i686"] +skip = ["pp*", "*-musllinux*", "*i686", "*cp312-cp312*"] environment = { PROJECT_NAME="essentia-tensorflow", ESSENTIA_PROJECT_NAME="${PROJECT_NAME}", ESSENTIA_WHEEL_SKIP_3RDPARTY=1, ESSENTIA_WHEEL_ONLY_PYTHON=1 } diff --git a/pyproject.toml b/pyproject.toml index 6313d0365..7f28b1f37 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ manylinux-x86_64-image = "mtgupf/essentia-builds:manylinux2014_x86_64" manylinux-i686-image = "mtgupf/essentia-builds:manylinux2014_i686" -skip = ["pp*", "*-musllinux*", "*i686"] +skip = ["pp*", "*-musllinux*", "*i686", "*cp312-cp312*"] environment = { PROJECT_NAME="essentia", ESSENTIA_PROJECT_NAME="${PROJECT_NAME}", ESSENTIA_WHEEL_SKIP_3RDPARTY=1, ESSENTIA_WHEEL_ONLY_PYTHON=1 } From 8fcf8af787e9bd91395e307a9855b158a85d6dfc Mon Sep 17 00:00:00 2001 From: Pablo Alonso Date: Thu, 1 Feb 2024 10:38:00 +0100 Subject: [PATCH 39/45] Try to fix skip pattern --- pyproject-tensorflow.toml | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject-tensorflow.toml b/pyproject-tensorflow.toml index 870a56006..f932b5c60 100644 --- a/pyproject-tensorflow.toml +++ b/pyproject-tensorflow.toml @@ -4,7 +4,7 @@ manylinux-x86_64-image = "mtgupf/essentia-builds:manylinux2014_x86_64" # Only support x86_64 for essentia-tensorflow build = "cp**-manylinux_x86_64" -skip = ["pp*", "*-musllinux*", "*i686", "*cp312-cp312*"] +skip = ["pp*", "*-musllinux*", "*i686", "*cp312*"] environment = { PROJECT_NAME="essentia-tensorflow", ESSENTIA_PROJECT_NAME="${PROJECT_NAME}", ESSENTIA_WHEEL_SKIP_3RDPARTY=1, ESSENTIA_WHEEL_ONLY_PYTHON=1 } diff --git a/pyproject.toml b/pyproject.toml index 7f28b1f37..4da2c2576 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ manylinux-x86_64-image = "mtgupf/essentia-builds:manylinux2014_x86_64" manylinux-i686-image = "mtgupf/essentia-builds:manylinux2014_i686" -skip = ["pp*", "*-musllinux*", "*i686", "*cp312-cp312*"] +skip = ["pp*", "*-musllinux*", "*i686", "*cp312*"] environment = { PROJECT_NAME="essentia", ESSENTIA_PROJECT_NAME="${PROJECT_NAME}", ESSENTIA_WHEEL_SKIP_3RDPARTY=1, ESSENTIA_WHEEL_ONLY_PYTHON=1 } From 392a6eef2d63e3e44a9f8b8d1dae9de469400495 Mon Sep 17 00:00:00 2001 From: Pablo Alonso Date: Thu, 1 Feb 2024 12:33:55 +0100 Subject: [PATCH 40/45] Bump artifact action to v4 --- .github/workflows/build-wheels-cibuildwheel.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-wheels-cibuildwheel.yml b/.github/workflows/build-wheels-cibuildwheel.yml index ed032b9c2..52b0a2585 100644 --- a/.github/workflows/build-wheels-cibuildwheel.yml +++ b/.github/workflows/build-wheels-cibuildwheel.yml @@ -25,6 +25,6 @@ jobs: with: config-file: ${{ matrix.config }} - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: path: ./wheelhouse/*.whl From 4105c42b683a407093ac0c520eaa7e463639fc14 Mon Sep 17 00:00:00 2001 From: Pablo Alonso Date: Thu, 1 Feb 2024 13:35:33 +0100 Subject: [PATCH 41/45] Add comment on skipping Python 3.12 --- pyproject-tensorflow.toml | 1 + pyproject.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/pyproject-tensorflow.toml b/pyproject-tensorflow.toml index f932b5c60..3ed00e088 100644 --- a/pyproject-tensorflow.toml +++ b/pyproject-tensorflow.toml @@ -4,6 +4,7 @@ manylinux-x86_64-image = "mtgupf/essentia-builds:manylinux2014_x86_64" # Only support x86_64 for essentia-tensorflow build = "cp**-manylinux_x86_64" +# TODO: skipping Python 3.12 for now until we create manylinux wheels supporting this version. skip = ["pp*", "*-musllinux*", "*i686", "*cp312*"] environment = { PROJECT_NAME="essentia-tensorflow", ESSENTIA_PROJECT_NAME="${PROJECT_NAME}", ESSENTIA_WHEEL_SKIP_3RDPARTY=1, ESSENTIA_WHEEL_ONLY_PYTHON=1 } diff --git a/pyproject.toml b/pyproject.toml index 4da2c2576..bf46c2106 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,6 +4,7 @@ manylinux-x86_64-image = "mtgupf/essentia-builds:manylinux2014_x86_64" manylinux-i686-image = "mtgupf/essentia-builds:manylinux2014_i686" +# TODO: skipping Python 3.12 for now until we create manylinux wheels supporting this version. skip = ["pp*", "*-musllinux*", "*i686", "*cp312*"] environment = { PROJECT_NAME="essentia", ESSENTIA_PROJECT_NAME="${PROJECT_NAME}", ESSENTIA_WHEEL_SKIP_3RDPARTY=1, ESSENTIA_WHEEL_ONLY_PYTHON=1 } From 2a156efc93644f66fd5b45d29c3a28460041149e Mon Sep 17 00:00:00 2001 From: Pablo Alonso Date: Thu, 1 Feb 2024 13:36:13 +0100 Subject: [PATCH 42/45] Write artifacts to different files --- .github/workflows/build-wheels-cibuildwheel.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-wheels-cibuildwheel.yml b/.github/workflows/build-wheels-cibuildwheel.yml index 52b0a2585..1881269e8 100644 --- a/.github/workflows/build-wheels-cibuildwheel.yml +++ b/.github/workflows/build-wheels-cibuildwheel.yml @@ -9,7 +9,7 @@ jobs: strategy: matrix: os: [ubuntu-20.04, macos-12, macos-14] - config: [pyproject.toml, pyproject-tensorflow.toml] + config: [pyproject, pyproject-tensorflow] steps: - uses: actions/checkout@v4 @@ -23,8 +23,9 @@ jobs: - name: Build wheels uses: pypa/cibuildwheel@v2.16.5 with: - config-file: ${{ matrix.config }} + config-file: ${{ matrix.config }}.toml - uses: actions/upload-artifact@v4 with: path: ./wheelhouse/*.whl + name: artifact-{{ matrix.os }}-{{ matrix.config }} From 34a08b879bed24ff8948098e0dbd5346ee94158c Mon Sep 17 00:00:00 2001 From: Pablo Alonso Date: Thu, 1 Feb 2024 14:07:44 +0100 Subject: [PATCH 43/45] Fix Actions syntax --- .github/workflows/build-wheels-cibuildwheel.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-wheels-cibuildwheel.yml b/.github/workflows/build-wheels-cibuildwheel.yml index 1881269e8..02596e3d9 100644 --- a/.github/workflows/build-wheels-cibuildwheel.yml +++ b/.github/workflows/build-wheels-cibuildwheel.yml @@ -28,4 +28,4 @@ jobs: - uses: actions/upload-artifact@v4 with: path: ./wheelhouse/*.whl - name: artifact-{{ matrix.os }}-{{ matrix.config }} + name: artifact-${{ matrix.os }}-${{ matrix.config }} From 7d729a5328fbc46e13d16e26a2442308de35eba9 Mon Sep 17 00:00:00 2001 From: Pablo Alonso Date: Thu, 1 Feb 2024 17:18:29 +0100 Subject: [PATCH 44/45] Fix comment --- pyproject-tensorflow.toml | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject-tensorflow.toml b/pyproject-tensorflow.toml index 3ed00e088..8bdf5f84b 100644 --- a/pyproject-tensorflow.toml +++ b/pyproject-tensorflow.toml @@ -4,7 +4,7 @@ manylinux-x86_64-image = "mtgupf/essentia-builds:manylinux2014_x86_64" # Only support x86_64 for essentia-tensorflow build = "cp**-manylinux_x86_64" -# TODO: skipping Python 3.12 for now until we create manylinux wheels supporting this version. +# TODO: skipping Python 3.12 for now until we create manylinux images supporting this version. skip = ["pp*", "*-musllinux*", "*i686", "*cp312*"] environment = { PROJECT_NAME="essentia-tensorflow", ESSENTIA_PROJECT_NAME="${PROJECT_NAME}", ESSENTIA_WHEEL_SKIP_3RDPARTY=1, ESSENTIA_WHEEL_ONLY_PYTHON=1 } diff --git a/pyproject.toml b/pyproject.toml index bf46c2106..43ee57436 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ manylinux-x86_64-image = "mtgupf/essentia-builds:manylinux2014_x86_64" manylinux-i686-image = "mtgupf/essentia-builds:manylinux2014_i686" -# TODO: skipping Python 3.12 for now until we create manylinux wheels supporting this version. +# TODO: skipping Python 3.12 for now until we create manylinux images supporting this version. skip = ["pp*", "*-musllinux*", "*i686", "*cp312*"] environment = { PROJECT_NAME="essentia", ESSENTIA_PROJECT_NAME="${PROJECT_NAME}", ESSENTIA_WHEEL_SKIP_3RDPARTY=1, ESSENTIA_WHEEL_ONLY_PYTHON=1 } From 480076ca4b673abf57f136d6c14042b3873bd384 Mon Sep 17 00:00:00 2001 From: Pablo Alonso Date: Fri, 9 Feb 2024 08:54:02 +0100 Subject: [PATCH 45/45] Fix essentia-tensorflow project name --- pyproject-tensorflow.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject-tensorflow.toml b/pyproject-tensorflow.toml index 8bdf5f84b..5f3eedc1b 100644 --- a/pyproject-tensorflow.toml +++ b/pyproject-tensorflow.toml @@ -47,7 +47,7 @@ select = "*macosx_arm64*" skip = ["pp*"] -environment = { PROJECT_NAME="essentia", ESSENTIA_PROJECT_NAME="${PROJECT_NAME}", ESSENTIA_WHEEL_SKIP_3RDPARTY=1, ESSENTIA_WHEEL_ONLY_PYTHON=1, ESSENTIA_MACOSX_ARM64=1 } +environment = { PROJECT_NAME="essentia-tensorflow", ESSENTIA_PROJECT_NAME="${PROJECT_NAME}", ESSENTIA_WHEEL_SKIP_3RDPARTY=1, ESSENTIA_WHEEL_ONLY_PYTHON=1, ESSENTIA_MACOSX_ARM64=1 } before-build = [ "brew install pkg-config gcc readline sqlite gdbm libpng",