chore(cibw): productify the cibuildwheel workflow (#2311)

* docs: add a (WiP) table of wheels

* docs(wheels): bikeshed a nice table

* chore: flesh out cibuildwheel workflow

* chore: more testing of tooling

* chore: try building out all artefacts

* chore: try to fix sdist download

* chore(cibw): clean up `check_dist.py`, use a more advanced Actions expr

* style: clean up cibw Sphinx ext

* chore(cibw): test all wheels

* chore: test (WOULD FAIL) if passing `-r` works

* chore(cibw): build out once again before moving out of Draft

* chore(cibw): remove from `pull_request` event

* chore(cibw): build pure-Python wheel

* chore(cibw): also test sdist/gen wheel

* chore: remove run from `pull_request` event

.github/workflows/cibuildwheel.yaml

@@ -1,5 +1,5 @@
 # Build wheels using cibuildwheel (https://cibuildwheel.pypa.io/)
-name: Build wheels
+name: build-wheels
 
 on:
   # Run when a release has been created
@@ -11,6 +11,7 @@ on:
 
 jobs:
   build-sdist:
+    # NOTE(vytas): We actually build sdist and pure-Python wheel.
     name: sdist
     runs-on: ubuntu-latest
 
@@ -25,16 +26,31 @@ jobs:
         with:
           python-version: "3.12"
 
-      - name: Build sdist
+      - name: Build sdist and pure-Python wheel
+        env:
+          FALCON_DISABLE_CYTHON: "Y"
+        run: |
+          pip install --upgrade pip
+          pip install --upgrade build
+          python -m build
+
+      - name: Check built artifacts
+        run: |
+          tools/check_dist.py ${{ github.event_name == 'release' && format('-r {0}', github.ref) || '' }}
+
+      - name: Test sdist
+        run: |
+          tools/test_dist.py dist/*.tar.gz
+
+      - name: Test pure-Python wheel
         run: |
-          pip install build
-          python -m build --sdist
+          tools/test_dist.py dist/*.whl
 
       - name: Upload artifacts
         uses: actions/upload-artifact@v4
         with:
           name: cibw-sdist
-          path: dist/*.tar.gz
+          path: dist/falcon-*
 
   build-wheels:
     name: ${{ matrix.python }}-${{ matrix.platform.name }}
@@ -105,37 +121,96 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: cibw-wheel-${{ matrix.python }}-${{ matrix.platform.name }}
-          path: wheelhouse/*.whl
+          path: wheelhouse/falcon-*.whl
+
+  publish-sdist:
+    name: publish-sdist
+    needs:
+      - build-sdist
+      - build-wheels
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          pattern: cibw-sdist
+          path: dist
+          merge-multiple: true
+
+      - name: Check collected artifacts
+        run: |
+          tools/check_dist.py ${{ github.event_name == 'release' && format('-r {0}', github.ref) || '' }}
+
+      - name: Upload sdist to release
+        uses: AButler/[email protected]
+        if: github.event_name == 'release'
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          files: 'dist/*.tar.gz'
+
+      - name: Publish sdist and pure-Python wheel to TestPyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        if: github.event_name == 'workflow_dispatch'
+        with:
+          password: ${{ secrets.TEST_PYPI_TOKEN }}
+          repository-url: https://test.pypi.org/legacy/
+
+      - name: Publish sdist and pure-Python wheel to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        if: github.event_name == 'release'
+        with:
+          password: ${{ secrets.PYPI_TOKEN }}
 
   publish-wheels:
-    name: publish
+    name: publish-wheels
     needs:
       - build-sdist
       - build-wheels
+      - publish-sdist
     runs-on: ubuntu-latest
 
     steps:
-      - name: Download packages
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Download artifacts
         uses: actions/download-artifact@v4
         with:
-          pattern: cibw-*
+          pattern: cibw-wheel-*
           path: dist
           merge-multiple: true
 
       - name: Check collected artifacts
-        # TODO(vytas): Run a script to perform version sanity checks instead.
-        run: ls -l dist/
+        run: |
+          tools/check_dist.py ${{ github.event_name == 'release' && format('-r {0}', github.ref) || '' }}
 
-      - name: Publish artifacts to TestPyPI
+      - name: Publish binary wheels to TestPyPI
         uses: pypa/gh-action-pypi-publish@release/v1
         if: github.event_name == 'workflow_dispatch'
         with:
           password: ${{ secrets.TEST_PYPI_TOKEN }}
           repository-url: https://test.pypi.org/legacy/
 
-      # TODO(vytas): Enable this nuclear option once happy with other tests.
-      # - name: Publish artifacts to PyPI
-      #   uses: pypa/gh-action-pypi-publish@release/v1
-      #   if: github.event_name == 'release'
-      #   with:
-      #     password: ${{ secrets.PYPI_TOKEN }}
+      - name: Publish binary wheels to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        if: github.event_name == 'release'
+        with:
+          password: ${{ secrets.PYPI_TOKEN }}

.github/workflows/create-wheels.yaml

@@ -1,9 +1,8 @@
 name: Create wheel
 
 on:
-  # run when a release has been created
-  release:
-    types: [created]
+  # TODO(vytas): Phase out this workflow in favour of cibuildwheel.yaml.
+  workflow_dispatch:
 
 env:
   # set this so the falcon test uses the installed version and not the local one

docs/conf.py

@@ -61,6 +61,7 @@
     'sphinx_tabs.tabs',
     'sphinx_tabs.tabs',
     # Falcon-specific extensions
+    'ext.cibuildwheel',
     'ext.doorway',
     'ext.private_args',
     'ext.rfc',

docs/ext/cibuildwheel.py

@@ -0,0 +1,115 @@
+# Copyright 2024 by Vytautas Liuolia.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Binary wheels table extension for Sphinx.
+
+This extension parses a GitHub Actions workflow for building binary wheels, and
+summarizes the build onfiguration in a stylish table.
+"""
+
+import itertools
+import pathlib
+
+import sphinx.util.docutils
+import yaml
+
+FALCON_ROOT = pathlib.Path(__file__).resolve().parent.parent.parent
+
+_CHECKBOX = '\u2705'
+_CPYTHON_PLATFORMS = {
+    'manylinux_x86_64': '**Linux Intel** ``manylinux`` 64-bit',
+    'musllinux_x86_64': '**Linux Intel** ``musllinux`` 64-bit',
+    'manylinux_i686': '**Linux Intel** ``manylinux`` 32-bit',
+    'musllinux_i686': '**Linux Intel** ``musllinux`` 32-bit',
+    'manylinux_aarch64': '**Linux ARM** ``manylinux`` 64-bit',
+    'musllinux_aarch64': '**Linux ARM** ``musllinux`` 64-bit',
+    'manylinux_ppc64le': '**Linux PowerPC** ``manylinux`` 64-bit',
+    'musllinux_ppc64le': '**Linux PowerPC** ``musllinux`` 64-bit',
+    'manylinux_s390x': '**Linux IBM Z** ``manylinux``',
+    'musllinux_s390x': '**Linux IBM Z** ``musllinux``',
+    'macosx_x86_64': '**macOS Intel**',
+    'macosx_arm64': '**macOS Apple Silicon**',
+    'win32': '**Windows** 32-bit',
+    'win_amd64': '**Windows** 64-bit',
+    'win_arm64': '**Windows ARM** 64-bit',
+}
+
+
+class WheelsDirective(sphinx.util.docutils.SphinxDirective):
+    """Directive to tabulate build info from a YAML workflow."""
+
+    required_arguments = 1
+
+    @classmethod
+    def _emit_table(cls, data):
+        columns = len(data[0])
+        assert all(
+            len(row) == columns for row in data
+        ), 'All rows must have the same number of columns'
+        # NOTE(vytas): +2 is padding inside cell borders.
+        width = max(len(cell) for cell in itertools.chain(*data)) + 2
+        hline = ('+' + '-' * width) * columns + '+\n'
+        output = [hline]
+
+        for row in data:
+            for cell in row:
+                # NOTE(vytas): Emojis take two spaces...
+                padded_width = width - 1 if cell == _CHECKBOX else width
+                output.append('|' + cell.center(padded_width))
+            output.append('|\n')
+
+            header_line = row == data[0]
+            output.append(hline.replace('-', '=') if header_line else hline)
+
+        return ''.join(output)
+
+    def run(self):
+        workflow_path = pathlib.Path(self.arguments[0])
+        if not workflow_path.is_absolute():
+            workflow_path = FALCON_ROOT / workflow_path
+        with open(workflow_path) as fp:
+            workflow = yaml.safe_load(fp)
+
+        matrix = workflow['jobs']['build-wheels']['strategy']['matrix']
+        platforms = matrix['platform']
+        include = matrix['include']
+        assert not matrix.get('exclude'), 'TODO: exclude is not supported yet'
+        supported = set(
+            itertools.product(
+                [platform['name'] for platform in platforms], matrix['python']
+            )
+        )
+        supported.update((item['platform']['name'], item['python']) for item in include)
+        cpythons = sorted({cp for _, cp in supported}, key=lambda val: (len(val), val))
+
+        header = ['Platform / CPython version']
+        table = [header + [cp.replace('cp3', '3.') for cp in cpythons]]
+        table.extend(
+            [description]
+            + [(_CHECKBOX if (name, cp) in supported else '') for cp in cpythons]
+            for name, description in _CPYTHON_PLATFORMS.items()
+        )
+
+        return self.parse_text_to_nodes(self._emit_table(table))
+
+
+def setup(app):
+    app.add_directive('wheels', WheelsDirective)
+
+    return {
+        'version': '0.1',
+        'parallel_read_safe': True,
+        'parallel_write_safe': True,
+    }

docs/user/install.rst

@@ -39,9 +39,13 @@ Or, to install the latest beta or release candidate, if any:
 
 In order to provide an extra speed boost, Falcon can compile itself with
 Cython. Wheels containing pre-compiled binaries are available from PyPI for
-several common platforms. However, if a wheel for your platform of choice is not
-available, you can choose to stick with the source distribution, or use the
-instructions below to cythonize Falcon for your environment.
+several common platforms (see :ref:`binary_wheels` below for the complete list
+of the platforms that we target, or simply check
+`Falcon files on PyPI <https://pypi.org/project/falcon/#files>`__).
+
+However, even if a wheel for your platform of choice is not available, you can
+choose to stick with the source distribution, or use the instructions below to
+cythonize Falcon for your environment.
 
 The following commands tell pip to install Cython, and then to invoke
 Falcon's ``setup.py``, which will in turn detect the presence of Cython
@@ -64,7 +68,8 @@ pass `-v` to pip in order to echo the compilation commands:
 
     $ pip install -v --no-build-isolation --no-binary :all: falcon
 
-**Installing on OS X**
+Installing on OS X
+^^^^^^^^^^^^^^^^^^
 
 Xcode Command Line Tools are required to compile Cython. Install them
 with this command:
@@ -87,6 +92,28 @@ these issues by setting additional Clang C compiler flags as follows:
 
     $ export CFLAGS="-Qunused-arguments -Wno-unused-function"
 
+.. _binary_wheels:
+
+Binary Wheels
+^^^^^^^^^^^^^
+
+Binary Falcon wheels for are automatically built for many CPython platforms,
+courtesy of `cibuildwheel <https://cibuildwheel.pypa.io/en/stable/>`__.
+
+The following table summarizes the wheel availability on different combinations
+of CPython versions vs CPython platforms:
+
+.. wheels:: .github/workflows/cibuildwheel.yaml
+
+.. note::
+    The `free-threaded build
+    <https://docs.python.org/3.13/whatsnew/3.13.html#free-threaded-cpython>`__
+    mode is not enabled for our wheels at this time.
+
+While we believe that the above configuration covers the most common
+development and deployment scenarios, :ref:`let us known <chat>` if you are
+interested in any builds that are currently missing from our selection!
+
 Dependencies
 ------------
 

requirements/docs

@@ -4,6 +4,7 @@ jinja2
 markupsafe
 pygments
 pygments-style-github
+PyYAML
 sphinx
 sphinx_rtd_theme
 sphinx-tabs

setup.cfg

@@ -77,7 +77,7 @@ console_scripts =
 
 [egg_info]
 # TODO replace
-tag_build = dev1
+tag_build = dev2
 
 [aliases]
 test=pytest