LG-77 Update build system and remove the Docker container

- Docker container is removed, because due to the simpler UPB protobuf backend, there is no need to build and distribute the legacy CPP based protobuf backend. - Stop using setup.py and use setuptools as a backend. Protobuf requirement is specified in setup.py directly for simpler builds. - Update the latest python version for the github workflow to 3.13. - Add instructions for local installation to the readme. - Make non-release versions somewhat PEP404 compliant in the github workflow.
wirepas · Jan 29, 2025 · ee4e131 · ee4e131
1 parent 096950a
commit ee4e131
Show file tree

Hide file tree

Showing 6 changed files with 49 additions and 197 deletions.
diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
@@ -1,7 +1,7 @@
 # This workflow will install Python dependencies, run tests and lint with a variety of Python versions
 # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
 
-name: Python package and Docker image
+name: Python package
 
 on:
   push:
@@ -14,50 +14,47 @@ on:
   workflow_dispatch:
     inputs:
       version_name:
-        description: 'Version used on testPypi and dockerHub'
+        description: 'Version used for manual triggering'
         required: true
 
-env:
-  IMAGE_NAME: 'wirepas/wmm_alpine_cpp'
-
 jobs:
   build:
 
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ['3.9', '3.10', '3.11']
+        python-version: ['3.9', '3.10', '3.11', '3.12', '3.13']
+        include:
+          # Mark below python version as the latest one
+          - python-version: '3.13'
+            python-latest: true
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: true
 
     - name: Set proto path
       run: echo "PROTO_FOLDER=backend-apis/gateway_to_backend/protocol_buffers_files" >> $GITHUB_ENV
 
     - name: Install Protoc
-      uses: arduino/setup-protoc@v1.1.2
+      uses: arduino/setup-protoc@v3
       with:
-         version: '3.20.3'
+         version: '29.x'
          repo-token: ${{ secrets.GITHUB_TOKEN }}
 
     - name: Generate proto files
       run: |
         protoc -I ${{ env.PROTO_FOLDER }} --python_out=./wirepas_mesh_messaging/proto ${{ env.PROTO_FOLDER }}/*.proto
 
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
 
     - name: Install dependencies
       run: |
-        # Use sepcific version of pip as temporary fix for deprecation of setup.py install
-        # being an error in 23.1
-        python -m pip install pip==23.0
-        python -m pip install flake8 pytest setuptools wheel
-        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+        python -m pip install flake8 pytest build
 
     - name: Lint with flake8
       run: |
@@ -81,15 +78,15 @@ jobs:
 
     - name: Set Version for master push
       if: ${{ github.event_name == 'push' }}
-      run: echo "VERSION=${{ steps.short-sha.outputs.sha }}" >> $GITHUB_ENV
+      run: echo "VERSION=0+${{ steps.short-sha.outputs.sha }}" >> $GITHUB_ENV
 
     - name: Set Version for release
       if: ${{ github.event_name == 'release' }}
       run: echo "VERSION=${{ github.event.release.tag_name }}" >> $GITHUB_ENV
 
     - name: Set Version for PR
       if: ${{ github.event_name == 'pull_request' }}
-      run: echo "VERSION=PR_${{ github.event.pull_request.number }}" >> $GITHUB_ENV
+      run: echo "VERSION=0+PR_${{ github.event.pull_request.number }}" >> $GITHUB_ENV
 
     - name: Set Version for manual triggering
       if: ${{ github.event_name == 'workflow_dispatch' }}
@@ -100,20 +97,20 @@ jobs:
         sed -i '/__version__ = /c\__version__ = "${{ env.VERSION }}"' wirepas_mesh_messaging/__about__.py
 
     - name: Build the wheel
-      run: python setup.py sdist bdist_wheel
+      run: python -m build .
 
     - name: Save produced wheel name and path
       run: echo "WHEEL_FILE=$(ls dist/*-py3-none-any.whl)" >> $GITHUB_ENV
 
     - name: Store artifacts only for last matrix version
-      if: ${{ matrix.python-version == '3.11' }}
+      if: ${{ matrix.python-latest }}
       uses: actions/upload-artifact@v4
       with:
         path: ${{ env.WHEEL_FILE }}
 
     # Produced wheels is same independently of python version
     - name: Upload Wheel for releases only for last matrix version
-      if: ${{ github.event_name == 'release' && matrix.python-version == '3.11' }}
+      if: ${{ github.event_name == 'release' && matrix.python-latest }}
       uses: actions/upload-release-asset@v1
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -123,81 +120,10 @@ jobs:
         asset_content_type: application/zip
         asset_name: wirepas_mesh_messaging-${{ env.VERSION }}-py3-none-any.whl
 
-    - name: Publish package to TestPyPI for push to master
-      if: ${{ github.event_name == 'workflow_dispatch' && matrix.python-version == '3.11' }}
-      uses: pypa/[email protected]
-      with:
-        user: __token__
-        password: ${{ secrets.PYPI_TEST_PWD }}
-        repository_url: https://test.pypi.org/legacy/
-
     - name: Publish package to PyPI for release
-      if: ${{ github.event_name == 'release' && matrix.python-version == '3.11' }}
+      if: ${{ github.event_name == 'release' && matrix.python-latest }}
       uses: pypa/[email protected]
       with:
         user: __token__
         password: ${{ secrets.PYPI_PWD }}
 
-  docker:
-    runs-on: ubuntu-latest
-    needs: build
-    steps:
-    - name: checkout code
-      uses: actions/checkout@v3
-
-    - name: Set up QEMU
-      uses: docker/setup-qemu-action@v2
-
-    - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v2
-
-    - name: Retrieve wheel to be installed image
-      uses: actions/download-artifact@v4
-
-    - name: Set tag for push
-      if: github.event_name == 'push'
-      run: |
-        echo "TAG1=$IMAGE_NAME:edge" >> $GITHUB_ENV
-        echo "PUSH=true" >> $GITHUB_ENV
-
-    - name: Set tag for manually triggered
-      if: github.event_name == 'workflow_dispatch'
-      run: |
-        echo "TAG1=$IMAGE_NAME:${{ github.event.inputs.version_name }}" >> $GITHUB_ENV
-        echo "PUSH=true" >> $GITHUB_ENV
-
-    - name: Set tag for release version
-      if: github.event_name == 'release'
-      run: |
-        echo "TAG1=$IMAGE_NAME:${{ github.event.release.tag_name }}" >> $GITHUB_ENV
-        echo "PUSH=true" >> $GITHUB_ENV
-
-    - name: Set additionnal latest tag also for official release
-      if: github.event_name == 'release' && !contains(github.event.release.tag_name, 'rc')
-      run: |
-        echo "TAG2=$IMAGE_NAME:latest" >> $GITHUB_ENV
-        echo "PUSH=true" >> $GITHUB_ENV
-
-    - name: Do not push on docker hub for pull_request
-      if: github.event_name == 'pull_request'
-      run: |
-        echo "PUSH=false" >> $GITHUB_ENV
-
-    - name: Login to docker hub
-      uses: docker/login-action@v2
-      with:
-        username: ${{ secrets.DOCKER_USERNAME }}
-        password: ${{ secrets.DOCKER_PASSWORD }}
-
-    - name: Build and push
-      uses: docker/build-push-action@v3
-      with:
-        context: .
-        file: docker/Dockerfile
-        platforms: linux/amd64, linux/arm64, linux/arm/v7, linux/arm/v6
-        push: ${{ env.PUSH }}
-        build-args: WIREPAS_MESH_MESSAGING_BUIL_SHA1=${{ github.sha }}
-        tags: |
-          ${{ env.TAG1 }}
-          ${{ env.TAG2 }}
-
diff --git a/README.md b/README.md
@@ -16,9 +16,32 @@ This package is available from [PyPi][pypi].
 
 ### From the source
 
-This wheel can be built from source directly.
-Flow will be described here but can already be seen from this repository [Github action](.github/workflows/python-package.yml).
+This wheel can be built from source directly. You need to have [protobuf
+compiler](https://grpc.io/docs/protoc-installation/) installed.
 
+First, .proto files should be compiled
+```shell
+protoc -I backend-apis/gateway_to_backend/protocol_buffers_files/ \
+    --python_out=./wirepas_mesh_messaging/proto \
+     backend-apis/gateway_to_backend/protocol_buffers_files/*.proto
+```
+
+After that, the wheel can be built using [build](https://pypi.org/project/build/).
+```shell
+python3 -m build .
+```
+
+**Note:** When installing the package,
+[protobuf](https://pypi.org/project/protobuf/) is installed automatically as a
+dependency. On some environments (such as Alpine) the [recommended UPB
+backend](https://github.com/protocolbuffers/protobuf/tree/main/python#implementation-backends)
+might not available as a binary and the system might use the slower python
+implementation. To get the UBP backend, you can configure pip to use the source
+version of protobuf. This way the UPB backend would be built when installing
+protobuf:
+```shell
+pip install . --no-binary protobuf
+```
 ## Usage example
 
 ### Create a message to be sent to a Gateway in protobuf format

diff --git a/docker/Dockerfile b/docker/Dockerfile
diff --git a/pyproject.toml b/pyproject.toml
@@ -0,0 +1,3 @@
+[build-system]
+requires = ["setuptools>=61.0", "protobuf_distutils"]
+build-backend = "setuptools.build_meta"
diff --git a/requirements.txt b/requirements.txt
diff --git a/setup.py b/setup.py
@@ -9,49 +9,22 @@
 """
 
 import os
-import re
-import glob
 
 from setuptools import setup, find_packages
 
 here = os.path.abspath(os.path.dirname(__file__))
 readme_file = "README.md"
 license_file = "LICENSE"
-requirements_file = "requirements.txt"
 
 with open(readme_file) as f:
     long_description = f.read()
 
 
-def get_list_files(root, flist=None):
-    if flist is None:
-        flist = list()
-
-    for path, subdirs, files in os.walk(root):
-        for name in files:
-            flist.append(os.path.join(path, name))
-    print("FLIST", flist)
-    return flist
-
-
 def get_absolute_path(*args):
     """ Transform relative pathnames into absolute pathnames """
     return os.path.join(here, *args)
 
 
-def get_requirements(*args):
-    """ Get requirements requirements.txt """
-    requirements = set()
-    with open(get_absolute_path(*args)) as handle:
-        for line in handle:
-            # Strip comments.
-            line = re.sub(r"^#.*|\s#.*", "", line)
-            # Ignore empty lines
-            if line and not line.isspace():
-                requirements.add(re.sub(r"\s+", "", line))
-    return sorted(requirements)
-
-
 about = {}
 with open(get_absolute_path("./wirepas_mesh_messaging/__about__.py")) as f:
     exec(f.read(), about)
@@ -69,5 +42,7 @@ def get_requirements(*args):
     classifiers=about["__classifiers__"],
     keywords=about["__keywords__"],
     packages=find_packages(exclude=["contrib", "docs", "tests", "examples"]),
-    install_requires=get_requirements(requirements_file),
+    install_requires=[
+        "protobuf~=5.0"
+    ]
 )