Merge branch 'main' into add-flwr-log

.github/CODEOWNERS

@@ -7,7 +7,10 @@
 README.md @jafermarq @tanertopal @danieljanes
 
 # Flower Baselines
-/baselines @jafermarq @tanertopal @danieljanes
+/baselines @jafermarq @danieljanes
+
+# Flower Benchmarks
+/benchmarks @jafermarq @danieljanes
 
 # Flower Datasets
 /datasets @jafermarq @tanertopal @danieljanes
@@ -27,3 +30,9 @@ README.md @jafermarq @tanertopal @danieljanes
 # GitHub Actions and Workflows
 /.github/workflows @Robert-Steiner @tanertopal @danieljanes
 /.github/actions @Robert-Steiner @tanertopal @danieljanes
+
+# Docker-related files
+/.devcontainer @Robert-Steiner @Moep90
+**/Dockerfile @Robert-Steiner @Moep90
+**/*.Dockerfile @Robert-Steiner @Moep90
+src/docker @Robert-Steiner @Moep90

.github/workflows/_docker-build.yml

@@ -36,18 +36,16 @@ permissions:
 jobs:
   build:
     name: Build image
-    runs-on: ubuntu-22.04
+    runs-on: ${{ matrix.platform.runner-os }}
     timeout-minutes: 180
     outputs:
       build-id: ${{ steps.build-id.outputs.id }}
     strategy:
       fail-fast: true
       matrix:
         platform: [
-            # build-push action and qemu use different platform names
-            # therefore we create a map
-            { name: "amd64", qemu: "", docker: "linux/amd64" },
-            { name: "arm64", qemu: "arm64", docker: "linux/arm64" },
+            { name: "amd64", docker: "linux/amd64", runner-os: "ubuntu-22.04" },
+            { name: "arm64", docker: "linux/arm64", runner-os: "ubuntu-4-core-arm64" },
           ]
     steps:
       - name: Create build id
@@ -79,23 +77,17 @@ jobs:
               print(build_args, file=fh)
               print("EOF", file=fh)
 
-      - name: Set up QEMU
-        if: matrix.platform.qemu != ''
-        uses: docker/setup-qemu-action@5927c834f5b4fdf503fca6f4c7eccda82949e1ee # v3.1.0
-        with:
-          platforms: ${{ matrix.platform.qemu }}
-
       - name: Extract metadata (tags, labels) for Docker
         id: meta
         uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1
         with:
           images: ${{ inputs.namespace-repository }}
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@4fd812986e6c8c2a69e18311145f9371337f27d4 # v3.4.0
+        uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v3.6.1
 
       - name: Login to Docker Hub
-        uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
         with:
           username: ${{ secrets.dockerhub-user }}
           password: ${{ secrets.dockerhub-token }}
@@ -104,7 +96,7 @@ jobs:
         uses: Wandalen/wretry.action@6feedb7dedadeb826de0f45ff482b53b379a7844 # v3.5.0
         id: build
         with:
-          action: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0
+          action: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85 # v6.7.0
           attempt_limit: 60 # 60 attempts * (9 secs delay + 1 sec retry) = ~10 mins
           attempt_delay: 9000 # 9 secs
           with: |
@@ -122,7 +114,7 @@ jobs:
           touch "/tmp/digests/${digest#sha256:}"
 
       - name: Upload digest
-        uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4
+        uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
         with:
           name: digests-${{ steps.build-id.outputs.id }}-${{ matrix.platform.name }}
           path: /tmp/digests/*
@@ -152,10 +144,10 @@ jobs:
           tags: ${{ inputs.tags }}
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@4fd812986e6c8c2a69e18311145f9371337f27d4 # v3.4.0
+        uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v3.6.1
 
       - name: Login to Docker Hub
-        uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
         with:
           username: ${{ secrets.dockerhub-user }}
           password: ${{ secrets.dockerhub-token }}

.github/workflows/docker-readme.yml

@@ -0,0 +1,51 @@
+name: Update Docker READMEs
+
+on:
+  push:
+    branches:
+      - 'main'
+    paths:
+      - 'src/docker/**/README.md'
+
+jobs:
+    collect:
+      if: ${{ github.repository == 'adap/flower' }}
+      name: Collect Docker READMEs
+      runs-on: ubuntu-22.04
+      timeout-minutes: 10
+      outputs:
+        readme_files: ${{ steps.filter.outputs.readme_files }}
+      steps:
+        - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+        - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
+          id: filter
+          with:
+            list-files: "json"
+            filters: |
+              readme:
+                - 'src/docker/**/README.md'
+
+    update:
+      if: ${{ needs.collect.outputs.readme_files != '' && toJson(fromJson(needs.collect.outputs.readme_files)) != '[]' }}
+      name: Update Docker READMEs
+      runs-on: ubuntu-22.04
+      timeout-minutes: 10
+      needs: collect
+      strategy:
+        matrix:
+          readme_path: ${{ fromJSON(needs.collect.outputs.readme_files) }}
+
+      steps:
+        - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+        - id: repository
+          run: echo "name=$(basename $(dirname ${{ matrix.readme_path }}))" >> "$GITHUB_OUTPUT"
+
+        - name: Docker Hub Description
+          uses: peter-evans/dockerhub-description@e98e4d1628a5f3be2be7c231e50981aee98723ae # v4.0.0
+          with:
+            repository: flwr/${{ steps.repository.outputs.name }}
+            readme-filepath: ${{ matrix.readme_path }}
+            username: ${{ secrets.DOCKERHUB_USERNAME }}
+            password: ${{ secrets.DOCKERHUB_TOKEN }}

.github/workflows/e2e.yml

@@ -60,45 +60,56 @@ jobs:
     strategy:
       matrix:
         include:
-          - directory: bare
+          - directory: e2e-bare
+            e2e: e2e_bare
 
-          - directory: bare-https
+          - directory: e2e-bare-https
+            e2e: e2e_bare_https
 
-          - directory: bare-client-auth
+          - directory: e2e-bare-auth
+            e2e: e2e_bare_auth
 
-          - directory: framework-jax
+          - directory: e2e-jax
+            e2e: e2e_jax
 
-          - directory: framework-pytorch
+          - directory: e2e-pytorch
+            e2e: e2e_pytorch
             dataset: |
               from torchvision.datasets import CIFAR10
               CIFAR10('./data', download=True)
 
-          - directory: framework-tensorflow
+          - directory: e2e-tensorflow
+            e2e: e2e_tensorflow
             dataset: |
               import tensorflow as tf
               tf.keras.datasets.cifar10.load_data()
 
-          - directory: framework-opacus
+          - directory: e2e-opacus
+            e2e: e2e_opacus
             dataset: |
               from torchvision.datasets import CIFAR10
               CIFAR10('./data', download=True)
 
-          - directory: framework-pytorch-lightning
+          - directory: e2e-pytorch-lightning
+            e2e: e2e_pytorch_lightning
             dataset: |
               from torchvision.datasets import MNIST
               MNIST('./data', download=True)
 
-          - directory: framework-scikit-learn
+          - directory: e2e-scikit-learn
+            e2e: e2e_scikit_learn
             dataset: |
               import openml
               openml.datasets.get_dataset(554)
 
-          - directory: framework-fastai
+          - directory: e2e-fastai
+            e2e: e2e_fastai
             dataset: |
               from fastai.vision.all import untar_data, URLs
               untar_data(URLs.MNIST)
 
-          - directory: framework-pandas
+          - directory: e2e-pandas
+            e2e: e2e_pandas
             dataset: |
               from pathlib import Path
               from sklearn.datasets import load_iris
@@ -135,32 +146,35 @@ jobs:
         if: ${{ github.repository == 'adap/flower' && !github.event.pull_request.head.repo.fork && github.actor != 'dependabot[bot]' }}
         run: |
           python -m pip install https://${{ env.ARTIFACT_BUCKET }}/py/${{ needs.wheel.outputs.dir }}/${{ needs.wheel.outputs.short_sha }}/${{ needs.wheel.outputs.whl_path }}
+      - name: Install e2e components
+        run: pip install .
       - name: Download dataset
         if: ${{ matrix.dataset }}
         run: python -c "${{ matrix.dataset }}"
       - name: Run edge client test
-        if: ${{ matrix.directory != 'bare-client-auth' }}
-        run: ./../test_legacy.sh "${{ matrix.directory }}"
+        if: ${{ matrix.directory != 'e2e-bare-auth' }}
+        working-directory: e2e/${{ matrix.directory }}/${{ matrix.e2e }}
+        run: ./../../test_legacy.sh "${{ matrix.directory }}"
       - name: Run virtual client test
-        if: ${{ matrix.directory != 'bare-client-auth' }}
+        if: ${{ matrix.directory != 'e2e-bare-auth' }}
         run: python simulation.py
       - name: Run simulation engine test
-        if: ${{ matrix.directory == 'pytorch' || matrix.directory == 'tensorflow'}}
+        if: ${{ matrix.directory == 'e2e-pytorch' || matrix.directory == 'e2e-tensorflow'}}
         run: python simulation_next.py
       - name: Run driver test
-        if: ${{ matrix.directory != 'bare-client-auth' }}
+        if: ${{ matrix.directory != 'e2e-bare-auth' }}
         run: ./../test_superlink.sh "${{ matrix.directory }}"
       - name: Run driver test with REST
-        if: ${{ matrix.directory == 'bare' }}
+        if: ${{ matrix.directory == 'e2e-bare' }}
         run: ./../test_superlink.sh bare rest
       - name: Run driver test with SQLite database
-        if: ${{ matrix.directory == 'bare' }}
+        if: ${{ matrix.directory == 'e2e-bare' }}
         run: ./../test_superlink.sh bare sqlite
       - name: Run driver test with client authentication
-        if: ${{ matrix.directory == 'bare-client-auth' }}
-        run: ./../test_superlink.sh bare client-auth
+        if: ${{ matrix.directory == 'e2e-bare-auth' }}
+        run: ./../test_superlink.sh "${{ matrix.directory }}" client-auth
       - name: Run reconnection test with SQLite database
-        if: ${{ matrix.directory == 'bare' }}
+        if: ${{ matrix.directory == 'e2e-bare' }}
         run: ./../test_reconnection.sh sqlite
       - name: Cache save Python location
         id: cache-save-python

.github/workflows/framework-release.yml

@@ -16,6 +16,8 @@ jobs:
     if: ${{ github.repository == 'adap/flower' }}
     name: Publish release
     runs-on: ubuntu-22.04
+    outputs:
+      flwr-version: ${{ steps.publish.outputs.flwr-version }}
     steps:
     - name: Checkout code
       uses: actions/checkout@v4
@@ -26,10 +28,12 @@ jobs:
       uses: ./.github/actions/bootstrap
 
     - name: Get artifacts and publish
+      id: publish
       env:
         GITHUB_REF: ${{ github.ref }}
       run: |
         TAG_NAME=$(echo "${GITHUB_REF_NAME}" | cut -c2-)
+        echo "flwr-version=$TAG_NAME" >> "$GITHUB_OUTPUT"
 
         wheel_name="flwr-${TAG_NAME}-py3-none-any.whl"
         tar_name="flwr-${TAG_NAME}.tar.gz"
@@ -67,8 +71,7 @@ jobs:
 
       - id: matrix
         run: |
-           FLWR_VERSION=$(poetry version -s)
-           python dev/build-docker-image-matrix.py --flwr-version "${FLWR_VERSION}" > matrix.json
+           python dev/build-docker-image-matrix.py --flwr-version "${{ needs.publish.outputs.flwr-version }}" > matrix.json
            echo "matrix=$(cat matrix.json)" >> $GITHUB_OUTPUT
 
   build-base-images:

.github/workflows/release-nightly.yml

@@ -70,7 +70,8 @@ jobs:
           { repository: "flwr/superlink", file_dir: "src/docker/superlink" },
           { repository: "flwr/supernode", file_dir: "src/docker/supernode" },
           { repository: "flwr/serverapp", file_dir: "src/docker/serverapp" },
-          { repository: "flwr/superexec", file_dir: "src/docker/superexec" }
+          { repository: "flwr/superexec", file_dir: "src/docker/superexec" },
+          { repository: "flwr/clientapp", file_dir: "src/docker/clientapp" }
         ]
     with:
       namespace-repository: ${{ matrix.images.repository }}

README.md

@@ -143,8 +143,8 @@ Other [examples](https://github.com/adap/flower/tree/main/examples):
 - [PyTorch: From Centralized to Federated](https://github.com/adap/flower/tree/main/examples/pytorch-from-centralized-to-federated)
 - [Vertical FL](https://github.com/adap/flower/tree/main/examples/vertical-fl)
 - [Federated Finetuning of OpenAI's Whisper](https://github.com/adap/flower/tree/main/examples/whisper-federated-finetuning)
-- [Federated Finetuning of Large Language Model](https://github.com/adap/flower/tree/main/examples/llm-flowertune)
-- [Federated Finetuning of a Vision Transformer](https://github.com/adap/flower/tree/main/examples/vit-finetune)
+- [Federated Finetuning of Large Language Model](https://github.com/adap/flower/tree/main/examples/flowertune-llm)
+- [Federated Finetuning of a Vision Transformer](https://github.com/adap/flower/tree/main/examples/flowertune-vit)
 - [Advanced Flower with TensorFlow/Keras](https://github.com/adap/flower/tree/main/examples/advanced-tensorflow)
 - [Advanced Flower with PyTorch](https://github.com/adap/flower/tree/main/examples/advanced-pytorch)
 - Single-Machine Simulation of Federated Learning Systems ([PyTorch](https://github.com/adap/flower/tree/main/examples/simulation-pytorch)) ([Tensorflow](https://github.com/adap/flower/tree/main/examples/simulation-tensorflow))
@@ -160,7 +160,7 @@ Other [examples](https://github.com/adap/flower/tree/main/examples):
 Flower is built by a wonderful community of researchers and engineers. [Join Slack](https://flower.ai/join-slack) to meet them, [contributions](#contributing-to-flower) are welcome.
 
 <a href="https://github.com/adap/flower/graphs/contributors">
-  <img src="https://contrib.rocks/image?repo=adap/flower" />
+  <img src="https://contrib.rocks/image?repo=adap/flower&columns=10" />
 </a>
 
 ## Citation