Edit

adap · Sep 20, 2024 · 3b5e9ac · 3b5e9ac
2 parents 026aa8b + 02e1813
commit 3b5e9ac
Show file tree

Hide file tree

Showing 1,033 changed files with 127,137 additions and 82,382 deletions.
diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
@@ -29,8 +29,8 @@ RUN apt-get install -y curl wget gnupg python3 python-is-python3 python3-pip git
     build-essential tmux vim
 
 RUN python -m pip install \
-    pip==24.0.0 \
-    setuptools==69.5.1 \
+    pip==24.1.2 \
+    setuptools==70.3.0 \
     poetry==1.7.1
 
 USER $USERNAME

diff --git a/.editorconfig b/.editorconfig
@@ -16,6 +16,14 @@ profile = black
 indent_style = space
 indent_size = 2
 
+[*.md]
+indent_style = space
+indent_size = 2
+
 [*.yml]
 indent_style = space
 indent_size = 2
+
+[*.toml]
+indent_style = space
+indent_size = 4
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -7,14 +7,20 @@
 README.md @jafermarq @tanertopal @danieljanes
 
 # Flower Baselines
-/baselines @jafermarq @tanertopal @danieljanes
+/baselines @jafermarq @danieljanes
+
+# Flower Benchmarks
+/benchmarks @jafermarq @danieljanes
 
 # Flower Datasets
 /datasets @jafermarq @tanertopal @danieljanes
 
 # Flower Examples
 /examples @jafermarq @tanertopal @danieljanes
 
+# Flower Templates
+/src/py/flwr/cli/new/templates @jafermarq @tanertopal @danieljanes
+
 # Changelog
 /doc/source/ref-changelog.md @jafermarq @tanertopal @danieljanes
 
@@ -24,3 +30,9 @@ README.md @jafermarq @tanertopal @danieljanes
 # GitHub Actions and Workflows
 /.github/workflows @Robert-Steiner @tanertopal @danieljanes
 /.github/actions @Robert-Steiner @tanertopal @danieljanes
+
+# Docker-related files
+/.devcontainer @Robert-Steiner @Moep90 @tanertopal @danieljanes
+**/Dockerfile @Robert-Steiner @Moep90 @tanertopal @danieljanes
+**/*.Dockerfile @Robert-Steiner @Moep90 @tanertopal @danieljanes
+/src/docker @Robert-Steiner @Moep90 @tanertopal @danieljanes
diff --git a/.github/actions/bootstrap/action.yml b/.github/actions/bootstrap/action.yml
@@ -3,13 +3,13 @@ description: "Bootstrap Python environment (install and configure Python version
 inputs:
   python-version:
     description: "Version range or exact version of Python or PyPy to use, using SemVer's version range syntax."
-    default: 3.8
+    default: 3.9
   pip-version:
     description: "Version of pip to be installed using pip"
-    default: 24.0.0
+    default: 24.1.2
   setuptools-version:
     description: "Version of setuptools to be installed using pip"
-    default: 69.5.1
+    default: 70.3.0
   poetry-version:
     description: "Version of poetry to be installed using pip"
     default: 1.7.1

diff --git a/.github/workflows/_docker-build.yml b/.github/workflows/_docker-build.yml
@@ -36,18 +36,16 @@ permissions:
 jobs:
   build:
     name: Build image
-    runs-on: ubuntu-22.04
+    runs-on: ${{ matrix.platform.runner-os }}
     timeout-minutes: 180
     outputs:
       build-id: ${{ steps.build-id.outputs.id }}
     strategy:
       fail-fast: true
       matrix:
         platform: [
-            # build-push action and qemu use different platform names
-            # therefore we create a map
-            { name: "amd64", qemu: "", docker: "linux/amd64" },
-            { name: "arm64", qemu: "arm64", docker: "linux/arm64" },
+            { name: "amd64", docker: "linux/amd64", runner-os: "ubuntu-22.04" },
+            { name: "arm64", docker: "linux/arm64", runner-os: "ubuntu-4-core-arm64" },
           ]
     steps:
       - name: Create build id
@@ -79,23 +77,17 @@ jobs:
               print(build_args, file=fh)
               print("EOF", file=fh)
 
-      - name: Set up QEMU
-        if: matrix.platform.qemu != ''
-        uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0
-        with:
-          platforms: ${{ matrix.platform.qemu }}
-
       - name: Extract metadata (tags, labels) for Docker
         id: meta
         uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1
         with:
           images: ${{ inputs.namespace-repository }}
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0
+        uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v3.6.1
 
       - name: Login to Docker Hub
-        uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
         with:
           username: ${{ secrets.dockerhub-user }}
           password: ${{ secrets.dockerhub-token }}
@@ -104,7 +96,7 @@ jobs:
         uses: Wandalen/wretry.action@6feedb7dedadeb826de0f45ff482b53b379a7844 # v3.5.0
         id: build
         with:
-          action: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0
+          action: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85 # v6.7.0
           attempt_limit: 60 # 60 attempts * (9 secs delay + 1 sec retry) = ~10 mins
           attempt_delay: 9000 # 9 secs
           with: |
@@ -122,7 +114,7 @@ jobs:
           touch "/tmp/digests/${digest#sha256:}"
 
       - name: Upload digest
-        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
+        uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
         with:
           name: digests-${{ steps.build-id.outputs.id }}-${{ matrix.platform.name }}
           path: /tmp/digests/*
@@ -138,7 +130,7 @@ jobs:
       metadata: ${{ steps.meta.outputs.json }}
     steps:
       - name: Download digests
-        uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
+        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
         with:
           pattern: digests-${{ needs.build.outputs.build-id }}-*
           path: /tmp/digests
@@ -152,10 +144,10 @@ jobs:
           tags: ${{ inputs.tags }}
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0
+        uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v3.6.1
 
       - name: Login to Docker Hub
-        uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0
+        uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
         with:
           username: ${{ secrets.dockerhub-user }}
           password: ${{ secrets.dockerhub-token }}

diff --git a/.github/workflows/cache-cleanup.yml b/.github/workflows/cache-cleanup.yml
@@ -34,7 +34,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: 3.8
+          python-version: 3.9
 
       - name: Cleanup caches by directories
         # Only keep caches that match the latest keys for each directory

diff --git a/.github/workflows/datasets-e2e.yml b/.github/workflows/datasets-e2e.yml
@@ -45,7 +45,7 @@ jobs:
       - name: Bootstrap
         uses: ./.github/actions/bootstrap
         with:
-          python-version: 3.8
+          python-version: 3.9
       - name: Install dependencies
         run: python -m poetry install
       - name: Run tests

diff --git a/.github/workflows/datasets.yml b/.github/workflows/datasets.yml
@@ -6,16 +6,23 @@ on:
       - main
     paths:
       - "datasets/**"
+      - ".github/workflows/datasets.yml"
+      - ".github/workflows/datasets-e2e.yml"
   pull_request:
     branches:
       - main
     paths:
       - "datasets/**"
+      - ".github/workflows/datasets.yml"
+      - ".github/workflows/datasets-e2e.yml"
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_id || github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
+env:
+  FLWR_TELEMETRY_ENABLED: 0
+
 defaults:
   run:
     working-directory: datasets
@@ -28,9 +35,9 @@ jobs:
         # Latest version which comes cached in the host image can be found here:
         # https://github.com/actions/runner-images/blob/main/images/linux/Ubuntu2204-Readme.md#python
         # In case of a mismatch, the job has to download Python to install it.
-        # Note: Due to a bug in actions/setup-python we have to put 3.10 in
-        # qoutes as it will otherwise will assume 3.1
-        python: [3.8, 3.9, '3.10']
+        # Note: Due to a bug in actions/setup-python, we have to put "3.10" in
+        # quotes as it will otherwise assume "3.1"
+        python: ['3.9', '3.10', '3.11']
 
     name: Python ${{ matrix.python }}
 

diff --git a/.github/workflows/docker-images.yml → .github/workflows/docker-build-main.yml b/.github/workflows/docker-images.yml → .github/workflows/docker-build-main.yml
@@ -1,25 +1,20 @@
-name: Build docker images
+name: Build Docker Images Main Branch
 
 on:
-  workflow_dispatch:
-    inputs:
-      flwr-version:
-        description: "Version of Flower."
-        required: true
-        type: string
-
-permissions:
-  contents: read
+  push:
+    branches:
+      - 'main'
 
 jobs:
   parameters:
-    name: Collect build parameters
+    if: github.repository == 'adap/flower'
+    name: Collect docker build parameters
     runs-on: ubuntu-22.04
     timeout-minutes: 10
     outputs:
       pip-version: ${{ steps.versions.outputs.pip-version }}
       setuptools-version: ${{ steps.versions.outputs.setuptools-version }}
-      matrix: ${{ steps.matrix.outputs.matrix }}
+      flwr-version-ref: ${{ steps.versions.outputs.flwr-version-ref }}
     steps:
       - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
@@ -30,46 +25,45 @@ jobs:
         run: |
           echo "pip-version=${{ steps.bootstrap.outputs.pip-version }}" >> "$GITHUB_OUTPUT"
           echo "setuptools-version=${{ steps.bootstrap.outputs.setuptools-version }}" >> "$GITHUB_OUTPUT"
+          echo "flwr-version-ref=git+${{ github.server_url }}/${{ github.repository }}.git@${{ github.sha }}" >> "$GITHUB_OUTPUT"
 
-      - id: matrix
-        run: |
-           python dev/build-docker-image-matrix.py --flwr-version ${{ github.event.inputs.flwr-version }} > matrix.json
-           echo "matrix=$(cat matrix.json)" >> $GITHUB_OUTPUT
-
-  build-base-images:
+  build-docker-base-images:
     name: Build base images
+    if: github.repository == 'adap/flower'
     uses: ./.github/workflows/_docker-build.yml
     needs: parameters
-    strategy:
-      fail-fast: false
-      matrix: ${{ fromJson(needs.parameters.outputs.matrix).base }}
     with:
-      namespace-repository: ${{ matrix.images.namespace_repository }}
-      file-dir: ${{ matrix.images.file_dir }}
+      namespace-repository: flwr/base
+      file-dir: src/docker/base/ubuntu
       build-args: |
-        PYTHON_VERSION=${{ matrix.images.python_version }}
         PIP_VERSION=${{ needs.parameters.outputs.pip-version }}
         SETUPTOOLS_VERSION=${{ needs.parameters.outputs.setuptools-version }}
-        DISTRO=${{ matrix.images.distro.name }}
-        DISTRO_VERSION=${{ matrix.images.distro.version }}
-        FLWR_VERSION=${{ matrix.images.flwr_version }}
-      tags: ${{ matrix.images.tag }}
+        FLWR_VERSION_REF=${{ needs.parameters.outputs.flwr-version-ref }}
+      tags: unstable
     secrets:
       dockerhub-user: ${{ secrets.DOCKERHUB_USERNAME }}
       dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }}
 
-  build-binary-images:
+  build-docker-binary-images:
     name: Build binary images
+    if: github.repository == 'adap/flower'
     uses: ./.github/workflows/_docker-build.yml
-    needs: [parameters, build-base-images]
+    needs: build-docker-base-images
     strategy:
       fail-fast: false
-      matrix: ${{ fromJson(needs.parameters.outputs.matrix).binary }}
+      matrix:
+        images: [
+          { repository: "flwr/superlink", file_dir: "src/docker/superlink" },
+          { repository: "flwr/supernode", file_dir: "src/docker/supernode" },
+          { repository: "flwr/serverapp", file_dir: "src/docker/serverapp" },
+          { repository: "flwr/superexec", file_dir: "src/docker/superexec" },
+          { repository: "flwr/clientapp", file_dir: "src/docker/clientapp" }
+        ]
     with:
-      namespace-repository: ${{ matrix.images.namespace_repository }}
+      namespace-repository: ${{ matrix.images.repository }}
       file-dir: ${{ matrix.images.file_dir }}
-      build-args: BASE_IMAGE=${{ matrix.images.base_image }}
-      tags: ${{ matrix.images.tags }}
+      build-args: BASE_IMAGE=unstable
+      tags: unstable
     secrets:
       dockerhub-user: ${{ secrets.DOCKERHUB_USERNAME }}
       dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }}
diff --git a/.github/workflows/docker-readme.yml b/.github/workflows/docker-readme.yml
@@ -0,0 +1,51 @@
+name: Update Docker READMEs
+
+on:
+  push:
+    branches:
+      - 'main'
+    paths:
+      - 'src/docker/**/README.md'
+
+jobs:
+    collect:
+      if: ${{ github.repository == 'adap/flower' }}
+      name: Collect Docker READMEs
+      runs-on: ubuntu-22.04
+      timeout-minutes: 10
+      outputs:
+        readme_files: ${{ steps.filter.outputs.readme_files }}
+      steps:
+        - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+        - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
+          id: filter
+          with:
+            list-files: "json"
+            filters: |
+              readme:
+                - 'src/docker/**/README.md'
+
+    update:
+      if: ${{ needs.collect.outputs.readme_files != '' && toJson(fromJson(needs.collect.outputs.readme_files)) != '[]' }}
+      name: Update Docker READMEs
+      runs-on: ubuntu-22.04
+      timeout-minutes: 10
+      needs: collect
+      strategy:
+        matrix:
+          readme_path: ${{ fromJSON(needs.collect.outputs.readme_files) }}
+
+      steps:
+        - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+        - id: repository
+          run: echo "name=$(basename $(dirname ${{ matrix.readme_path }}))" >> "$GITHUB_OUTPUT"
+
+        - name: Docker Hub Description
+          uses: peter-evans/dockerhub-description@e98e4d1628a5f3be2be7c231e50981aee98723ae # v4.0.0
+          with:
+            repository: flwr/${{ steps.repository.outputs.name }}
+            readme-filepath: ${{ matrix.readme_path }}
+            username: ${{ secrets.DOCKERHUB_USERNAME }}
+            password: ${{ secrets.DOCKERHUB_TOKEN }}