Merge branch 'main' into rename-resplitter-in-examples

.devcontainer/Dockerfile

@@ -29,8 +29,8 @@ RUN apt-get install -y curl wget gnupg python3 python-is-python3 python3-pip git
     build-essential tmux vim
 
 RUN python -m pip install \
-    pip==23.3.1 \
-    setuptools==68.2.2 \
+    pip==24.0.0 \
+    setuptools==69.5.1 \
     poetry==1.7.1
 
 USER $USERNAME

.github/CODEOWNERS

@@ -15,9 +15,15 @@ README.md @jafermarq @tanertopal @danieljanes
 # Flower Examples
 /examples @jafermarq @tanertopal @danieljanes
 
+# Flower Templates
+/src/py/flwr/cli/new/templates @jafermarq @tanertopal @danieljanes
+
 # Changelog
 /doc/source/ref-changelog.md @jafermarq @tanertopal @danieljanes
 
+# Translations
+/doc/locales @charlesbvll @tanertopal @danieljanes
+
 # GitHub Actions and Workflows
 /.github/workflows @Robert-Steiner @tanertopal @danieljanes
 /.github/actions @Robert-Steiner @tanertopal @danieljanes

.github/actions/bootstrap/action.yml

@@ -6,17 +6,17 @@ inputs:
     default: 3.8
   pip-version:
     description: "Version of pip to be installed using pip"
-    default: 23.3.1
+    default: 24.0.0
   setuptools-version:
     description: "Version of setuptools to be installed using pip"
-    default: 68.2.2
+    default: 69.5.1
   poetry-version:
     description: "Version of poetry to be installed using pip"
     default: 1.7.1
   poetry-skip:
     description: "Option to skip the poetry installation"
     required: false
-    default: 'false'
+    default: "false"
 outputs:
   python-version:
     description: "Version range or exact version of Python or PyPy"

.github/workflows/_docker-build.yml

@@ -60,8 +60,24 @@ jobs:
           hash = hashlib.sha256('''${{ inputs.namespace-repository }}
           ${{ inputs.file-dir }}
           ${{ inputs.build-args }}'''.encode())
+          # Adds two spaces to the line breaks to ensure proper indentation
+          # when passing the multi-line string to the wretry.action.
+          # Without it, the multi-line string is passed like this:
+          #
+          # build-args: |
+          #   ARG1=
+          # ARG2=
+          # ARG3=
+          #
+          # This causes the Docker action to interpret ARG2 and ARG3 as keys instead
+          # of values ​​of the multi-line string.
+          build_args = '''${{ inputs.build-args }}'''.replace("\n", "\n  ")
+
           with open(os.environ['GITHUB_OUTPUT'], 'a') as fh:
               print(f"id={hash.hexdigest()}", file=fh)
+              print("build-args<<EOF", file=fh)
+              print(build_args, file=fh)
+              print("EOF", file=fh)
 
       - name: Set up QEMU
         if: matrix.platform.qemu != ''
@@ -79,24 +95,30 @@ jobs:
         uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0
 
       - name: Login to Docker Hub
-        uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0
+        uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0
         with:
           username: ${{ secrets.dockerhub-user }}
           password: ${{ secrets.dockerhub-token }}
 
       - name: Build and push
+        uses: Wandalen/wretry.action@6feedb7dedadeb826de0f45ff482b53b379a7844 # v3.5.0
         id: build
-        uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0
         with:
-          platforms: ${{ matrix.platform.docker }}
-          context: "{{defaultContext}}:${{ inputs.file-dir }}"
-          build-args: ${{ inputs.build-args }}
-          outputs: type=image,name=${{ inputs.namespace-repository }},push-by-digest=true,name-canonical=true,push=true
+          action: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0
+          attempt_limit: 60 # 60 attempts * (9 secs delay + 1 sec retry) = ~10 mins
+          attempt_delay: 9000 # 9 secs
+          with: |
+            pull: true
+            platforms: ${{ matrix.platform.docker }}
+            context: "{{defaultContext}}:${{ inputs.file-dir }}"
+            outputs: type=image,name=${{ inputs.namespace-repository }},push-by-digest=true,name-canonical=true,push=true
+            build-args: |
+              ${{ steps.build-id.outputs.build-args }}
 
       - name: Export digest
         run: |
           mkdir -p /tmp/digests
-          digest="${{ steps.build.outputs.digest }}"
+          digest="${{ fromJSON(steps.build.outputs.outputs).digest }}"
           touch "/tmp/digests/${digest#sha256:}"
 
       - name: Upload digest
@@ -133,7 +155,7 @@ jobs:
         uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v3.3.0
 
       - name: Login to Docker Hub
-        uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0
+        uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0
         with:
           username: ${{ secrets.dockerhub-user }}
           password: ${{ secrets.dockerhub-token }}

.github/workflows/cache-cleanup.yml

@@ -0,0 +1,63 @@
+name: Cache cleanup
+
+on:
+  workflow_dispatch
+
+jobs:
+  e2e-cleanup:
+    runs-on: ubuntu-22.04
+    timeout-minutes: 10
+    # Using approach described here:
+    # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs
+    strategy:
+      matrix:
+        include:
+          - directory: bare
+          - directory: bare-https
+          - directory: bare-client-auth
+          - directory: jax
+          - directory: pytorch
+          - directory: tensorflow
+          - directory: tabnet
+          - directory: opacus
+          - directory: pytorch-lightning
+          - directory: scikit-learn
+          - directory: fastai
+          - directory: pandas
+
+    name: Framework / ${{ matrix.directory }}
+
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.8
+
+      - name: Cleanup caches by directories
+        # Only keep caches that match the latest keys for each directory
+        run: |
+          gh extension install actions/gh-actions-cache
+          
+          REPO=${{ github.repository }}
+          LATEST_KEY=pythonloc-${{ matrix.directory }}-${{ env.pythonLocation }}-${{ hashFiles(format('./e2e/{0}/pyproject.toml', matrix.directory)) }}
+
+          echo "Fetching list of cache keys"
+          cacheKeys=$(gh actions-cache list -R $REPO | grep "${{ matrix.directory }}" | cut -f 1 )
+
+          ## Setting this to not fail the workflow while deleting cache keys. 
+          set +e
+          echo "Deleting caches..."
+          for cacheKey in $cacheKeys
+          do
+            if [ "$cacheKey" != "$LATEST_KEY" ]; then
+              echo -e "\tOld key found -> $cacheKey"
+              gh actions-cache delete "$cacheKey" --confirm
+            fi
+          done
+
+          echo "Done"
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/datasets.yml

@@ -16,6 +16,9 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/main' && github.run_id || github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
 
+env:
+  FLWR_TELEMETRY_ENABLED: 0
+
 defaults:
   run:
     working-directory: datasets

.github/workflows/docker-images.yml

@@ -0,0 +1,75 @@
+name: Build docker images
+
+on:
+  workflow_dispatch:
+    inputs:
+      flwr-version:
+        description: "Version of Flower."
+        required: true
+        type: string
+
+permissions:
+  contents: read
+
+jobs:
+  parameters:
+    name: Collect build parameters
+    runs-on: ubuntu-22.04
+    timeout-minutes: 10
+    outputs:
+      pip-version: ${{ steps.versions.outputs.pip-version }}
+      setuptools-version: ${{ steps.versions.outputs.setuptools-version }}
+      matrix: ${{ steps.matrix.outputs.matrix }}
+    steps:
+      - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+      - uses: ./.github/actions/bootstrap
+        id: bootstrap
+
+      - id: versions
+        run: |
+          echo "pip-version=${{ steps.bootstrap.outputs.pip-version }}" >> "$GITHUB_OUTPUT"
+          echo "setuptools-version=${{ steps.bootstrap.outputs.setuptools-version }}" >> "$GITHUB_OUTPUT"
+
+      - id: matrix
+        run: |
+           python dev/build-docker-image-matrix.py --flwr-version ${{ github.event.inputs.flwr-version }} > matrix.json
+           echo "matrix=$(cat matrix.json)" >> $GITHUB_OUTPUT
+
+  build-base-images:
+    name: Build base images
+    uses: ./.github/workflows/_docker-build.yml
+    needs: parameters
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.parameters.outputs.matrix).base }}
+    with:
+      namespace-repository: ${{ matrix.images.namespace_repository }}
+      file-dir: ${{ matrix.images.file_dir }}
+      build-args: |
+        PYTHON_VERSION=${{ matrix.images.python_version }}
+        PIP_VERSION=${{ needs.parameters.outputs.pip-version }}
+        SETUPTOOLS_VERSION=${{ needs.parameters.outputs.setuptools-version }}
+        DISTRO=${{ matrix.images.distro.name }}
+        DISTRO_VERSION=${{ matrix.images.distro.version }}
+        FLWR_VERSION=${{ matrix.images.flwr_version }}
+      tags: ${{ matrix.images.tag }}
+    secrets:
+      dockerhub-user: ${{ secrets.DOCKERHUB_USERNAME }}
+      dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }}
+
+  build-binary-images:
+    name: Build binary images
+    uses: ./.github/workflows/_docker-build.yml
+    needs: [parameters, build-base-images]
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.parameters.outputs.matrix).binary }}
+    with:
+      namespace-repository: ${{ matrix.images.namespace_repository }}
+      file-dir: ${{ matrix.images.file_dir }}
+      build-args: BASE_IMAGE=${{ matrix.images.base_image }}
+      tags: ${{ matrix.images.tags }}
+    secrets:
+      dockerhub-user: ${{ secrets.DOCKERHUB_USERNAME }}
+      dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }}