Merge branch 'flanders' into flanders_tests

.devcontainer/Dockerfile

@@ -31,7 +31,7 @@ RUN apt-get install -y curl wget gnupg python3 python-is-python3 python3-pip git
 RUN python -m pip install \
     pip==23.3.1 \
     setuptools==68.2.2 \
-    poetry==1.5.1
+    poetry==1.7.1
 
 USER $USERNAME
 ENV PATH="/home/$USERNAME/.local/bin:${PATH}"

.github/CODEOWNERS

@@ -3,5 +3,17 @@
 # Default code owners
 * @danieljanes @tanertopal
 
+# README.md
+README.md @jafermarq @tanertopal @danieljanes
+
 # Flower Baselines
 /baselines @jafermarq @tanertopal @danieljanes
+
+# Flower Datasets
+/datasets @jafermarq @tanertopal @danieljanes
+
+# Flower Examples
+/examples @jafermarq @tanertopal @danieljanes
+
+# Changelog
+/doc/source/ref-changelog.md @jafermarq @tanertopal @danieljanes

.github/ISSUE_TEMPLATE/baseline_request.yml

@@ -38,18 +38,18 @@ body:
     attributes:
       label: For first time contributors
       value: |
-        - [ ]  Read the [`first contribution` doc](https://flower.dev/docs/first-time-contributors.html)
+        - [ ]  Read the [`first contribution` doc](https://flower.ai/docs/first-time-contributors.html)
         - [ ]  Complete the Flower tutorial
         - [ ]  Read the Flower Baselines docs to get an overview:
-            - [ ]  [How to use Flower Baselines](https://flower.dev/docs/baselines/how-to-use-baselines.html)
-            - [ ]  [How to contribute a Flower Baseline](https://flower.dev/docs/baselines/how-to-contribute-baselines.html)
+            - [ ]  [How to use Flower Baselines](https://flower.ai/docs/baselines/how-to-use-baselines.html)
+            - [ ]  [How to contribute a Flower Baseline](https://flower.ai/docs/baselines/how-to-contribute-baselines.html)
   - type: checkboxes
     attributes:
       label: Prepare - understand the scope
       options:
         - label: Read the paper linked above
         - label: Decide which experiments you'd like to reproduce. The more the better!
-        - label: Follow the steps outlined in [Add a new Flower Baseline](https://flower.dev/docs/baselines/how-to-contribute-baselines.html#add-a-new-flower-baseline).
+        - label: Follow the steps outlined in [Add a new Flower Baseline](https://flower.ai/docs/baselines/how-to-contribute-baselines.html#add-a-new-flower-baseline).
         - label: You can use as reference [other baselines](https://github.com/adap/flower/tree/main/baselines) that the community merged following those steps.
   - type: checkboxes
     attributes:

.github/ISSUE_TEMPLATE/config.yml

@@ -1,14 +1,14 @@
 blank_issues_enabled: false
 contact_links:
   - name: Slack Channel
-    url: https://flower.dev/join-slack
+    url: https://flower.ai/join-slack
     about: Connect with other Flower users and contributors and discuss with them or ask them questions.
   - name: Discussion
     url: https://github.com/adap/flower/discussions
-    about: Ask about new features or general questions. Please use the discussion area in most of the cases instead of the issues. 
+    about: Ask about new features or general questions. Please use the discussion area in most of the cases instead of the issues.
   - name: Flower Issues
     url: https://github.com/adap/flower/issues
     about: Contribute new features/enhancements, report bugs, or improve the documentation.
   - name: Flower Mail
-    url: https://flower.dev/
-    about: If your project needs professional support please contact the Flower team (hello@flower.dev).
+    url: https://flower.ai/
+    about: If your project needs professional support please contact the Flower team (hello@flower.ai).

.github/PULL_REQUEST_TEMPLATE.md

@@ -36,10 +36,29 @@ Example: The variable `rnd` was renamed to `server_round` to improve readability
 
 - [ ] Implement proposed change
 - [ ] Write tests
-- [ ] Update [documentation](https://flower.dev/docs/writing-documentation.html)
-- [ ] Update [changelog](https://github.com/adap/flower/blob/main/doc/source/changelog.rst)
+- [ ] Update [documentation](https://flower.ai/docs/writing-documentation.html)
+- [ ] Update the changelog entry below
 - [ ] Make CI checks pass
-- [ ] Ping maintainers on [Slack](https://flower.dev/join-slack/) (channel `#contributions`)
+- [ ] Ping maintainers on [Slack](https://flower.ai/join-slack/) (channel `#contributions`)
+
+<!--
+Inside the following 'Changelog entry' section, you should put the description of your changes that will be added to the changelog alongside your PR title.
+
+If the section is completely empty (without any token) or non-existant, the changelog will just contain the title of the PR for the changelog entry, without any description. If the section contains some text other than tokens, it will use it to add a description to the change. If the section contains one of the following tokens it will ignore any other text and put the PR under the corresponding section of the changelog:
+
+<general> is for classifying a PR as a general improvement.
+<skip> is to not add the PR to the changelog
+<baselines> is to add a general baselines change to the PR
+<examples> is to add a general examples change to the PR
+<sdk> is to add a general sdk change to the PR
+<simulations> is to add a general simulations change to the PR
+
+Note that only one token should be used.
+-->
+
+### Changelog entry
+
+
 
 ### Any other comments?
 
@@ -49,7 +68,7 @@ Smaller PRs with good descriptions can be considered much more easily.
 
 If you have an urgent request or question, please use the Flower Slack:
 
-    https://flower.dev/join-slack/ (channel: #contributions)
+    https://flower.ai/join-slack/ (channel: #contributions)
 
 Thank you for contributing to Flower!
 -->

.github/actions/bootstrap/action.yml

@@ -12,7 +12,7 @@ inputs:
     default: 68.2.2
   poetry-version:
     description: "Version of poetry to be installed using pip"
-    default: 1.5.1
+    default: 1.7.1
 outputs:
   python-version:
     description: "Version range or exact version of Python or PyPy"
@@ -30,7 +30,7 @@ runs:
   using: "composite"
   steps:
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ inputs.python-version }}
     - name: Install build tools

.github/workflows/_docker-build.yml

@@ -1,4 +1,4 @@
-name: Reusable docker server image build workflow
+name: Reusable docker image build workflow
 
 on:
   workflow_call:
@@ -35,19 +35,32 @@ permissions:
 # based on https://docs.docker.com/build/ci/github-actions/multi-platform/#distribute-build-across-multiple-runners
 jobs:
   build:
-    name: Build server image
+    name: Build image
     runs-on: ubuntu-22.04
     timeout-minutes: 60
+    outputs:
+      build-id: ${{ steps.build-id.outputs.id }}
     strategy:
       fail-fast: true
       matrix:
         platform: [
             # build-push action and qemu use different platform names
             # therefore we create a map
-            { qemu: "", docker: "linux/amd64" },
-            { qemu: "arm64", docker: "linux/arm64" },
+            { name: "amd64", qemu: "", docker: "linux/amd64" },
+            { name: "arm64", qemu: "arm64", docker: "linux/arm64" },
           ]
     steps:
+      - name: Create build id
+        id: build-id
+        shell: python
+        run: |
+          import hashlib
+          import os
+
+          hash = hashlib.sha256('''${{ inputs.build-args }}'''.encode())
+          with open(os.environ['GITHUB_OUTPUT'], 'a') as fh:
+              print(f"id={hash.hexdigest()}", file=fh)
+
       - name: Set up QEMU
         if: matrix.platform.qemu != ''
         uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0
@@ -85,9 +98,9 @@ jobs:
           touch "/tmp/digests/${digest#sha256:}"
 
       - name: Upload digest
-        uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3
+        uses: actions/upload-artifact@1eb3cb2b3e0f29609092a73eb033bb759a334595 # v4.1.0
         with:
-          name: digests
+          name: digests-${{ steps.build-id.outputs.id }}-${{ matrix.platform.name }}
           path: /tmp/digests/*
           if-no-files-found: error
           retention-days: 1
@@ -101,10 +114,11 @@ jobs:
       metadata: ${{ steps.meta.outputs.json }}
     steps:
       - name: Download digests
-        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
+        uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
         with:
-          name: digests
+          pattern: digests-${{ needs.build.outputs.build-id }}-*
           path: /tmp/digests
+          merge-multiple: true
 
       - name: Docker meta
         id: meta

.github/workflows/baselines.yml

@@ -1,14 +1,5 @@
 name: Baselines
-# The aim of this workflow is to test only the changed (or added) baseline.
-# Here is the rough idea of how it works (more details are presented later in the comments):
-# 1. Checks for the changes between the current branch and the main - in case of PR -
-#  or between the HEAD and HEAD~1 (main last commit and the previous one) - in case of
-#  a push to main.
-# 2. Fails the test if there are changes to more than one baseline. Passes the test
-#  (skips the rests) if there are no changes to any baselines. Follows the test if only
-#  one baseline is added or modified.
-# 3. Sets up the env specified for the baseline.
-# 4. Runs the tests.
+
 on:
   push:
     branches:
@@ -24,112 +15,75 @@ concurrency:
 env:
   FLWR_TELEMETRY_ENABLED: 0
 
-defaults:
-  run:
-    working-directory: baselines
-
 jobs:
-  test_baselines:
-    name: Test
+  changes:
     runs-on: ubuntu-22.04
+    permissions:
+      pull-requests: read
+    outputs:
+      baselines: ${{ steps.filter.outputs.changes }}
     steps:
       - uses: actions/checkout@v4
-        # The depth two of the checkout is needed in case of merging to the main
-        # because we compare the HEAD (current version) with HEAD~1 (version before
-        # the PR was merged)
-        with:
-          fetch-depth: 2
-      - name: Fetch main branch
-        run: |
-          # The main branch is needed in case of the PR to make a comparison (by 
-          # default the workflow takes as little information as possible - it does not
-          # have the history
-          if [ ${{ github.event_name }} == "pull_request" ]
-          then
-            git fetch origin main:main
-          fi
-      - name: Find changed/new baselines
-        id: find_changed_baselines_dirs
+
+      - shell: bash
         run: |
-          if [ ${{ github.event_name }} == "push" ]
-          then
-            # Push event triggered when merging to main
-            change_references="HEAD..HEAD~1"
-          else
-            # Pull request event triggered for any commit to a pull request
-            change_references="main..HEAD"
-          fi
-          dirs=$(git diff --dirstat=files,0 ${change_references} . | awk '{print $2}' | grep -E '^baselines/[^/]*/$' | \
-          grep -v \
-            -e '^baselines/dev' \
-            -e '^baselines/baseline_template' \
-            -e '^baselines/flwr_baselines' \
-            -e '^baselines/doc' \
-          | sed 's/^baselines\///')
-          # git diff --dirstat=files,0 ${change_references} . - checks the differences
-          #   and a file is counted as changed if more than 0 lines were changed
-          #   it returns the results in the format x.y% path/to/dir/
-          # awk '{print $2}' - takes only the directories (skips the percentages)
-          # grep -E '^baselines/[^/]*/$' - takes only the paths that start with 
-          #   baseline (and have at least one subdirectory)
-          # grep -v -e ... - excludes the `baseline_template`, `dev`, `flwr_baselines`
-          # sed 's/^baselines\///' - narrows down the path to baseline/<subdirectory>
-          echo "Detected changed directories: ${dirs}"
-          # Save changed dirs to output of this step
-          EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64)
-          echo "dirs<<EOF" >> "$GITHUB_OUTPUT"
-          for dir in $dirs
-          do
-            echo "$dir" >> "$GITHUB_OUTPUT"
-          done
-          echo "EOF" >> "$GITHUB_OUTPUT"
-      - name: Validate changed/new baselines
-        id: validate_changed_baselines_dirs
+          # create a list of all directories in baselines
+          {
+            echo 'FILTER_PATHS<<EOF'
+            FILTER=""
+            while read -d $'\0' BASELINES_PATH; do
+                DIR=$(basename $BASELINES_PATH)
+                FILTER+=$(echo "$DIR: ${BASELINES_PATH}/**\n")
+            done < <(find baselines -maxdepth 1 \
+                -name ".*" -prune -o \
+                -path "baselines/doc" -prune -o \
+                -path "baselines/dev" -prune -o \
+                -path "baselines/baseline_template" -prune -o \
+                -path "baselines/flwr_baselines" -prune -o \
+                -type d -print0)
+            FILTER=$(echo -e "$FILTER")
+            # remove first line
+            FILTER=${FILTER#*$'\n'}
+            echo "$FILTER"
+            echo EOF
+          } >> "$GITHUB_ENV"
+
+      - uses: dorny/paths-filter@v3
+        id: filter
+        with:
+          filters: ${{ env.FILTER_PATHS }}
+
+      - if: ${{ github.event.pull_request.head.repo.fork }}
         run: |
-          dirs="${{ steps.find_changed_baselines_dirs.outputs.dirs }}"
-          dirs_array=()
-          if [[ -n $dirs ]]; then
-            while IFS= read -r line; do
-              dirs_array+=("$line")
-            done <<< "$dirs"
-          fi
-          length=${#dirs_array[@]}
-          echo "The number of changed baselines is $length"
-          
-          if [ $length -gt 1 ]; then
-          echo "The changes should only apply to a single baseline"
-          exit 1
+          CHANGES=$(echo "${{ toJson(steps.filter.outputs.changes) }}" | jq '. | length')
+          if [ "$CHANGES" -gt 1 ]; then
+            echo "::error ::The changes should only apply to a single baseline."
+            exit 1
           fi
-          
-          if [ $length -eq 0 ]; then
-          echo "The baselines were not changed - skipping the remaining steps."
-          echo "baseline_changed=false" >> "$GITHUB_OUTPUT"
-          exit 0
-          fi
-          
-          echo "changed_dir=${dirs[0]}" >> "$GITHUB_OUTPUT"
-          echo "baseline_changed=true" >> "$GITHUB_OUTPUT"
+
+  test:
+    runs-on: ubuntu-22.04
+    needs: changes
+    if: ${{ needs.changes.outputs.baselines != '' && toJson(fromJson(needs.changes.outputs.baselines)) != '[]' }}
+    strategy:
+      matrix:
+        baseline: ${{ fromJSON(needs.changes.outputs.baselines) }}
+    steps:
+      - uses: actions/checkout@v4
+
       - name: Bootstrap
-        if: steps.validate_changed_baselines_dirs.outputs.baseline_changed == 'true'
         uses: ./.github/actions/bootstrap
         with:
           python-version: '3.10'
+
       - name: Install dependencies
-        if: steps.validate_changed_baselines_dirs.outputs.baseline_changed == 'true'
-        run: |
-          changed_dir="${{ steps.validate_changed_baselines_dirs.outputs.changed_dir }}"
-          cd "${changed_dir}"
-          python -m poetry install
-      - name: Test
-        if: steps.validate_changed_baselines_dirs.outputs.baseline_changed == 'true'
-        run: |
-          dir="${{ steps.validate_changed_baselines_dirs.outputs.changed_dir }}"
-          echo "Testing ${dir}"
-          ./dev/test-baseline.sh $dir
-      - name: Test Structure
-        if: steps.validate_changed_baselines_dirs.outputs.baseline_changed == 'true'
-        run: |
-          dir="${{ steps.validate_changed_baselines_dirs.outputs.changed_dir }}"
-          echo "Testing ${dir}"
-          ./dev/test-baseline-structure.sh $dir
+        working-directory: baselines/${{ matrix.baseline }}
+        run: python -m poetry install
+
+      - name: Testing ${{ matrix.baseline }}
+        working-directory: baselines
+        run: ./dev/test-baseline.sh ${{ matrix.baseline }}
 
+      - name: Test Structure of ${{ matrix.baseline }}
+        working-directory: baselines
+        run: ./dev/test-baseline-structure.sh ${{ matrix.baseline }}