Merge branch 'main' into fds-add-partition-splitting

adap · Mar 6, 2024 · 5f13155 · 5f13155
2 parents 6052e2a + 6da8079
commit 5f13155
Show file tree

Hide file tree

Showing 562 changed files with 15,472 additions and 4,200 deletions.
diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
@@ -31,7 +31,7 @@ RUN apt-get install -y curl wget gnupg python3 python-is-python3 python3-pip git
 RUN python -m pip install \
     pip==23.3.1 \
     setuptools==68.2.2 \
-    poetry==1.5.1
+    poetry==1.7.1
 
 USER $USERNAME
 ENV PATH="/home/$USERNAME/.local/bin:${PATH}"
diff --git a/.github/ISSUE_TEMPLATE/baseline_request.yml b/.github/ISSUE_TEMPLATE/baseline_request.yml
@@ -38,18 +38,18 @@ body:
     attributes:
       label: For first time contributors
       value: |
-        - [ ]  Read the [`first contribution` doc](https://flower.dev/docs/first-time-contributors.html)
+        - [ ]  Read the [`first contribution` doc](https://flower.ai/docs/first-time-contributors.html)
         - [ ]  Complete the Flower tutorial
         - [ ]  Read the Flower Baselines docs to get an overview:
-            - [ ]  [How to use Flower Baselines](https://flower.dev/docs/baselines/how-to-use-baselines.html)
-            - [ ]  [How to contribute a Flower Baseline](https://flower.dev/docs/baselines/how-to-contribute-baselines.html)
+            - [ ]  [How to use Flower Baselines](https://flower.ai/docs/baselines/how-to-use-baselines.html)
+            - [ ]  [How to contribute a Flower Baseline](https://flower.ai/docs/baselines/how-to-contribute-baselines.html)
   - type: checkboxes
     attributes:
       label: Prepare - understand the scope
       options:
         - label: Read the paper linked above
         - label: Decide which experiments you'd like to reproduce. The more the better!
-        - label: Follow the steps outlined in [Add a new Flower Baseline](https://flower.dev/docs/baselines/how-to-contribute-baselines.html#add-a-new-flower-baseline).
+        - label: Follow the steps outlined in [Add a new Flower Baseline](https://flower.ai/docs/baselines/how-to-contribute-baselines.html#add-a-new-flower-baseline).
         - label: You can use as reference [other baselines](https://github.com/adap/flower/tree/main/baselines) that the community merged following those steps.
   - type: checkboxes
     attributes:

diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml
@@ -1,14 +1,14 @@
 blank_issues_enabled: false
 contact_links:
   - name: Slack Channel
-    url: https://flower.dev/join-slack
+    url: https://flower.ai/join-slack
     about: Connect with other Flower users and contributors and discuss with them or ask them questions.
   - name: Discussion
     url: https://github.com/adap/flower/discussions
-    about: Ask about new features or general questions. Please use the discussion area in most of the cases instead of the issues. 
+    about: Ask about new features or general questions. Please use the discussion area in most of the cases instead of the issues.
   - name: Flower Issues
     url: https://github.com/adap/flower/issues
     about: Contribute new features/enhancements, report bugs, or improve the documentation.
   - name: Flower Mail
-    url: https://flower.dev/
-    about: If your project needs professional support please contact the Flower team (hello@flower.dev).
+    url: https://flower.ai/
+    about: If your project needs professional support please contact the Flower team (hello@flower.ai).
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
@@ -36,10 +36,10 @@ Example: The variable `rnd` was renamed to `server_round` to improve readability
 
 - [ ] Implement proposed change
 - [ ] Write tests
-- [ ] Update [documentation](https://flower.dev/docs/writing-documentation.html)
+- [ ] Update [documentation](https://flower.ai/docs/writing-documentation.html)
 - [ ] Update the changelog entry below
 - [ ] Make CI checks pass
-- [ ] Ping maintainers on [Slack](https://flower.dev/join-slack/) (channel `#contributions`)
+- [ ] Ping maintainers on [Slack](https://flower.ai/join-slack/) (channel `#contributions`)
 
 <!--
 Inside the following 'Changelog entry' section, you should put the description of your changes that will be added to the changelog alongside your PR title.
@@ -68,7 +68,7 @@ Smaller PRs with good descriptions can be considered much more easily.
 
 If you have an urgent request or question, please use the Flower Slack:
 
-    https://flower.dev/join-slack/ (channel: #contributions)
+    https://flower.ai/join-slack/ (channel: #contributions)
 
 Thank you for contributing to Flower!
 -->
diff --git a/.github/actions/bootstrap/action.yml b/.github/actions/bootstrap/action.yml
@@ -12,7 +12,7 @@ inputs:
     default: 68.2.2
   poetry-version:
     description: "Version of poetry to be installed using pip"
-    default: 1.5.1
+    default: 1.7.1
 outputs:
   python-version:
     description: "Version range or exact version of Python or PyPy"

diff --git a/.github/workflows/baselines.yml b/.github/workflows/baselines.yml
@@ -1,14 +1,5 @@
 name: Baselines
-# The aim of this workflow is to test only the changed (or added) baseline.
-# Here is the rough idea of how it works (more details are presented later in the comments):
-# 1. Checks for the changes between the current branch and the main - in case of PR -
-#  or between the HEAD and HEAD~1 (main last commit and the previous one) - in case of
-#  a push to main.
-# 2. Fails the test if there are changes to more than one baseline. Passes the test
-#  (skips the rests) if there are no changes to any baselines. Follows the test if only
-#  one baseline is added or modified.
-# 3. Sets up the env specified for the baseline.
-# 4. Runs the tests.
+
 on:
   push:
     branches:
@@ -24,112 +15,75 @@ concurrency:
 env:
   FLWR_TELEMETRY_ENABLED: 0
 
-defaults:
-  run:
-    working-directory: baselines
-
 jobs:
-  test_baselines:
-    name: Test
+  changes:
     runs-on: ubuntu-22.04
+    permissions:
+      pull-requests: read
+    outputs:
+      baselines: ${{ steps.filter.outputs.changes }}
     steps:
       - uses: actions/checkout@v4
-        # The depth two of the checkout is needed in case of merging to the main
-        # because we compare the HEAD (current version) with HEAD~1 (version before
-        # the PR was merged)
-        with:
-          fetch-depth: 2
-      - name: Fetch main branch
-        run: |
-          # The main branch is needed in case of the PR to make a comparison (by 
-          # default the workflow takes as little information as possible - it does not
-          # have the history
-          if [ ${{ github.event_name }} == "pull_request" ]
-          then
-            git fetch origin main:main
-          fi
-      - name: Find changed/new baselines
-        id: find_changed_baselines_dirs
+
+      - shell: bash
         run: |
-          if [ ${{ github.event_name }} == "push" ]
-          then
-            # Push event triggered when merging to main
-            change_references="HEAD..HEAD~1"
-          else
-            # Pull request event triggered for any commit to a pull request
-            change_references="main..HEAD"
-          fi
-          dirs=$(git diff --dirstat=files,0 ${change_references} . | awk '{print $2}' | grep -E '^baselines/[^/]*/$' | \
-          grep -v \
-            -e '^baselines/dev' \
-            -e '^baselines/baseline_template' \
-            -e '^baselines/flwr_baselines' \
-            -e '^baselines/doc' \
-          | sed 's/^baselines\///')
-          # git diff --dirstat=files,0 ${change_references} . - checks the differences
-          #   and a file is counted as changed if more than 0 lines were changed
-          #   it returns the results in the format x.y% path/to/dir/
-          # awk '{print $2}' - takes only the directories (skips the percentages)
-          # grep -E '^baselines/[^/]*/$' - takes only the paths that start with 
-          #   baseline (and have at least one subdirectory)
-          # grep -v -e ... - excludes the `baseline_template`, `dev`, `flwr_baselines`
-          # sed 's/^baselines\///' - narrows down the path to baseline/<subdirectory>
-          echo "Detected changed directories: ${dirs}"
-          # Save changed dirs to output of this step
-          EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64)
-          echo "dirs<<EOF" >> "$GITHUB_OUTPUT"
-          for dir in $dirs
-          do
-            echo "$dir" >> "$GITHUB_OUTPUT"
-          done
-          echo "EOF" >> "$GITHUB_OUTPUT"
-      - name: Validate changed/new baselines
-        id: validate_changed_baselines_dirs
+          # create a list of all directories in baselines
+          {
+            echo 'FILTER_PATHS<<EOF'
+            FILTER=""
+            while read -d $'\0' BASELINES_PATH; do
+                DIR=$(basename $BASELINES_PATH)
+                FILTER+=$(echo "$DIR: ${BASELINES_PATH}/**\n")
+            done < <(find baselines -maxdepth 1 \
+                -name ".*" -prune -o \
+                -path "baselines/doc" -prune -o \
+                -path "baselines/dev" -prune -o \
+                -path "baselines/baseline_template" -prune -o \
+                -path "baselines/flwr_baselines" -prune -o \
+                -type d -print0)
+            FILTER=$(echo -e "$FILTER")
+            # remove first line
+            FILTER=${FILTER#*$'\n'}
+            echo "$FILTER"
+            echo EOF
+          } >> "$GITHUB_ENV"
+
+      - uses: dorny/paths-filter@v3
+        id: filter
+        with:
+          filters: ${{ env.FILTER_PATHS }}
+
+      - if: ${{ github.event.pull_request.head.repo.fork }}
         run: |
-          dirs="${{ steps.find_changed_baselines_dirs.outputs.dirs }}"
-          dirs_array=()
-          if [[ -n $dirs ]]; then
-            while IFS= read -r line; do
-              dirs_array+=("$line")
-            done <<< "$dirs"
-          fi
-          length=${#dirs_array[@]}
-          echo "The number of changed baselines is $length"
-          
-          if [ $length -gt 1 ]; then
-          echo "The changes should only apply to a single baseline"
-          exit 1
+          CHANGES=$(echo "${{ toJson(steps.filter.outputs.changes) }}" | jq '. | length')
+          if [ "$CHANGES" -gt 1 ]; then
+            echo "::error ::The changes should only apply to a single baseline."
+            exit 1
           fi
-          
-          if [ $length -eq 0 ]; then
-          echo "The baselines were not changed - skipping the remaining steps."
-          echo "baseline_changed=false" >> "$GITHUB_OUTPUT"
-          exit 0
-          fi
-          
-          echo "changed_dir=${dirs[0]}" >> "$GITHUB_OUTPUT"
-          echo "baseline_changed=true" >> "$GITHUB_OUTPUT"
+
+  test:
+    runs-on: ubuntu-22.04
+    needs: changes
+    if: ${{ needs.changes.outputs.baselines != '' && toJson(fromJson(needs.changes.outputs.baselines)) != '[]' }}
+    strategy:
+      matrix:
+        baseline: ${{ fromJSON(needs.changes.outputs.baselines) }}
+    steps:
+      - uses: actions/checkout@v4
+
       - name: Bootstrap
-        if: steps.validate_changed_baselines_dirs.outputs.baseline_changed == 'true'
         uses: ./.github/actions/bootstrap
         with:
           python-version: '3.10'
+
       - name: Install dependencies
-        if: steps.validate_changed_baselines_dirs.outputs.baseline_changed == 'true'
-        run: |
-          changed_dir="${{ steps.validate_changed_baselines_dirs.outputs.changed_dir }}"
-          cd "${changed_dir}"
-          python -m poetry install
-      - name: Test
-        if: steps.validate_changed_baselines_dirs.outputs.baseline_changed == 'true'
-        run: |
-          dir="${{ steps.validate_changed_baselines_dirs.outputs.changed_dir }}"
-          echo "Testing ${dir}"
-          ./dev/test-baseline.sh $dir
-      - name: Test Structure
-        if: steps.validate_changed_baselines_dirs.outputs.baseline_changed == 'true'
-        run: |
-          dir="${{ steps.validate_changed_baselines_dirs.outputs.changed_dir }}"
-          echo "Testing ${dir}"
-          ./dev/test-baseline-structure.sh $dir
+        working-directory: baselines/${{ matrix.baseline }}
+        run: python -m poetry install
+
+      - name: Testing ${{ matrix.baseline }}
+        working-directory: baselines
+        run: ./dev/test-baseline.sh ${{ matrix.baseline }}
 
+      - name: Test Structure of ${{ matrix.baseline }}
+        working-directory: baselines
+        run: ./dev/test-baseline-structure.sh ${{ matrix.baseline }}
diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml
@@ -19,13 +19,13 @@ jobs:
       uses: ./.github/actions/bootstrap
 
     - name: Cache restore SDK build
-      uses: actions/cache/restore@v3
+      uses: actions/cache/restore@v4
       with:
         path: build/
         key: ${{ runner.os }}-sdk-build
 
     - name: Cache restore example build
-      uses: actions/cache/restore@v3
+      uses: actions/cache/restore@v4
       with:
         path: examples/quickstart-cpp/build/
         key: ${{ runner.os }}-example-build
@@ -82,14 +82,14 @@ jobs:
         fi
 
     - name: Cache save SDK build
-      uses: actions/cache/save@v3
+      uses: actions/cache/save@v4
       if: github.ref_name == 'main'
       with:
         path: build/
         key: ${{ runner.os }}-sdk-build
 
     - name: Cache save example build
-      uses: actions/cache/save@v3
+      uses: actions/cache/save@v4
       if: github.ref_name == 'main'
       with:
         path: examples/quickstart-cpp/build/

diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -43,8 +43,9 @@ jobs:
           AWS_DEFAULT_REGION: ${{ secrets. AWS_DEFAULT_REGION }}
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets. AWS_SECRET_ACCESS_KEY }}
+          DOCS_BUCKET: flower.ai
         run: |
-          aws s3 sync --delete --exclude ".*" --exclude "v/*" --cache-control "no-cache" ./doc/build/html/ s3://flower.dev/docs/framework
-          aws s3 sync --delete --exclude ".*" --exclude "v/*" --cache-control "no-cache" ./baselines/doc/build/html/ s3://flower.dev/docs/baselines
-          aws s3 sync --delete --exclude ".*" --exclude "v/*" --cache-control "no-cache" ./examples/doc/build/html/ s3://flower.dev/docs/examples
-          aws s3 sync --delete --exclude ".*" --exclude "v/*" --cache-control "no-cache" ./datasets/doc/build/html/ s3://flower.dev/docs/datasets
+          aws s3 sync --delete --exclude ".*" --exclude "v/*" --cache-control "no-cache" ./doc/build/html/ s3://${{ env.DOCS_BUCKET }}/docs/framework
+          aws s3 sync --delete --exclude ".*" --exclude "v/*" --cache-control "no-cache" ./baselines/doc/build/html/ s3://${{ env.DOCS_BUCKET }}/docs/baselines
+          aws s3 sync --delete --exclude ".*" --exclude "v/*" --cache-control "no-cache" ./examples/doc/build/html/ s3://${{ env.DOCS_BUCKET }}/docs/examples
+          aws s3 sync --delete --exclude ".*" --exclude "v/*" --cache-control "no-cache" ./datasets/doc/build/html/ s3://${{ env.DOCS_BUCKET }}/docs/datasets
diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml
@@ -14,6 +14,7 @@ concurrency:
 
 env:
   FLWR_TELEMETRY_ENABLED: 0
+  ARTIFACT_BUCKET: artifact.flower.ai
 
 jobs:
   wheel:
@@ -43,7 +44,7 @@ jobs:
           echo "SHORT_SHA=$sha_short" >> "$GITHUB_OUTPUT"
           [ -z "${{ github.head_ref }}" ] && dir="${{ github.ref_name }}" || dir="pr/${{ github.head_ref }}"
           echo "DIR=$dir" >> "$GITHUB_OUTPUT"
-          aws s3 cp --content-disposition "attachment" --cache-control "no-cache" ./ s3://artifact.flower.dev/py/$dir/$sha_short --recursive
+          aws s3 cp --content-disposition "attachment" --cache-control "no-cache" ./ s3://${{ env.ARTIFACT_BUCKET }}/py/$dir/$sha_short --recursive
     outputs:
       whl_path: ${{ steps.upload.outputs.WHL_PATH }}
       short_sha: ${{ steps.upload.outputs.SHORT_SHA }}
@@ -123,7 +124,7 @@ jobs:
       - name: Install Flower wheel from artifact store
         if: ${{ github.repository == 'adap/flower' && !github.event.pull_request.head.repo.fork && github.actor != 'dependabot[bot]' }}
         run: |
-          python -m pip install https://artifact.flower.dev/py/${{ needs.wheel.outputs.dir }}/${{ needs.wheel.outputs.short_sha }}/${{ needs.wheel.outputs.whl_path }}
+          python -m pip install https://${{ env.ARTIFACT_BUCKET }}/py/${{ needs.wheel.outputs.dir }}/${{ needs.wheel.outputs.short_sha }}/${{ needs.wheel.outputs.whl_path }}
       - name: Download dataset
         if: ${{ matrix.dataset }}
         run: python -c "${{ matrix.dataset }}"
@@ -164,9 +165,9 @@ jobs:
       - name: Install Flower wheel from artifact store
         if: ${{ github.repository == 'adap/flower' && !github.event.pull_request.head.repo.fork && github.actor != 'dependabot[bot]' }}
         run: |
-          python -m pip install https://artifact.flower.dev/py/${{ needs.wheel.outputs.dir }}/${{ needs.wheel.outputs.short_sha }}/${{ needs.wheel.outputs.whl_path }}
+          python -m pip install https://${{ env.ARTIFACT_BUCKET }}/py/${{ needs.wheel.outputs.dir }}/${{ needs.wheel.outputs.short_sha }}/${{ needs.wheel.outputs.whl_path }}
       - name: Cache Datasets
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: "~/.keras"
           key: keras-datasets