Skip to content

Commit

Permalink
Merge branch 'main' into add-xgboost-template
Browse files Browse the repository at this point in the history
  • Loading branch information
charlesbvll authored Sep 24, 2024
2 parents c4a0a04 + 983217f commit c99464a
Show file tree
Hide file tree
Showing 510 changed files with 104,166 additions and 80,863 deletions.
8 changes: 8 additions & 0 deletions .editorconfig
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,14 @@ profile = black
indent_style = space
indent_size = 2

[*.md]
indent_style = space
indent_size = 2

[*.yml]
indent_style = space
indent_size = 2

[*.toml]
indent_style = space
indent_size = 4
11 changes: 10 additions & 1 deletion .github/CODEOWNERS
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,10 @@
README.md @jafermarq @tanertopal @danieljanes

# Flower Baselines
/baselines @jafermarq @tanertopal @danieljanes
/baselines @jafermarq @danieljanes

# Flower Benchmarks
/benchmarks @jafermarq @danieljanes

# Flower Datasets
/datasets @jafermarq @tanertopal @danieljanes
Expand All @@ -27,3 +30,9 @@ README.md @jafermarq @tanertopal @danieljanes
# GitHub Actions and Workflows
/.github/workflows @Robert-Steiner @tanertopal @danieljanes
/.github/actions @Robert-Steiner @tanertopal @danieljanes

# Docker-related files
/.devcontainer @Robert-Steiner @Moep90 @tanertopal @danieljanes
**/Dockerfile @Robert-Steiner @Moep90 @tanertopal @danieljanes
**/*.Dockerfile @Robert-Steiner @Moep90 @tanertopal @danieljanes
/src/docker @Robert-Steiner @Moep90 @tanertopal @danieljanes
2 changes: 1 addition & 1 deletion .github/actions/bootstrap/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ description: "Bootstrap Python environment (install and configure Python version
inputs:
python-version:
description: "Version range or exact version of Python or PyPy to use, using SemVer's version range syntax."
default: 3.8
default: 3.9
pip-version:
description: "Version of pip to be installed using pip"
default: 24.1.2
Expand Down
18 changes: 5 additions & 13 deletions .github/workflows/_docker-build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -36,18 +36,16 @@ permissions:
jobs:
build:
name: Build image
runs-on: ubuntu-22.04
runs-on: ${{ matrix.platform.runner-os }}
timeout-minutes: 180
outputs:
build-id: ${{ steps.build-id.outputs.id }}
strategy:
fail-fast: true
matrix:
platform: [
# build-push action and qemu use different platform names
# therefore we create a map
{ name: "amd64", qemu: "", docker: "linux/amd64" },
{ name: "arm64", qemu: "arm64", docker: "linux/arm64" },
{ name: "amd64", docker: "linux/amd64", runner-os: "ubuntu-22.04" },
{ name: "arm64", docker: "linux/arm64", runner-os: "ubuntu-4-core-arm64" },
]
steps:
- name: Create build id
Expand Down Expand Up @@ -79,12 +77,6 @@ jobs:
print(build_args, file=fh)
print("EOF", file=fh)
- name: Set up QEMU
if: matrix.platform.qemu != ''
uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v3.2.0
with:
platforms: ${{ matrix.platform.qemu }}

- name: Extract metadata (tags, labels) for Docker
id: meta
uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1
Expand All @@ -104,7 +96,7 @@ jobs:
uses: Wandalen/wretry.action@6feedb7dedadeb826de0f45ff482b53b379a7844 # v3.5.0
id: build
with:
action: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5.3.0
action: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85 # v6.7.0
attempt_limit: 60 # 60 attempts * (9 secs delay + 1 sec retry) = ~10 mins
attempt_delay: 9000 # 9 secs
with: |
Expand All @@ -122,7 +114,7 @@ jobs:
touch "/tmp/digests/${digest#sha256:}"
- name: Upload digest
uses: actions/upload-artifact@834a144ee995460fba8ed112a2fc961b36a5ec5a # v4.3.6
uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
with:
name: digests-${{ steps.build-id.outputs.id }}-${{ matrix.platform.name }}
path: /tmp/digests/*
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/cache-cleanup.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ jobs:
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: 3.8
python-version: 3.9

- name: Cleanup caches by directories
# Only keep caches that match the latest keys for each directory
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/datasets-e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ jobs:
- name: Bootstrap
uses: ./.github/actions/bootstrap
with:
python-version: 3.8
python-version: 3.9
- name: Install dependencies
run: python -m poetry install
- name: Run tests
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/datasets.yml
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ jobs:
# In case of a mismatch, the job has to download Python to install it.
# Note: Due to a bug in actions/setup-python, we have to put "3.10" in
# quotes as it will otherwise assume "3.1"
python: [3.8, 3.9, '3.10', '3.11']
python: ['3.9', '3.10', '3.11']

name: Python ${{ matrix.python }}

Expand Down
69 changes: 69 additions & 0 deletions .github/workflows/docker-build-main.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
name: Build Docker Images Main Branch

on:
push:
branches:
- 'main'

jobs:
parameters:
if: github.repository == 'adap/flower'
name: Collect docker build parameters
runs-on: ubuntu-22.04
timeout-minutes: 10
outputs:
pip-version: ${{ steps.versions.outputs.pip-version }}
setuptools-version: ${{ steps.versions.outputs.setuptools-version }}
flwr-version-ref: ${{ steps.versions.outputs.flwr-version-ref }}
steps:
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1

- uses: ./.github/actions/bootstrap
id: bootstrap

- id: versions
run: |
echo "pip-version=${{ steps.bootstrap.outputs.pip-version }}" >> "$GITHUB_OUTPUT"
echo "setuptools-version=${{ steps.bootstrap.outputs.setuptools-version }}" >> "$GITHUB_OUTPUT"
echo "flwr-version-ref=git+${{ github.server_url }}/${{ github.repository }}.git@${{ github.sha }}" >> "$GITHUB_OUTPUT"
build-docker-base-images:
name: Build base images
if: github.repository == 'adap/flower'
uses: ./.github/workflows/_docker-build.yml
needs: parameters
with:
namespace-repository: flwr/base
file-dir: src/docker/base/ubuntu
build-args: |
PIP_VERSION=${{ needs.parameters.outputs.pip-version }}
SETUPTOOLS_VERSION=${{ needs.parameters.outputs.setuptools-version }}
FLWR_VERSION_REF=${{ needs.parameters.outputs.flwr-version-ref }}
tags: unstable
secrets:
dockerhub-user: ${{ secrets.DOCKERHUB_USERNAME }}
dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }}

build-docker-binary-images:
name: Build binary images
if: github.repository == 'adap/flower'
uses: ./.github/workflows/_docker-build.yml
needs: build-docker-base-images
strategy:
fail-fast: false
matrix:
images: [
{ repository: "flwr/superlink", file_dir: "src/docker/superlink" },
{ repository: "flwr/supernode", file_dir: "src/docker/supernode" },
{ repository: "flwr/serverapp", file_dir: "src/docker/serverapp" },
{ repository: "flwr/superexec", file_dir: "src/docker/superexec" },
{ repository: "flwr/clientapp", file_dir: "src/docker/clientapp" }
]
with:
namespace-repository: ${{ matrix.images.repository }}
file-dir: ${{ matrix.images.file_dir }}
build-args: BASE_IMAGE=unstable
tags: unstable
secrets:
dockerhub-user: ${{ secrets.DOCKERHUB_USERNAME }}
dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }}
51 changes: 51 additions & 0 deletions .github/workflows/docker-readme.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
name: Update Docker READMEs

on:
push:
branches:
- 'main'
paths:
- 'src/docker/**/README.md'

jobs:
collect:
if: ${{ github.repository == 'adap/flower' }}
name: Collect Docker READMEs
runs-on: ubuntu-22.04
timeout-minutes: 10
outputs:
readme_files: ${{ steps.filter.outputs.readme_files }}
steps:
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1

- uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
id: filter
with:
list-files: "json"
filters: |
readme:
- 'src/docker/**/README.md'
update:
if: ${{ needs.collect.outputs.readme_files != '' && toJson(fromJson(needs.collect.outputs.readme_files)) != '[]' }}
name: Update Docker READMEs
runs-on: ubuntu-22.04
timeout-minutes: 10
needs: collect
strategy:
matrix:
readme_path: ${{ fromJSON(needs.collect.outputs.readme_files) }}

steps:
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1

- id: repository
run: echo "name=$(basename $(dirname ${{ matrix.readme_path }}))" >> "$GITHUB_OUTPUT"

- name: Docker Hub Description
uses: peter-evans/dockerhub-description@e98e4d1628a5f3be2be7c231e50981aee98723ae # v4.0.0
with:
repository: flwr/${{ steps.repository.outputs.name }}
readme-filepath: ${{ matrix.readme_path }}
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
6 changes: 2 additions & 4 deletions .github/workflows/e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ jobs:
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: 3.8
python-version: 3.9
- name: Install build tools
run: |
python -m pip install -U pip==23.3.1
Expand All @@ -146,8 +146,6 @@ jobs:
if: ${{ github.repository == 'adap/flower' && !github.event.pull_request.head.repo.fork && github.actor != 'dependabot[bot]' }}
run: |
python -m pip install https://${{ env.ARTIFACT_BUCKET }}/py/${{ needs.wheel.outputs.dir }}/${{ needs.wheel.outputs.short_sha }}/${{ needs.wheel.outputs.whl_path }}
- name: Install e2e components
run: pip install .
- name: Download dataset
if: ${{ matrix.dataset }}
run: python -c "${{ matrix.dataset }}"
Expand All @@ -172,7 +170,7 @@ jobs:
run: ./../test_superlink.sh bare sqlite
- name: Run driver test with client authentication
if: ${{ matrix.directory == 'e2e-bare-auth' }}
run: ./../test_superlink.sh bare client-auth
run: ./../test_superlink.sh "${{ matrix.directory }}" client-auth
- name: Run reconnection test with SQLite database
if: ${{ matrix.directory == 'e2e-bare' }}
run: ./../test_reconnection.sh sqlite
Expand Down
7 changes: 5 additions & 2 deletions .github/workflows/framework-release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ jobs:
if: ${{ github.repository == 'adap/flower' }}
name: Publish release
runs-on: ubuntu-22.04
outputs:
flwr-version: ${{ steps.publish.outputs.flwr-version }}
steps:
- name: Checkout code
uses: actions/checkout@v4
Expand All @@ -26,10 +28,12 @@ jobs:
uses: ./.github/actions/bootstrap

- name: Get artifacts and publish
id: publish
env:
GITHUB_REF: ${{ github.ref }}
run: |
TAG_NAME=$(echo "${GITHUB_REF_NAME}" | cut -c2-)
echo "flwr-version=$TAG_NAME" >> "$GITHUB_OUTPUT"
wheel_name="flwr-${TAG_NAME}-py3-none-any.whl"
tar_name="flwr-${TAG_NAME}.tar.gz"
Expand Down Expand Up @@ -67,8 +71,7 @@ jobs:
- id: matrix
run: |
FLWR_VERSION=$(poetry version -s)
python dev/build-docker-image-matrix.py --flwr-version "${FLWR_VERSION}" > matrix.json
python dev/build-docker-image-matrix.py --flwr-version "${{ needs.publish.outputs.flwr-version }}" > matrix.json
echo "matrix=$(cat matrix.json)" >> $GITHUB_OUTPUT
build-base-images:
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/framework.yml
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ jobs:
# In case of a mismatch, the job has to download Python to install it.
# Note: Due to a bug in actions/setup-python, we have to put "3.10" in
# quotes as it will otherwise assume "3.1"
python: [3.8, 3.9, '3.10', '3.11']
python: ['3.9', '3.10', '3.11']

name: Python ${{ matrix.python }}

Expand Down
79 changes: 79 additions & 0 deletions .github/workflows/update_translations.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
name: Translations

on:
schedule:
- cron: '0 0 * * *' # Runs every day at midnight
workflow_dispatch: # Allows to manually trigger the workflow

jobs:
update-and-pr:
runs-on: ubuntu-22.04
permissions:
contents: write
pull-requests: write
env:
branch-name: auto-update-trans-text
name: Update text
steps:
- uses: actions/checkout@v4

- name: Bootstrap
uses: ./.github/actions/bootstrap
with:
python-version: '3.10'

- name: Install dependencies
run: |
python -m poetry install
pip install sphinx==7.3.7
- name: Install pandoc
uses: nikeee/setup-pandoc@v1

- name: Update text and translations for all locales
run: |
cd doc
make update-text
for langDir in locales/*; do
if [ -d "$langDir" ]; then
lang=$(basename $langDir)
echo "Updating language $lang"
make update-lang lang=$lang
fi
done
- name: Commit changes
run: |
git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
git config --local user.name "github-actions[bot]"
git add doc/locales
git commit -m "Update text and language files"
continue-on-error: true

- name: Calculate diff # Even without doc changes the update-lang command will generate 228 additions and 60 deletions, so we only want to open a PR when there is more
id: calculate_diff
run: |
additions=$(git diff --numstat HEAD^1 | awk '{s+=$1} END {print s}')
deletions=$(git diff --numstat HEAD^1 | awk '{s+=$2} END {print s}')
echo "Additions: $additions"
echo "Deletions: $deletions"
echo "additions=$additions" >> $GITHUB_OUTPUT
echo "deletions=$deletions" >> $GITHUB_OUTPUT
- name: Push changes
if: steps.calculate_diff.outputs.additions > 228 && steps.calculate_diff.outputs.deletions > 60
uses: ad-m/github-push-action@master
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
branch: '${{ env.branch-name }}'

- name: Create Pull Request
if: steps.calculate_diff.outputs.additions > 228 && steps.calculate_diff.outputs.deletions > 60
uses: peter-evans/create-pull-request@v6
with:
token: ${{ secrets.GITHUB_TOKEN }}
branch: '${{ env.branch-name }}'
delete-branch: true
title: 'docs(framework:skip) Update source texts for translations (automated)'
body: 'This PR is auto-generated to update text and language files.'
draft: false
Loading

0 comments on commit c99464a

Please sign in to comment.