From f67d25a45e9e28cc96c0f67179516408e06a735e Mon Sep 17 00:00:00 2001 From: Robert Steiner Date: Tue, 12 Nov 2024 14:32:42 +0100 Subject: [PATCH] refactor ci Signed-off-by: Robert Steiner --- .github/workflows/docker-build-main.yml | 20 +++++++++-------- .github/workflows/framework-release.yml | 8 +++---- .github/workflows/release-nightly.yml | 29 +++++++++++-------------- dev/build-docker-image-matrix.py | 20 +++++++---------- 4 files changed, 36 insertions(+), 41 deletions(-) diff --git a/.github/workflows/docker-build-main.yml b/.github/workflows/docker-build-main.yml index 550568372e37..38a9cd56942b 100644 --- a/.github/workflows/docker-build-main.yml +++ b/.github/workflows/docker-build-main.yml @@ -14,7 +14,6 @@ jobs: outputs: pip-version: ${{ steps.versions.outputs.pip-version }} setuptools-version: ${{ steps.versions.outputs.setuptools-version }} - flwr-version-ref: ${{ steps.versions.outputs.flwr-version-ref }} matrix: ${{ steps.versions.outputs.matrix }} steps: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 @@ -26,8 +25,8 @@ jobs: run: | echo "pip-version=${{ steps.bootstrap.outputs.pip-version }}" >> "$GITHUB_OUTPUT" echo "setuptools-version=${{ steps.bootstrap.outputs.setuptools-version }}" >> "$GITHUB_OUTPUT" - echo "flwr-version-ref=git+${{ github.server_url }}/${{ github.repository }}.git@${{ github.sha }}" >> "$GITHUB_OUTPUT" - python dev/build-docker-image-matrix.py --flwr-version unstable --simple > matrix.json + FLWR_VERSION_REF="git+${{ github.server_url }}/${{ github.repository }}.git@${{ github.sha }}" + python dev/build-docker-image-matrix.py --flwr-version "${FLWR_VERSION_REF}" --matrix unstable > matrix.json echo "matrix=$(cat matrix.json)" >> $GITHUB_OUTPUT build-docker-base-images: @@ -35,14 +34,17 @@ jobs: if: github.repository == 'adap/flower' uses: ./.github/workflows/_docker-build.yml needs: parameters + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.parameters.outputs.matrix).base }} with: - namespace-repository: ${{ fromJson(needs.parameters.outputs.matrix).base.images[0].namespace_repository }} - file-dir: ${{ fromJson(needs.parameters.outputs.matrix).base.images[0].file_dir }} + namespace-repository: ${{ matrix.images.namespace_repository }} + file-dir: ${{ matrix.images.file_dir }} build-args: | PIP_VERSION=${{ needs.parameters.outputs.pip-version }} SETUPTOOLS_VERSION=${{ needs.parameters.outputs.setuptools-version }} - FLWR_VERSION_REF=${{ needs.parameters.outputs.flwr-version-ref }} - tags: unstable + ${{ matrix.images.build_args_encoded }} + tags: ${{ matrix.images.tags_encoded }} secrets: dockerhub-user: ${{ secrets.DOCKERHUB_USERNAME }} dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }} @@ -58,8 +60,8 @@ jobs: with: namespace-repository: ${{ matrix.images.namespace_repository }} file-dir: ${{ matrix.images.file_dir }} - build-args: BASE_IMAGE=unstable - tags: unstable + build-args: BASE_IMAGE=${{ matrix.images.base_image }} + tags: ${{ matrix.images.tags_encoded }} secrets: dockerhub-user: ${{ secrets.DOCKERHUB_USERNAME }} dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }} diff --git a/.github/workflows/framework-release.yml b/.github/workflows/framework-release.yml index 6e9225c4ad16..6af0c281882b 100644 --- a/.github/workflows/framework-release.yml +++ b/.github/workflows/framework-release.yml @@ -71,7 +71,7 @@ jobs: - id: matrix run: | - python dev/build-docker-image-matrix.py --flwr-version "${{ needs.publish.outputs.flwr-version }}" > matrix.json + python dev/build-docker-image-matrix.py --flwr-version "${{ needs.publish.outputs.flwr-version }}" --matrix stable > matrix.json echo "matrix=$(cat matrix.json)" >> $GITHUB_OUTPUT build-base-images: @@ -88,8 +88,8 @@ jobs: build-args: | PIP_VERSION=${{ needs.parameters.outputs.pip-version }} SETUPTOOLS_VERSION=${{ needs.parameters.outputs.setuptools-version }} - ${{ matrix.images.build_args }} - tags: ${{ matrix.images.tag }} + ${{ matrix.images.build_args_encoded }} + tags: ${{ matrix.images.tags_encoded }} secrets: dockerhub-user: ${{ secrets.DOCKERHUB_USERNAME }} dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }} @@ -106,7 +106,7 @@ jobs: namespace-repository: ${{ matrix.images.namespace_repository }} file-dir: ${{ matrix.images.file_dir }} build-args: BASE_IMAGE=${{ matrix.images.base_image }} - tags: ${{ matrix.images.tags }} + tags: ${{ matrix.images.tags_encoded }} secrets: dockerhub-user: ${{ secrets.DOCKERHUB_USERNAME }} dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }} diff --git a/.github/workflows/release-nightly.yml b/.github/workflows/release-nightly.yml index 504159615ddb..d1de7bed531e 100644 --- a/.github/workflows/release-nightly.yml +++ b/.github/workflows/release-nightly.yml @@ -13,8 +13,6 @@ jobs: name: Relase nightly on PyPI if: github.repository == 'adap/flower' outputs: - name: ${{ steps.release.outputs.name }} - version: ${{ steps.release.outputs.version }} skip: ${{ steps.release.outputs.skip }} pip-version: ${{ steps.release.outputs.pip-version }} setuptools-version: ${{ steps.release.outputs.setuptools-version }} @@ -34,11 +32,12 @@ jobs: echo "skip=true" >> $GITHUB_OUTPUT fi - echo "name=$(poetry version | awk {'print $1'})" >> $GITHUB_OUTPUT - echo "version=$(poetry version -s)" >> $GITHUB_OUTPUT echo "pip-version=${{ steps.bootstrap.outputs.pip-version }}" >> "$GITHUB_OUTPUT" echo "setuptools-version=${{ steps.bootstrap.outputs.setuptools-version }}" >> "$GITHUB_OUTPUT" - python dev/build-docker-image-matrix.py --flwr-version nightly --simple > matrix.json + + NAME=$(poetry version | awk {'print $1'}) + VERSION=$(poetry version -s) + python dev/build-docker-image-matrix.py --flwr-version "${VERSION}" --matrix nightly --flwr-package "${NAME}" > matrix.json echo "matrix=$(cat matrix.json)" >> $GITHUB_OUTPUT build-docker-base-images: @@ -46,17 +45,17 @@ jobs: if: github.repository == 'adap/flower' && needs.release-nightly.outputs.skip != 'true' uses: ./.github/workflows/_docker-build.yml needs: release-nightly + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.release-nightly.outputs.matrix).base }} with: - namespace-repository: ${{ fromJson(needs.release-nightly.outputs.matrix).base.images[0].namespace_repository }} - file-dir: ${{ fromJson(needs.release-nightly.outputs.matrix).base.images[0].file_dir }} + namespace-repository: ${{ matrix.images.namespace_repository }} + file-dir: ${{ matrix.images.file_dir }} build-args: | PIP_VERSION=${{ needs.release-nightly.outputs.pip-version }} SETUPTOOLS_VERSION=${{ needs.release-nightly.outputs.setuptools-version }} - FLWR_VERSION=${{ needs.release-nightly.outputs.version }} - FLWR_PACKAGE=${{ needs.release-nightly.outputs.name }} - tags: | - ${{ needs.release-nightly.outputs.version }} - nightly + ${{ matrix.images.build_args_encoded }} + tags: ${{ matrix.images.tags_encoded }} secrets: dockerhub-user: ${{ secrets.DOCKERHUB_USERNAME }} dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }} @@ -72,10 +71,8 @@ jobs: with: namespace-repository: ${{ matrix.images.namespace_repository }} file-dir: ${{ matrix.images.file_dir }} - build-args: BASE_IMAGE=${{ needs.release-nightly.outputs.version }} - tags: | - ${{ needs.release-nightly.outputs.version }} - nightly + build-args: BASE_IMAGE=${{ matrix.images.base_image }} + tags: ${{ matrix.images.tags_encoded }} secrets: dockerhub-user: ${{ secrets.DOCKERHUB_USERNAME }} dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }} diff --git a/dev/build-docker-image-matrix.py b/dev/build-docker-image-matrix.py index 32b20123f9f9..9d255ac5471f 100644 --- a/dev/build-docker-image-matrix.py +++ b/dev/build-docker-image-matrix.py @@ -29,7 +29,7 @@ # class DistroName(StrEnum): # ALPINE = "alpine" # UBUNTU = "ubuntu" -assert sys.version_info < (3, 10), "Script requires Python 3.9 or lower." +assert sys.version_info < (3, 11), "Script requires Python 3.9 or lower." class DistroName(str, Enum): @@ -146,7 +146,7 @@ class BinaryImage: namespace_repository: str file_dir: str base_image: str - tags: List[str] + tags_encoded: str def new_binary_image( @@ -317,7 +317,7 @@ class StableBaseImageBuildArgs: # # Build matrix for unstable releases # -def build_unstable_matrix(flwr_version: str) -> List[BaseImage]: +def build_unstable_matrix(flwr_version_ref: str) -> List[BaseImage]: @dataclass class UnstableBaseImageBuildArgs: variant: Variant @@ -325,7 +325,7 @@ class UnstableBaseImageBuildArgs: flwr_version_ref: str cpu_ubuntu_build_args_variant = UnstableBaseImageBuildArgs( - UBUNTU_VARIANT, LATEST_SUPPORTED_PYTHON_VERSION, flwr_version + UBUNTU_VARIANT, LATEST_SUPPORTED_PYTHON_VERSION, flwr_version_ref ) cpu_build_args = """PYTHON_VERSION={python_version} @@ -347,7 +347,7 @@ class UnstableBaseImageBuildArgs: ) cuda_build_args_variant = UnstableBaseImageBuildArgs( - LATEST_SUPPORTED_CUDA_VERSION, LATEST_SUPPORTED_PYTHON_VERSION, flwr_version + LATEST_SUPPORTED_CUDA_VERSION, LATEST_SUPPORTED_PYTHON_VERSION, flwr_version_ref ) cuda_build_args = cpu_build_args + """CUDA_VERSION={cuda_version}""" @@ -412,7 +412,7 @@ class NightlyBaseImageBuildArgs: cpu_base_image = BaseImage( file_dir_fn=lambda args: f"{DOCKERFILE_ROOT}/base/{args.variant.distro.name.value}", - tags_fn=lambda args: ["nightly", args.flwr_version], + tags_fn=lambda args: [args.flwr_version, "nightly"], build_args_fn=lambda args: cpu_build_args.format( python_version=args.python_version, flwr_version=args.flwr_version, @@ -434,7 +434,7 @@ class NightlyBaseImageBuildArgs: cuda_base_image = BaseImage( file_dir_fn=lambda args: f"{DOCKERFILE_ROOT}/base/{args.variant.distro.name.value}-cuda", - tags_fn=lambda args: ["nightly-cuda", f"{args.flwr_version}-cuda"], + tags_fn=lambda args: [f"{args.flwr_version}-cuda", "nightly-cuda"], build_args_fn=lambda args: cuda_build_args.format( python_version=args.python_version, flwr_version=args.flwr_version, @@ -474,7 +474,7 @@ class NightlyBaseImageBuildArgs: description="Generate Github Docker workflow matrix" ) arg_parser.add_argument("--flwr-version", type=str, required=True) - arg_parser.add_argument("--flwr-package", type=str) + arg_parser.add_argument("--flwr-package", type=str, default="flwr") arg_parser.add_argument( "--matrix", choices=["stable", "nightly", "unstable"], default="stable" ) @@ -488,10 +488,6 @@ class NightlyBaseImageBuildArgs: if matrix == "stable": base_images, binary_images = build_stable_matrix(flwr_version) elif matrix == "nightly": - if flwr_package is None: - arg_parser.error( - "flag `--matrix nightly` requires to specify `--flwr-package`" - ) base_images, binary_images = build_nightly_matrix(flwr_version, flwr_package) else: base_images, binary_images = build_unstable_matrix(flwr_version)