Skip to content

Commit

Permalink
refactor ci
Browse files Browse the repository at this point in the history 
Signed-off-by: Robert Steiner <[email protected]>
  • Loading branch information
@Robert-Steiner
Robert-Steiner committed Nov 13, 2024
1 parent 3b92eff commit f67d25a
Show file tree
Hide file tree
Showing 4 changed files with 36 additions and 41 deletions.
20 changes: 11 additions & 9 deletions .github/workflows/docker-build-main.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ jobs:
outputs:
pip-version: ${{ steps.versions.outputs.pip-version }}
setuptools-version: ${{ steps.versions.outputs.setuptools-version }}
flwr-version-ref: ${{ steps.versions.outputs.flwr-version-ref }}
matrix: ${{ steps.versions.outputs.matrix }}
steps:
- uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
Expand All @@ -26,23 +25,26 @@ jobs:
run: |
echo "pip-version=${{ steps.bootstrap.outputs.pip-version }}" >> "$GITHUB_OUTPUT"
echo "setuptools-version=${{ steps.bootstrap.outputs.setuptools-version }}" >> "$GITHUB_OUTPUT"
echo "flwr-version-ref=git+${{ github.server_url }}/${{ github.repository }}.git@${{ github.sha }}" >> "$GITHUB_OUTPUT"
python dev/build-docker-image-matrix.py --flwr-version unstable --simple > matrix.json
FLWR_VERSION_REF="git+${{ github.server_url }}/${{ github.repository }}.git@${{ github.sha }}"
python dev/build-docker-image-matrix.py --flwr-version "${FLWR_VERSION_REF}" --matrix unstable > matrix.json
echo "matrix=$(cat matrix.json)" >> $GITHUB_OUTPUT
build-docker-base-images:
name: Build base images
if: github.repository == 'adap/flower'
uses: ./.github/workflows/_docker-build.yml
needs: parameters
strategy:
fail-fast: false
matrix: ${{ fromJson(needs.parameters.outputs.matrix).base }}
with:
namespace-repository: ${{ fromJson(needs.parameters.outputs.matrix).base.images[0].namespace_repository }}
file-dir: ${{ fromJson(needs.parameters.outputs.matrix).base.images[0].file_dir }}
namespace-repository: ${{ matrix.images.namespace_repository }}
file-dir: ${{ matrix.images.file_dir }}
build-args: |
PIP_VERSION=${{ needs.parameters.outputs.pip-version }}
SETUPTOOLS_VERSION=${{ needs.parameters.outputs.setuptools-version }}
FLWR_VERSION_REF=${{ needs.parameters.outputs.flwr-version-ref }}
tags: unstable
${{ matrix.images.build_args_encoded }}
tags: ${{ matrix.images.tags_encoded }}
secrets:
dockerhub-user: ${{ secrets.DOCKERHUB_USERNAME }}
dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }}
Expand All @@ -58,8 +60,8 @@ jobs:
with:
namespace-repository: ${{ matrix.images.namespace_repository }}
file-dir: ${{ matrix.images.file_dir }}
build-args: BASE_IMAGE=unstable
tags: unstable
build-args: BASE_IMAGE=${{ matrix.images.base_image }}
tags: ${{ matrix.images.tags_encoded }}
secrets:
dockerhub-user: ${{ secrets.DOCKERHUB_USERNAME }}
dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }}
8 changes: 4 additions & 4 deletions .github/workflows/framework-release.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ jobs:
- id: matrix
run: |
python dev/build-docker-image-matrix.py --flwr-version "${{ needs.publish.outputs.flwr-version }}" > matrix.json
python dev/build-docker-image-matrix.py --flwr-version "${{ needs.publish.outputs.flwr-version }}" --matrix stable > matrix.json
echo "matrix=$(cat matrix.json)" >> $GITHUB_OUTPUT
build-base-images:
Expand All @@ -88,8 +88,8 @@ jobs:
build-args: |
PIP_VERSION=${{ needs.parameters.outputs.pip-version }}
SETUPTOOLS_VERSION=${{ needs.parameters.outputs.setuptools-version }}
${{ matrix.images.build_args }}
tags: ${{ matrix.images.tag }}
${{ matrix.images.build_args_encoded }}
tags: ${{ matrix.images.tags_encoded }}
secrets:
dockerhub-user: ${{ secrets.DOCKERHUB_USERNAME }}
dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }}
Expand All @@ -106,7 +106,7 @@ jobs:
namespace-repository: ${{ matrix.images.namespace_repository }}
file-dir: ${{ matrix.images.file_dir }}
build-args: BASE_IMAGE=${{ matrix.images.base_image }}
tags: ${{ matrix.images.tags }}
tags: ${{ matrix.images.tags_encoded }}
secrets:
dockerhub-user: ${{ secrets.DOCKERHUB_USERNAME }}
dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }}
29 changes: 13 additions & 16 deletions .github/workflows/release-nightly.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,6 @@ jobs:
name: Relase nightly on PyPI
if: github.repository == 'adap/flower'
outputs:
name: ${{ steps.release.outputs.name }}
version: ${{ steps.release.outputs.version }}
skip: ${{ steps.release.outputs.skip }}
pip-version: ${{ steps.release.outputs.pip-version }}
setuptools-version: ${{ steps.release.outputs.setuptools-version }}
Expand All @@ -34,29 +32,30 @@ jobs:
echo "skip=true" >> $GITHUB_OUTPUT
fi
echo "name=$(poetry version | awk {'print $1'})" >> $GITHUB_OUTPUT
echo "version=$(poetry version -s)" >> $GITHUB_OUTPUT
echo "pip-version=${{ steps.bootstrap.outputs.pip-version }}" >> "$GITHUB_OUTPUT"
echo "setuptools-version=${{ steps.bootstrap.outputs.setuptools-version }}" >> "$GITHUB_OUTPUT"
python dev/build-docker-image-matrix.py --flwr-version nightly --simple > matrix.json
NAME=$(poetry version | awk {'print $1'})
VERSION=$(poetry version -s)
python dev/build-docker-image-matrix.py --flwr-version "${VERSION}" --matrix nightly --flwr-package "${NAME}" > matrix.json
echo "matrix=$(cat matrix.json)" >> $GITHUB_OUTPUT
build-docker-base-images:
name: Build nightly base images
if: github.repository == 'adap/flower' && needs.release-nightly.outputs.skip != 'true'
uses: ./.github/workflows/_docker-build.yml
needs: release-nightly
strategy:
fail-fast: false
matrix: ${{ fromJson(needs.release-nightly.outputs.matrix).base }}
with:
namespace-repository: ${{ fromJson(needs.release-nightly.outputs.matrix).base.images[0].namespace_repository }}
file-dir: ${{ fromJson(needs.release-nightly.outputs.matrix).base.images[0].file_dir }}
namespace-repository: ${{ matrix.images.namespace_repository }}
file-dir: ${{ matrix.images.file_dir }}
build-args: |
PIP_VERSION=${{ needs.release-nightly.outputs.pip-version }}
SETUPTOOLS_VERSION=${{ needs.release-nightly.outputs.setuptools-version }}
FLWR_VERSION=${{ needs.release-nightly.outputs.version }}
FLWR_PACKAGE=${{ needs.release-nightly.outputs.name }}
tags: |
${{ needs.release-nightly.outputs.version }}
nightly
${{ matrix.images.build_args_encoded }}
tags: ${{ matrix.images.tags_encoded }}
secrets:
dockerhub-user: ${{ secrets.DOCKERHUB_USERNAME }}
dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }}
Expand All @@ -72,10 +71,8 @@ jobs:
with:
namespace-repository: ${{ matrix.images.namespace_repository }}
file-dir: ${{ matrix.images.file_dir }}
build-args: BASE_IMAGE=${{ needs.release-nightly.outputs.version }}
tags: |
${{ needs.release-nightly.outputs.version }}
nightly
build-args: BASE_IMAGE=${{ matrix.images.base_image }}
tags: ${{ matrix.images.tags_encoded }}
secrets:
dockerhub-user: ${{ secrets.DOCKERHUB_USERNAME }}
dockerhub-token: ${{ secrets.DOCKERHUB_TOKEN }}
20 changes: 8 additions & 12 deletions dev/build-docker-image-matrix.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
# class DistroName(StrEnum):
# ALPINE = "alpine"
# UBUNTU = "ubuntu"
assert sys.version_info < (3, 10), "Script requires Python 3.9 or lower."
assert sys.version_info < (3, 11), "Script requires Python 3.9 or lower."


class DistroName(str, Enum):
Expand Down Expand Up @@ -146,7 +146,7 @@ class BinaryImage:
namespace_repository: str
file_dir: str
base_image: str
tags: List[str]
tags_encoded: str


def new_binary_image(
Expand Down Expand Up @@ -317,15 +317,15 @@ class StableBaseImageBuildArgs:
#
# Build matrix for unstable releases
#
def build_unstable_matrix(flwr_version: str) -> List[BaseImage]:
def build_unstable_matrix(flwr_version_ref: str) -> List[BaseImage]:
@dataclass
class UnstableBaseImageBuildArgs:
variant: Variant
python_version: str
flwr_version_ref: str

cpu_ubuntu_build_args_variant = UnstableBaseImageBuildArgs(
UBUNTU_VARIANT, LATEST_SUPPORTED_PYTHON_VERSION, flwr_version
UBUNTU_VARIANT, LATEST_SUPPORTED_PYTHON_VERSION, flwr_version_ref
)

cpu_build_args = """PYTHON_VERSION={python_version}
Expand All @@ -347,7 +347,7 @@ class UnstableBaseImageBuildArgs:
)

cuda_build_args_variant = UnstableBaseImageBuildArgs(
LATEST_SUPPORTED_CUDA_VERSION, LATEST_SUPPORTED_PYTHON_VERSION, flwr_version
LATEST_SUPPORTED_CUDA_VERSION, LATEST_SUPPORTED_PYTHON_VERSION, flwr_version_ref
)

cuda_build_args = cpu_build_args + """CUDA_VERSION={cuda_version}"""
Expand Down Expand Up @@ -412,7 +412,7 @@ class NightlyBaseImageBuildArgs:

cpu_base_image = BaseImage(
file_dir_fn=lambda args: f"{DOCKERFILE_ROOT}/base/{args.variant.distro.name.value}",
tags_fn=lambda args: ["nightly", args.flwr_version],
tags_fn=lambda args: [args.flwr_version, "nightly"],
build_args_fn=lambda args: cpu_build_args.format(
python_version=args.python_version,
flwr_version=args.flwr_version,
Expand All @@ -434,7 +434,7 @@ class NightlyBaseImageBuildArgs:

cuda_base_image = BaseImage(
file_dir_fn=lambda args: f"{DOCKERFILE_ROOT}/base/{args.variant.distro.name.value}-cuda",
tags_fn=lambda args: ["nightly-cuda", f"{args.flwr_version}-cuda"],
tags_fn=lambda args: [f"{args.flwr_version}-cuda", "nightly-cuda"],
build_args_fn=lambda args: cuda_build_args.format(
python_version=args.python_version,
flwr_version=args.flwr_version,
Expand Down Expand Up @@ -474,7 +474,7 @@ class NightlyBaseImageBuildArgs:
description="Generate Github Docker workflow matrix"
)
arg_parser.add_argument("--flwr-version", type=str, required=True)
arg_parser.add_argument("--flwr-package", type=str)
arg_parser.add_argument("--flwr-package", type=str, default="flwr")
arg_parser.add_argument(
"--matrix", choices=["stable", "nightly", "unstable"], default="stable"
)
Expand All @@ -488,10 +488,6 @@ class NightlyBaseImageBuildArgs:
if matrix == "stable":
base_images, binary_images = build_stable_matrix(flwr_version)
elif matrix == "nightly":
if flwr_package is None:
arg_parser.error(
"flag `--matrix nightly` requires to specify `--flwr-package`"
)
base_images, binary_images = build_nightly_matrix(flwr_version, flwr_package)
else:
base_images, binary_images = build_unstable_matrix(flwr_version)
Expand Down

0 comments on commit f67d25a

Please sign in to comment.