Skip to content

Commit 5238988

Browse files
safoinmeactions-user
safoinme
and
actions-user
authored
Fix Nightly Release (#2711)
* add pernmessions * fix nighly release * fix format * update nightly workflow * update * fix input * fix docker version * format * review update * fix publish docker and format * update docker build * Auto-update of Starter template * fix nightly release docker push * format * update docker build * temproarly ignore test * format * get test back * add license and format * format * format * format * Update license_header * Auto-update of Starter template --------- Co-authored-by: GitHub Actions <[email protected]>
1 parent a15ed41 commit 5238988

9 files changed

+96
-97
lines changed

‎.github/workflows/nightly_build.yml

+4
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,9 @@ jobs:
1515
secrets: inherit
1616
if: github.repository == 'zenml-io/zenml'
1717
publish-python-package:
18+
permissions:
19+
id-token: write
20+
contents: read
1821
needs: setup-and-test
1922
uses: ./.github/workflows/publish_to_pypi_nightly.yml
2023
secrets: inherit
@@ -33,4 +36,5 @@ jobs:
3336
uses: ./.github/workflows/publish_docker_image.yml
3437
with:
3538
config_file: release-cloudbuild-nightly.yaml
39+
zenml_nightly: true
3640
secrets: inherit

‎.github/workflows/publish_docker_image.yml

+19-36
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,11 @@ on:
88
required: false
99
type: string
1010
default: release-cloudbuild.yaml
11-
11+
zenml_nightly:
12+
description: Set to true for nightly builds
13+
required: false
14+
type: boolean
15+
default: false
1216
jobs:
1317
publish_to_docker:
1418
name: Publish Docker 🐋 image 🖼️ to Dockerhub
@@ -19,10 +23,18 @@ jobs:
1923
PYTHONIOENCODING: utf-8
2024
steps:
2125
- uses: actions/[email protected]
22-
- name: Get the version from the github tag ref
23-
id: get_version
24-
run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
25-
26+
- name: Determine version
27+
run: |-
28+
if [[ "${GITHUB_REF}" == refs/tags/* ]]; then
29+
VERSION=${GITHUB_REF#refs/tags/}
30+
else
31+
CURRENT_VERSION=$(cat src/zenml/VERSION)
32+
DATE=$(date +"%Y%m%d")
33+
VERSION="${CURRENT_VERSION}.dev${DATE}"
34+
fi
35+
VERSION=$(echo $VERSION | sed 's/[^a-zA-Z0-9._-]/-/g')
36+
echo "VERSION=$VERSION" >> $GITHUB_ENV
37+
echo "Determined VERSION=$VERSION"
2638
# Setup gcloud CLI
2739
- uses: google-github-actions/setup-gcloud@v0
2840
with:
@@ -33,37 +45,8 @@ jobs:
3345
# Cloudbuild
3446
- name: Build docker images
3547
run: |-
48+
echo "Building Docker image with tag: $VERSION"
3649
gcloud builds submit \
3750
--quiet \
3851
--config=${{inputs.config_file}} \
39-
--substitutions=TAG_NAME=${{github.ref_name}} .
40-
wait-for-package-release:
41-
runs-on: arc-runner-set
42-
needs: publish_to_docker
43-
steps:
44-
- name: Sleep for 4 minutes
45-
run: sleep 240
46-
shell: bash
47-
# create a tag on the ZenML cloud plugins repo
48-
create_tag_on_cloud_plugins_repo:
49-
runs-on: ubuntu-latest
50-
needs: wait-for-package-release
51-
steps:
52-
- name: Get the sha of the latest commit on plugins/main
53-
id: get_sha
54-
run: |
55-
echo "::set-output name=sha::$(curl -s -H "Authorization: token ${{ secrets.CLOUD_PLUGINS_REPO_PAT }}" https://api.github.com/repos/zenml-io/zenml-cloud-plugins/commits/main | jq -r '.sha')"
56-
- name: Get the version from the github tag ref
57-
id: get_version
58-
run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
59-
- name: Create a tag on ZenML Cloud plugins repo
60-
uses: actions/github-script@v7
61-
with:
62-
github-token: ${{ secrets.CLOUD_PLUGINS_REPO_PAT }}
63-
script: |-
64-
await github.rest.git.createRef({
65-
owner: 'zenml-io',
66-
repo: 'zenml-cloud-plugins',
67-
ref: 'refs/tags/${{ steps.get_version.outputs.VERSION }}',
68-
sha: '${{ steps.get_sha.outputs.sha }}'
69-
})
52+
--substitutions=TAG_NAME=$VERSION

‎.github/workflows/publish_to_pypi_nightly.yml

+17-3
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,9 @@ jobs:
4444
4545
# Update the version to the nightly version
4646
poetry version $NIGHTLY_VERSION
47+
48+
# Update the src/zenml/VERSION file with the nightly version
49+
echo $NIGHTLY_VERSION > src/zenml/VERSION
4750
- name: Include latest dashboard
4851
run: bash scripts/install-dashboard.sh
4952
- name: Build package
@@ -53,7 +56,18 @@ jobs:
5356
uses: tschm/[email protected]
5457
- name: Publish the package with poetry
5558
run: |-
56-
if [ "$(cat src/zenml/VERSION)" = "$(echo ${GITHUB_REF} | sed 's|refs/tags/||g')" ];
57-
then poetry publish -u __token__ -p '${{ steps.mint.outputs.api-token }}';
58-
else echo "Version mismatch between src/zenml/VERSION and branch tag" && exit 1;
59+
# Extract the tag from GITHUB_REF if it exists
60+
TAG=$(echo ${GITHUB_REF} | sed 's|refs/tags/||g')
61+
62+
# Check if the current ref is a tag
63+
if [[ "${GITHUB_REF}" == refs/tags/* ]]; then
64+
# Check if the VERSION file matches the tag
65+
if [ "$(cat src/zenml/VERSION)" = "$TAG" ]; then
66+
poetry publish -u __token__ -p '${{ steps.mint.outputs.api-token }}';
67+
else
68+
echo "Version mismatch between src/zenml/VERSION and branch tag" && exit 1;
69+
fi
70+
else
71+
# For branches, publish the nightly version without version check
72+
poetry publish -u __token__ -p '${{ steps.mint.outputs.api-token }}';
5973
fi

‎.github/workflows/release.yml

+30
Original file line numberDiff line numberDiff line change
@@ -114,3 +114,33 @@ jobs:
114114
needs: publish-docker-image
115115
uses: ./.github/workflows/publish_helm_chart.yml
116116
secrets: inherit
117+
wait-for-package-release-again:
118+
runs-on: arc-runner-set
119+
needs: publish-helm-chart
120+
steps:
121+
- name: Sleep for 4 minutes
122+
run: sleep 240
123+
shell: bash
124+
# create a tag on the ZenML cloud plugins repo
125+
create_tag_on_cloud_plugins_repo:
126+
runs-on: ubuntu-latest
127+
needs: wait-for-package-release-again
128+
steps:
129+
- name: Get the sha of the latest commit on plugins/main
130+
id: get_sha
131+
run: |
132+
echo "::set-output name=sha::$(curl -s -H "Authorization: token ${{ secrets.CLOUD_PLUGINS_REPO_PAT }}" https://api.github.com/repos/zenml-io/zenml-cloud-plugins/commits/main | jq -r '.sha')"
133+
- name: Get the version from the github tag ref
134+
id: get_version
135+
run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
136+
- name: Create a tag on ZenML Cloud plugins repo
137+
uses: actions/github-script@v7
138+
with:
139+
github-token: ${{ secrets.CLOUD_PLUGINS_REPO_PAT }}
140+
script: |-
141+
await github.rest.git.createRef({
142+
owner: 'zenml-io',
143+
repo: 'zenml-cloud-plugins',
144+
ref: 'refs/tags/${{ steps.get_version.outputs.VERSION }}',
145+
sha: '${{ steps.get_sha.outputs.sha }}'
146+
})

‎docker/base.Dockerfile

+23-26
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
ARG PYTHON_VERSION=3.11
2+
ARG ZENML_VERSION=""
3+
ARG ZENML_NIGHTLY="false"
24

35
# Use a minimal base image to reduce the attack surface
46
FROM python:${PYTHON_VERSION}-slim-bookworm AS base
@@ -20,6 +22,7 @@ FROM base AS builder
2022

2123
ARG VIRTUAL_ENV=/opt/venv
2224
ARG ZENML_VERSION
25+
ARG ZENML_NIGHTLY="false"
2326

2427
ENV \
2528
# Set up virtual environment
@@ -42,45 +45,39 @@ FROM builder as client-builder
4245

4346
ARG VIRTUAL_ENV=/opt/venv
4447
ARG ZENML_VERSION
48+
ARG ZENML_NIGHTLY="false"
4549

4650
RUN python3 -m venv $VIRTUAL_ENV
4751
ENV PATH="$VIRTUAL_ENV/bin:$PATH"
4852

49-
# Install client build dependencies
50-
#
51-
# NOTE: System packages required for the client build stage should be installed
52-
# here
53-
54-
# Upgrade pip to the latest version and install the given zenml version (default
55-
# to latest).
56-
# Also create a requirements.txt file to keep track of
57-
# dependencies for reproducibility and debugging.
58-
RUN pip install --upgrade pip \
59-
&& pip install zenml${ZENML_VERSION:+==$ZENML_VERSION} \
60-
&& pip freeze > requirements.txt
61-
53+
# Determine the package name based on ZENML_NIGHTLY
54+
RUN if [ "$ZENML_NIGHTLY" = "true" ]; then \
55+
PACKAGE_NAME="zenml-nightly"; \
56+
else \
57+
PACKAGE_NAME="zenml"; \
58+
fi \
59+
&& pip install --upgrade pip \
60+
&& pip install ${PACKAGE_NAME}${ZENML_VERSION:+==$ZENML_VERSION} \
61+
&& pip freeze > requirements.txt
6262

6363
FROM builder as server-builder
6464

6565
ARG VIRTUAL_ENV=/opt/venv
6666
ARG ZENML_VERSION
67+
ARG ZENML_NIGHTLY="false"
6768

6869
RUN python3 -m venv $VIRTUAL_ENV
6970
ENV PATH="$VIRTUAL_ENV/bin:$PATH"
7071

71-
72-
# Install server build dependencies
73-
#
74-
# NOTE: System packages required for the server build stage should be installed
75-
# here
76-
77-
# Upgrade pip to the latest version and install the given zenml server version
78-
# (default to latest).
79-
# Also create a requirements.txt file to keep track of
80-
# dependencies for reproducibility and debugging.
81-
RUN pip install --upgrade pip \
82-
&& pip install "zenml[server,secrets-aws,secrets-gcp,secrets-azure,secrets-hashicorp,s3fs,gcsfs,adlfs,connectors-aws,connectors-gcp,connectors-azure]${ZENML_VERSION:+==$ZENML_VERSION}" \
83-
&& pip freeze > requirements.txt
72+
# Determine the package name based on ZENML_NIGHTLY
73+
RUN if [ "$ZENML_NIGHTLY" = "true" ]; then \
74+
PACKAGE_NAME="zenml-nightly"; \
75+
else \
76+
PACKAGE_NAME="zenml"; \
77+
fi \
78+
&& pip install --upgrade pip \
79+
&& pip install "${PACKAGE_NAME}[server,secrets-aws,secrets-gcp,secrets-azure,secrets-hashicorp,s3fs,gcsfs,adlfs,connectors-aws,connectors-gcp,connectors-azure]${ZENML_VERSION:+==$ZENML_VERSION}" \
80+
&& pip freeze > requirements.txt
8481

8582
FROM base as client
8683

‎examples/quickstart/license

-15
This file was deleted.

‎examples/quickstart/license_header

-16
This file was deleted.

‎examples/quickstart/pipelines/inference.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828

2929

3030
@pipeline
31-
def inference(random_state: str, target: str):
31+
def inference(random_state: int, target: str):
3232
"""
3333
Model inference pipeline.
3434

‎release-cloudbuild-nightly.yaml

+2
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ steps:
77
docker build \
88
--build-arg ZENML_VERSION=$TAG_NAME \
99
--build-arg PYTHON_VERSION=3.10 \
10+
--build-arg ZENML_NIGHTLY=true \
1011
--target client \
1112
-f docker/base.Dockerfile . \
1213
-t $$USERNAME/zenml:$TAG_NAME-py3.10-nightly
@@ -28,6 +29,7 @@ steps:
2829
docker build \
2930
--build-arg ZENML_VERSION=$TAG_NAME \
3031
--build-arg PYTHON_VERSION=3.11 \
32+
--build-arg ZENML_NIGHTLY=true \
3133
--target server \
3234
-f docker/base.Dockerfile . \
3335
-t $$USERNAME/zenml-server:$TAG_NAME-nightly

0 commit comments

Comments
 (0)