-
-
Notifications
You must be signed in to change notification settings - Fork 256
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'main' into poc/scan_llm
- Loading branch information
Showing
63 changed files
with
2,809 additions
and
1,116 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,59 +1,249 @@ | ||
name: Build & Push dev images | ||
|
||
on: | ||
# schedule: | ||
# - cron: '0 5 * * *' | ||
push: | ||
branches: | ||
- main | ||
tags: | ||
- "v*.*.*" | ||
workflow_dispatch: | ||
inputs: | ||
run-tests: | ||
description: 'If all python tests should be run during build' | ||
required: true | ||
type: boolean | ||
default: true | ||
build_only: | ||
description: 'If images should be built and not push' | ||
required: true | ||
type: boolean | ||
default: false | ||
|
||
workflow_call: | ||
inputs: | ||
run-tests: | ||
description: 'If all python tests should be run during build' | ||
required: true | ||
type: boolean | ||
default: false | ||
build_only: | ||
description: 'If images should be built and not push' | ||
required: true | ||
type: boolean | ||
default: true | ||
|
||
pull_request: # This will allow to trigger on PR only with a specific label | ||
types: [opened, reopened, synchronize, labeled, unlabeled] | ||
# Concurrency : auto-cancel "old" jobs ie when pushing again | ||
# https://docs.github.com/fr/actions/using-jobs/using-concurrency | ||
concurrency: | ||
group: ${{ github.workflow }}-${{ github.ref || github.run_id }} | ||
cancel-in-progress: true | ||
env: | ||
RUN_TESTS: false | ||
BUILD_ONLY: false | ||
REGISTRY_IMAGE: 'giskardai/giskard' | ||
DOCKERHUB_USER: giskardai | ||
# https://docs.docker.com/build/ci/github-actions/multi-platform/#distribute-build-across-multiple-runners | ||
jobs: | ||
build-images: | ||
# Debug | ||
strategy: | ||
matrix: | ||
platform: ["linux/amd64", "linux/arm64"] | ||
fail-fast: false | ||
if: ${{ !github.event.pull_request || contains( github.event.pull_request.labels.*.name, 'Docker') }} | ||
runs-on: ubuntu-latest | ||
steps: | ||
- name: Check if python tests should be run | ||
if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' || inputs.run-tests }} | ||
run: echo 'RUN_TESTS=true' >> $GITHUB_ENV | ||
|
||
- name: Check if it should be build only (if yes, do not push the images) | ||
if: ${{ github.event_name == 'pull_request' || inputs.build_only }} | ||
run: echo 'BUILD_ONLY=true' >> $GITHUB_ENV | ||
|
||
- name: Check disk space | ||
if: ${{ env.RUN_TESTS == 'true' }} | ||
run: | | ||
df -h | ||
- name: List installed packages | ||
run: | | ||
dpkg-query --show --showformat='${Installed-Size}\t${Package}\n' | sort -rh | head -25 | awk '{print $1/1024, $2}' | ||
apt list --installed | wc -l | ||
- name: Free disk space | ||
if: ${{ env.RUN_TESTS == 'true' }} | ||
run: | | ||
sudo swapoff -a | ||
sudo rm -f /swapfile | ||
sudo rm -rf /usr/local/lib/android | ||
docker rmi $(docker image ls -aq) | ||
docker system prune -f | ||
sudo apt remove -y \ | ||
alsa-topology-conf alsa-ucm-conf \ | ||
google-cloud-cli azure-cli microsoft-edge-stable dotnet-sdk-7.0 dotnet-sdk-6.0 temurin-17-jdk \ | ||
google-chrome-stable temurin-11-jdk llvm-14-dev llvm-13-dev llvm-12-dev firefox temurin-8-jdk \ | ||
powershell mysql-server-core-8.0 libllvm15 libllvm14 libllvm13 libllvm12 libclang-common-13-dev | ||
sudo apt autoremove -y | ||
sudo apt autoclean -y | ||
- name: Check new disk space | ||
if: ${{ env.RUN_TESTS == 'true' }} | ||
run: | | ||
df -h | ||
- name: List remaning installed packages | ||
if: ${{ env.RUN_TESTS == 'true' }} | ||
run: | | ||
dpkg-query --show --showformat='${Installed-Size}\t${Package}\n' | sort -rh | head -25 | awk '{print $1/1024, $2}' | ||
apt list --installed | wc -l | ||
- name: Checkout code | ||
uses: actions/[email protected] | ||
uses: actions/[email protected] | ||
with: | ||
fetch-depth: 1 | ||
|
||
- name: Set up Docker Buildx | ||
id: builder | ||
uses: docker/setup-buildx-action@master | ||
uses: docker/setup-buildx-action@v3 | ||
|
||
- name: "Append ARM buildx builder from Oracle cloud" | ||
if: ${{ matrix.platform == 'linux/arm64'}} | ||
uses: baschny/append-buildx-action@v1 | ||
with: | ||
builder: ${{ steps.builder.outputs.name }} | ||
endpoint: ssh://${{ secrets.ARM_SSH_CONNECTION_STRING }} | ||
ssh_private_key: ${{ secrets.ARM_SSH_PRIVATE_KEY }} | ||
|
||
- name: Extract branch name | ||
shell: bash | ||
run: | | ||
BRANCH=${GITHUB_REF#refs/heads/} | ||
SANITIZED_BRANCH=${BRANCH//"/"/-} | ||
echo $BRANCH | ||
echo $SANITIZED_BRANCH | ||
echo "branch=$BRANCH" >> $GITHUB_OUTPUT | ||
echo "tag=$SANITIZED_BRANCH" >> $GITHUB_OUTPUT | ||
id: extract_branch | ||
|
||
- name: Docker meta | ||
id: meta | ||
uses: docker/metadata-action@v5 | ||
with: | ||
images: | | ||
${{ env.REGISTRY_IMAGE }} | ||
tags: | | ||
type=edge,branch=main | ||
type=ref,event=branch | ||
type=schedule | ||
type=ref,event=tag | ||
type=ref,event=pr | ||
- name: Login to Docker registry | ||
uses: docker/login-action@v2 | ||
if: ${{ env.BUILD_ONLY != 'true' }} | ||
uses: docker/login-action@v3 | ||
with: | ||
username: giskardai | ||
username: ${{ env.DOCKERHUB_USER }} | ||
password: ${{ secrets.DOCKERHUB_TOKEN }} | ||
|
||
- name: Build and push image based on the current branch | ||
env: | ||
IMAGE: docker.io/giskardai/giskard | ||
- name: Install deps inside docker | ||
if: ${{ env.RUN_TESTS == 'true' }} | ||
uses: docker/build-push-action@v5 | ||
with: | ||
context: . | ||
target: full-install-python | ||
load: true | ||
tags: ${{ steps.meta.outputs.tags }} | ||
labels: ${{ steps.meta.outputs.labels }} | ||
builder: ${{ steps.builder.outputs.name }} | ||
platforms: | | ||
${{ matrix.platform}} | ||
- name: Run python test inside docker | ||
if: ${{ env.RUN_TESTS == 'true' }} | ||
uses: docker/build-push-action@v5 | ||
with: | ||
context: . | ||
target: test-python | ||
tags: ${{ steps.meta.outputs.tags }} | ||
labels: ${{ steps.meta.outputs.labels }} | ||
builder: ${{ steps.builder.outputs.name }} | ||
platforms: | | ||
${{ matrix.platform}} | ||
# - name: Run python integration test inside docker | ||
# if: ${{ env.RUN_TESTS == 'true' }} | ||
# uses: docker/build-push-action@v5 | ||
# with: | ||
# context: . | ||
# target: integration-test-python | ||
# push: false | ||
# load: false | ||
# tags: ${{ steps.meta.outputs.tags }} | ||
# labels: ${{ steps.meta.outputs.labels }} | ||
# builder: ${{ steps.builder.outputs.name }} | ||
# platforms: | | ||
# ${{ matrix.platform}} | ||
# cache-from: type=gha | ||
|
||
- name: Build and push | ||
id: build | ||
uses: docker/build-push-action@v5 | ||
with: | ||
context: . | ||
target: prod | ||
labels: ${{ steps.meta.outputs.labels }} | ||
builder: ${{ steps.builder.outputs.name }} | ||
platforms: | | ||
${{ matrix.platform}} | ||
outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=${{ env.BUILD_ONLY != 'true' }} | ||
|
||
# For details, see link below | ||
# https://docs.docker.com/build/ci/github-actions/multi-platform/ | ||
- name: Export digest | ||
run: | | ||
docker buildx build \ | ||
--platform linux/amd64,linux/arm64 \ | ||
--tag $IMAGE:${{ steps.extract_branch.outputs.tag }} \ | ||
--file Dockerfile \ | ||
--push \ | ||
. | ||
mkdir -p /tmp/digests | ||
digest="${{ steps.build.outputs.digest }}" | ||
touch "/tmp/digests/${digest#sha256:}" | ||
- name: Upload digest | ||
uses: actions/upload-artifact@v3 | ||
with: | ||
name: digests | ||
path: /tmp/digests/* | ||
if-no-files-found: error | ||
retention-days: 1 | ||
|
||
- name: Invoke deployment hook | ||
if: ${{ steps.extract_branch.outputs.branch == 'main' }} | ||
merge: | ||
runs-on: ubuntu-latest | ||
if: ${{ github.event_name != 'pull_request' && !inputs.build_only }} | ||
needs: | ||
- build-images | ||
steps: | ||
- name: Download digests | ||
uses: actions/download-artifact@v3 | ||
with: | ||
name: digests | ||
path: /tmp/digests | ||
- name: Set up Docker Buildx | ||
uses: docker/setup-buildx-action@v3 | ||
- name: Docker meta | ||
id: meta | ||
uses: docker/metadata-action@v5 | ||
with: | ||
images: | | ||
${{ env.REGISTRY_IMAGE }} | ||
tags: | | ||
type=edge,branch=main | ||
type=ref,event=branch | ||
type=schedule | ||
type=ref,event=tag | ||
type=ref,event=pr | ||
- name: Login to Docker Hub | ||
uses: docker/login-action@v3 | ||
with: | ||
username: ${{ env.DOCKERHUB_USER }} | ||
password: ${{ secrets.DOCKERHUB_TOKEN }} | ||
- name: Create manifest list and push | ||
working-directory: /tmp/digests | ||
run: | | ||
curl -L --silent --max-time 900 ${{ secrets.TEST_SERVER_WEBHOOK_URL_BASE }}/redeploy-dev | ||
docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ | ||
$(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *) | ||
- name: Inspect image | ||
run: | | ||
docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ steps.meta.outputs.version }} | ||
# - name: Invoke deployment hook | ||
# if: ${{ steps.extract_branch.outputs.branch == 'main' && env.BUILD_ONLY != 'true' }} | ||
# run: | | ||
# curl -L --silent --max-time 900 ${{ secrets.TEST_SERVER_WEBHOOK_URL_BASE }}/redeploy-dev |
This file was deleted.
Oops, something went wrong.
Oops, something went wrong.