Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ZIL-5326: remove dependency on Python3 & boost::filesystem #1241

Merged
merged 6 commits into from
Sep 8, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 44 additions & 0 deletions .github/workflows/ci-image-dev.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
name: CI - Development

on:
workflow_dispatch:
inputs:
commitOrTag:
description: 'Commit or tag'
required: false
default: ''
push:
branches-ignore: []

jobs:
push-to-ecr:
permissions:
id-token: write
contents: write
name: build
runs-on: docker
steps:
- name: Clean environment
# Prune the Docker resources created over 10 days before the current execution (change the value for a more/less aggressive cleanup).
shell: bash
run: |
docker system df
docker system prune -a -f --filter "until=168h"
docker system df
- name: 'Checkout scm ${{ inputs.commitOrTag }}'
uses: actions/checkout@v3
with:
fetch-depth: 0
ref: ${{ inputs.commitOrTag }}
- name: Docker build and push
uses: Zilliqa/gh-actions-workflows/actions/ci-dockerized-app-build-push@v1
with:
file: docker/Dockerfile
push: true
tag: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com/scilla
tag-length: 8
registry: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com
aws-region: ${{ secrets.AWS_REGION_ZILLIQA }}
role-to-assume: ${{ secrets.ECR_DEPLOYER_ROLE }}
oidc-role: ${{ secrets.OIDC_ROLE }}
cache-key: ${{ github.event.repository.name }}
66 changes: 66 additions & 0 deletions .github/workflows/ci-image-release.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
name: CI - Release

on:
workflow_dispatch:
inputs:
commitOrTag:
description: 'Commit or tag'
required: false
default: ''
push:
tags:
- v*

jobs:
release-image:
permissions:
id-token: write
contents: write
runs-on: docker
steps:
- name: 'Checkout scm ${{ inputs.commitOrTag }}'
uses: actions/checkout@v3
with:
fetch-depth: 0
ref: ${{ inputs.commitOrTag }}
- name: Check if the commit or tag was passed manually
id: set-tag
run: |
if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
echo "tag=$(git rev-parse HEAD | cut -c1-8)" >> $GITHUB_OUTPUT
else
echo "tag=${{ github.ref_name }}" >> $GITHUB_OUTPUT
fi
shell: bash
- name: Check if the tag head is the same as release-v0.13.4 head
id: check-latest
if: github.event_name != 'workflow_dispatch'
run: |
if test $(git rev-parse origin/release-v0.13.4) = $(git rev-parse HEAD); then
echo "latest=true" >> $GITHUB_OUTPUT
else
echo "latest=false" >> $GITHUB_OUTPUT
fi
shell: bash
- name: Docker build and push
uses: Zilliqa/gh-actions-workflows/actions/ci-dockerized-app-build-push@v1
with:
file: docker/Dockerfile
push: true
tag: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com/scilla:${{ steps.set-tag.outputs.tag }}
tag-latest: ${{ steps.check-latest.outputs.latest }}
registry: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com
aws-region: ${{ secrets.AWS_REGION_ZILLIQA }}
role-to-assume: ${{ secrets.ECR_DEPLOYER_ROLE }}
oidc-role: ${{ secrets.OIDC_ROLE }}
cache-key: ${{ github.event.repository.name }}
- name: Docker build and push (Dockerhub)
uses: Zilliqa/gh-actions-workflows/actions/ci-dockerized-app-build-push@v1
with:
file: docker/Dockerfile
push: true
tag: Zilliqa/scilla:${{ steps.set-tag.outputs.tag }}
tag-latest: ${{ steps.check-latest.outputs.latest }}
cache-key: ${{ github.event.repository.name }}
registry-username: ${{ secrets.DOCKERHUB_USERNAME }}
registry-password: ${{ secrets.DOCKERHUB_PASSWORD }}
63 changes: 63 additions & 0 deletions .github/workflows/ci-image-test.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
name: CI - Integration tests

on:
workflow_dispatch:
inputs:
commitOrTag:
description: 'Commit or tag'
required: false
default: ''
pull_request:
branches:
- 'release-v**'

jobs:
run-tests:
permissions:
id-token: write
contents: write
name: tests
runs-on: docker
steps:
- name: Clean environment
# Prune the Docker resources created over 10 days before the current execution (change the value for a more/less aggressive cleanup).
shell: bash
run: |
docker system df
docker system prune -a -f --filter "until=336h"
docker system df
- name: 'Checkout scm ${{ inputs.commitOrTag }}'
uses: actions/checkout@v3
with:
fetch-depth: 0
ref: ${{ inputs.commitOrTag }}
- name: Configure AWS Credentials
uses: Zilliqa/gh-actions-workflows/actions/configure-aws-credentials@v1
with:
role-to-assume: ${{ secrets.ECR_DEPLOYER_ROLE }}
oidc-role: ${{ secrets.OIDC_ROLE }}
aws-region: ${{ secrets.AWS_REGION_ZILLIQA }}
- name: Login to the registry
uses: docker/login-action@v2
with:
registry: ${{ secrets.AWS_ACCOUNT_ID_ZILLIQA }}.dkr.ecr.${{ secrets.AWS_REGION_ZILLIQA }}.amazonaws.com
- name: Build Docker images
run: |
DOCKER_BUILDKIT=1 docker build --target test_runner -t scilla:tests -f docker/Dockerfile .
shell: bash
- name: Run make test
run: |
docker run --rm -i scilla:tests bash -c 'eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make test'
shell: bash
- name: Run make test_server
run: |
docker run --rm -i scilla:tests bash -c 'eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make test_server'
shell: bash
- name: Run make coveralls
run: |
docker run --rm -i scilla:tests bash -c 'eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make coveralls TRAVIS_JOB_ID=${{ github.run_number }}'
shell: bash
- name: Run make lint
run: |
docker run --rm -i scilla:tests bash -c 'eval $(opam env) && LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib make lint'
shell: bash
5 changes: 4 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,10 @@ clean:
dune clean
# Remove remaining files/folders ignored by git as defined in .gitignore (-X)
# but keeping a local opam switch and other dependencies built.
git clean -dfXq --exclude=\!deps/** --exclude=\!_opam/** --exclude=\!_esy/** --exclude=\!vcpkg_installed
@if [ -d .git ]; \
then \
git clean -dfXq --exclude=\!deps/** --exclude=\!_opam/** --exclude=\!_esy/** --exclude=\!vcpkg_installed; \
fi

# Clean up libff installation
cleanall: clean
Expand Down
21 changes: 15 additions & 6 deletions docker/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -100,10 +100,9 @@ RUN apt update -y \

ENV VCPKG_BINARY_SOURCES="default,readwrite;x-gcs,gs://vcpkg/ubuntu/22.04/x64-linux-dynamic/${VCPKG_COMMIT_OR_TAG}/,read"

# Make sure vcpkg installs brings in the dependencies
RUN --mount=type=cache,target=/root/.cache/vcpkg/ ${VCPKG_ROOT}/vcpkg install --triplet=x64-linux-dynamic

RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10

ENV PKG_CONFIG_PATH="${SOURCE_DIR}/vcpkg_installed/x64-linux-dynamic/lib/pkgconfig"

RUN make opamdep-ci \
Expand All @@ -115,11 +114,21 @@ ARG BUILD_DIR="${SOURCE_DIR}/_build/default"
ARG VCPKG_INSTALL_LIB_DIR="${BUILD_DIR}/vcpkg_installed/x64-linux-dynamic/lib"

RUN mkdir -p ${VCPKG_INSTALL_LIB_DIR} \
&& ldd ${BUILD_DIR}/src/runners/*.exe | grep vcpkg_installed | gawk '{print $3}' | xargs -I{} cp {} ${VCPKG_INSTALL_LIB_DIR} \
&& rm -rf vcpkg_installed \
&& ldd ${BUILD_DIR}/src/runners/*.exe | grep vcpkg_installed | gawk '{print $3}' | xargs -I{} cp {} ${VCPKG_INSTALL_LIB_DIR}

FROM builder AS test_runner

COPY easyrun.sh .
ENV VCPKG_ROOT=/vcpkg
RUN apt update -y && apt install -y sudo
RUN ./scripts/install_shellcheck_ubuntu.sh

FROM builder AS cleanup_vcpkg

RUN rm -rf vcpkg_installed \
&& ln -s ${BUILD_DIR}/vcpkg_installed vcpkg_installed

FROM ubuntu:22.04
FROM ubuntu:22.04 AS base

RUN apt-get update -y \
&& apt-get install -y build-essential \
Expand All @@ -128,5 +137,5 @@ RUN apt-get update -y \

ARG SOURCE_DIR="/scilla/${MAJOR_VERSION}"

COPY --from=builder ${SOURCE_DIR} ${SOURCE_DIR}
COPY --from=cleanup_vcpkg ${SOURCE_DIR} ${SOURCE_DIR}

17 changes: 17 additions & 0 deletions docker/Dockerfile.test
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
ARG ACCOUNT_ID

FROM ${ACCOUNT_ID}.dkr.ecr.us-west-2.amazonaws.com/scilla:429e2f9

ENV VCPKG_ROOT="/vcpkg"
ENV SCILLA_REPO_ROOT="/scilla/0"

WORKDIR /scilla/0/
COPY . /scilla/0/

RUN apt update \
&& apt install -y sudo

RUN eval $(opam env) \
&& LD_LIBRARY_PATH=/scilla/0/vcpkg_installed/x64-linux-dynamic/lib opam install reason.3.8.2 --yes

RUN ./scripts/install_shellcheck_ubuntu.sh
Loading