[WIP] Move CI from Gitlab to GitHub (#1889)

* [WIP] Move CI from Gitlab to GitHub * upd version * disable checkout * debug * more debug * try buildah * rm unused arg * back to docker * find * fix postinstall * change context * rm context * use other Docker * npm build * ls * use context * test push * rm debug * bump versions and dedupe * lint * try example job * usu ubuntu runner * comment nonroot * rm nonroot * mkdir * typo * fix local dir * run in container * fix to run in gha * allow networking from arc-runners * add more tests * fix set env * typo * upload logs * refactor * more debug on reruns * less debug * less debug * typo * add logs to ci * change logs names --------- Co-authored-by: Javier Viola <[email protected]>
paritytech · Dec 9, 2024 · c1887cf · c1887cf
1 parent 1950ef1
commit c1887cf
Show file tree

Hide file tree

Showing 6 changed files with 293 additions and 39 deletions.
diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml
@@ -0,0 +1,256 @@
+name: Zombienet Integration Tests
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+permissions: {}
+
+env:
+  IMAGE_NAME: paritypr/zombienet
+  VERSION: ${{ github.sha }}
+  RUN_IN_CONTAINER: 1
+  FF_DISABLE_UMASK_FOR_DOCKER_EXECUTOR: 1
+  GHA_CLUSTER_SERVER_ADDR: "https://kubernetes.default:443"
+
+jobs:
+  # build_image:
+  #   name: Build image
+  #   runs-on: ubuntu-latest
+  #   timeout-minutes: 30
+  #   steps:
+  #     - name: Check out the repo
+  #       uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v.4.2.0
+
+  #     - name: npm build
+  #       run: |
+  #         cd javascript
+  #         npm install
+  #         npm dedupe
+  #         npm run clean
+  #         npm run build
+
+  #     - name: Build Docker image
+  #       uses: docker/build-push-action@4f58ea79222b3b9dc2c8bbdd6debcef730109a75 # v6.9.0
+  #       with:
+  #         file: ./scripts/ci/docker/zombienet_injected.Dockerfile
+  #         context: .
+  #         push: false
+  #         tags: |
+  #           ${{ env.IMAGE_NAME }}:${{ env.VERSION }}
+  #           ${{ env.IMAGE_NAME }}:latest
+
+  build_push_image:
+    name: Build and Push Docker image to Docker Hub
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    # needs: [build_image]
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v.4.2.0
+
+      - name: npm build
+        run: |
+          cd javascript
+          npm install
+          npm dedupe
+          npm run clean
+          npm run build
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USER }}
+          password: ${{ secrets.DOCKERHUB_PASS }}
+
+      - name: Build Docker image
+        uses: docker/build-push-action@4f58ea79222b3b9dc2c8bbdd6debcef730109a75 # v6.9.0
+        with:
+          file: ./scripts/ci/docker/zombienet_injected.Dockerfile
+          context: .
+          push: true
+          tags: |
+            ${{ env.IMAGE_NAME }}:${{ env.VERSION }}
+            ${{ env.IMAGE_NAME }}:latest
+
+  set-variables:
+    name: Set variables
+    runs-on: ubuntu-latest
+    outputs:
+      VERSION: ${{ steps.version.outputs.IMAGE }}
+      DEBUG: ${{ steps.env.outputs.DEBUG }}
+      ZOMBIENET_INTEGRATION_TEST_IMAGE: ${{ steps.env.outputs.ZOMBIENET_INTEGRATION_TEST_IMAGE }}
+      COL_IMAGE: ${{ steps.env.outputs.COL_IMAGE }}
+      CUMULUS_TEST_IMAGE: ${{ steps.env.outputs.CUMULUS_TEST_IMAGE }}
+    steps:
+      - name: Define version
+        id: version
+        run: |
+          export IMAGE="${{ env.IMAGE_NAME }}:${{ env.VERSION }}"
+          echo "IMAGE=${IMAGE}" >> $GITHUB_OUTPUT
+          echo "set IMAGE=${IMAGE}"
+      - name: Define env
+        id: env
+        run: |
+          if [[ ${{ github.run_attempt }} -gt 1 ]]; then
+            # more debug in rerun
+            export DEBUG=zombie,zombie::js-helpers*,zombie::network-node,zombie::kube::client::logs
+          else
+            export DEBUG=zombie
+          fi;
+          echo "DEBUG=${DEBUG}" >> $GITHUB_OUTPUT
+          echo "set DEBUG=${DEBUG}"
+          export ZOMBIENET_INTEGRATION_TEST_IMAGE="docker.io/paritypr/polkadot-debug:master"
+          echo "ZOMBIENET_INTEGRATION_TEST_IMAGE=${ZOMBIENET_INTEGRATION_TEST_IMAGE}" >> $GITHUB_OUTPUT
+          echo "set ZOMBIENET_INTEGRATION_TEST_IMAGE=${ZOMBIENET_INTEGRATION_TEST_IMAGE}"
+          export COL_IMAGE="docker.io/paritypr/colander:master"
+          echo "COL_IMAGE=${COL_IMAGE}" >> $GITHUB_OUTPUT
+          echo "set COL_IMAGE=${COL_IMAGE}"
+          # TODO: add script to get latest from dockerhub
+          export CUMULUS_TEST_IMAGE="docker.io/paritypr/test-parachain:c90f9713b5bc73a9620b2e72b226b4d11e018190"
+          echo "CUMULUS_TEST_IMAGE=${CUMULUS_TEST_IMAGE}" >> $GITHUB_OUTPUT
+          echo "set CUMULUS_TEST_IMAGE=${CUMULUS_TEST_IMAGE}"
+
+  zombienet-smoke:
+    name: Zombienet Smoke
+    runs-on: zombienet-arc-runner
+    timeout-minutes: 30
+    container:
+      image: ${{ needs.set-variables.outputs.VERSION }}
+    needs: [build_push_image, set-variables]
+    env:
+      # LOCAL_DIR: "/builds/parity/mirrors/zombienet/tests"
+      DEBUG: ${{ needs.set-variables.outputs.DEBUG }}
+      ZOMBIENET_INTEGRATION_TEST_IMAGE: ${{ needs.set-variables.outputs.ZOMBIENET_INTEGRATION_TEST_IMAGE }}
+      COL_IMAGE: ${{ needs.set-variables.outputs.COL_IMAGE }}
+      CUMULUS_TEST_IMAGE: ${{ needs.set-variables.outputs.CUMULUS_TEST_IMAGE }}
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v.4.2.0
+
+      - name: smoke test
+        run: |
+          echo "Zombienet Tests"
+          echo "${ZOMBIENET_IMAGE}"
+          echo "${GH_DIR}"
+          echo "relay_image ${ZOMBIENET_INTEGRATION_TEST_IMAGE}"
+          echo "col_image ${COL_IMAGE}"
+
+          /home/nonroot/zombie-net/scripts/ci/run-test-local-env-manager.sh --local-dir="$(pwd)/tests" --test="0001-smoke.zndsl"
+
+      - name: upload logs
+        uses: actions/upload-artifact@v4
+        with:
+          name: zombienet-logs-smoke
+          path: |
+            /tmp/zombie*/logs/*
+
+  zombienet-scale-net-config:
+    name: Zombienet Scale net
+    runs-on: zombienet-arc-runner
+    timeout-minutes: 30
+    container:
+      image: ${{ needs.set-variables.outputs.VERSION }}
+    needs: [build_push_image, set-variables]
+    env:
+      # LOCAL_DIR: "/builds/parity/mirrors/zombienet/tests"
+      DEBUG: ${{ needs.set-variables.outputs.DEBUG }}
+      ZOMBIENET_INTEGRATION_TEST_IMAGE: ${{ needs.set-variables.outputs.ZOMBIENET_INTEGRATION_TEST_IMAGE }}
+      COL_IMAGE: ${{ needs.set-variables.outputs.COL_IMAGE }}
+      CUMULUS_TEST_IMAGE: ${{ needs.set-variables.outputs.CUMULUS_TEST_IMAGE }}
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v.4.2.0
+
+      - name: scale-net test
+        run: |
+          echo "Zombienet Scale Net"
+          echo "${ZOMBIENET_IMAGE}"
+          echo "${GH_DIR}"
+          echo "relay_image ${ZOMBIENET_INTEGRATION_TEST_IMAGE}"
+          echo "col_image ${COL_IMAGE}"
+
+          /home/nonroot/zombie-net/scripts/ci/run-test-local-env-manager.sh --local-dir="$(pwd)/tests/scale-net" --test="0001-scale-net.zndsl"
+
+      - name: upload logs
+        uses: actions/upload-artifact@v4
+        with:
+          name: zombienet-logs-scale-net
+          path: |
+            /tmp/zombie*/logs/*
+
+  zombienet-upgrade-node:
+    name: Zombienet Upgrade Node
+    runs-on: zombienet-arc-runner
+    timeout-minutes: 30
+    container:
+      image: ${{ needs.set-variables.outputs.VERSION }}
+    needs: [build_push_image, set-variables]
+    env:
+      # LOCAL_DIR: "/builds/parity/mirrors/zombienet/tests"
+      DEBUG: ${{ needs.set-variables.outputs.DEBUG }}
+      ZOMBIENET_INTEGRATION_TEST_IMAGE: ${{ needs.set-variables.outputs.ZOMBIENET_INTEGRATION_TEST_IMAGE }}
+      COL_IMAGE: ${{ needs.set-variables.outputs.COL_IMAGE }}
+      CUMULUS_TEST_IMAGE: ${{ needs.set-variables.outputs.CUMULUS_TEST_IMAGE }}
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v.4.2.0
+
+      - name: upgrade node test
+        run: |
+          echo "Zombienet Upgrade Node"
+          echo "${ZOMBIENET_IMAGE}"
+          echo "${GH_DIR}"
+          export COL_IMAGE="docker.io/paritypr/colander:4519"
+          echo "relay_image ${ZOMBIENET_INTEGRATION_TEST_IMAGE}"
+          echo "col_image ${COL_IMAGE}"
+
+          /home/nonroot/zombie-net/scripts/ci/run-test-local-env-manager.sh --local-dir="$(pwd)/tests/k8s" --test="0001-upgrade-node.zndsl"
+
+      - name: upload logs
+        uses: actions/upload-artifact@v4
+        with:
+          name: zombienet-logs-upgrade
+          path: |
+            /tmp/zombie*/logs/*
+
+  zombienet-chaos-delay:
+    name: Zombienet Chaos Delay
+    runs-on: zombienet-arc-runner
+    timeout-minutes: 30
+    container:
+      image: ${{ needs.set-variables.outputs.VERSION }}
+    needs: [build_push_image, set-variables]
+    env:
+      # LOCAL_DIR: "/builds/parity/mirrors/zombienet/tests"
+      DEBUG: ${{ needs.set-variables.outputs.DEBUG }}
+      ZOMBIENET_INTEGRATION_TEST_IMAGE: ${{ needs.set-variables.outputs.ZOMBIENET_INTEGRATION_TEST_IMAGE }}
+      COL_IMAGE: ${{ needs.set-variables.outputs.COL_IMAGE }}
+      CUMULUS_TEST_IMAGE: ${{ needs.set-variables.outputs.CUMULUS_TEST_IMAGE }}
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v.4.2.0
+
+      - name: chaos-delay test
+        run: |
+          echo "Zombienet Chaos Delay"
+          echo "${ZOMBIENET_IMAGE}"
+          echo "${GH_DIR}"
+          echo "relay_image ${ZOMBIENET_INTEGRATION_TEST_IMAGE}"
+          echo "col_image ${COL_IMAGE}"
+
+          /home/nonroot/zombie-net/scripts/ci/run-test-local-env-manager.sh --local-dir="$(pwd)/tests/chaos" --test="0001-delay.zndsl"
+
+      - name: upload logs
+        uses: actions/upload-artifact@v4
+        with:
+          name: zombienet-logs-chaos
+          path: |
+            /tmp/zombie*/logs/*
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -14,7 +14,6 @@ stages:
   - publish
   - zombienet
 
-
 variables:
   CI_IMAGE: "node:18"
   DOCKERFILE: scripts/ci/docker/zombienet_injected.Dockerfile
@@ -32,13 +31,13 @@ variables:
 cache:
   - key:
       files:
-      - javascript/package-lock.json
+        - javascript/package-lock.json
   - paths:
-    - javascript/packages/cli/dist
-    - javascript/packages/orchestrator/dist
-    - javascript/packages/utils/dist
-    - javascript/package.json
-    - javascript/package-lock.json
+      - javascript/packages/cli/dist
+      - javascript/packages/orchestrator/dist
+      - javascript/packages/utils/dist
+      - javascript/package.json
+      - javascript/package-lock.json
 
 .kubernetes-env: &kubernetes-env
   image: $CI_IMAGE
@@ -52,9 +51,9 @@ cache:
     - if: $CI_PIPELINE_SOURCE == "schedule"
     - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
     - changes:
-      - javascript/**/*
-      - tests/**/*
-      - crates/**/*
+        - javascript/**/*
+        - tests/**/*
+        - crates/**/*
 
 # run jobs always
 .common-refs: &common-refs
@@ -73,16 +72,6 @@ cache:
     - if: $CI_COMMIT_REF_NAME == "main"
     - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
 
-test:
-  stage: test
-  <<: *kubernetes-env
-  <<: *common-refs
-  script:
-    - cd javascript
-    - npm install
-    # - npm run test
-    - echo testme
-
 build:
   stage: build
   <<: *kubernetes-env
@@ -175,7 +164,6 @@ publish-docker-image-description:
   script:
     - cd / && sh entrypoint.sh
 
-
 .zombienet-common:
   before_script:
     - echo "Zombienet Tests Config"
@@ -288,4 +276,3 @@ chaos-delay:
     - /home/nonroot/zombie-net/scripts/ci/run-test-local-env-manager.sh
       --local-dir="${LOCAL_DIR}"
       --test="0001-delay.zndsl"
-
diff --git a/javascript/package.json b/javascript/package.json
@@ -24,7 +24,7 @@
     "package:macos:arm64": "npm run -w packages/cli package:macos:arm64",
     "zombie": "node ./packages/cli/dist/cli.js",
     "test": "npm run test --workspaces --if-present",
-    "postinstall": "if [[ -f .husky/post-install.js ]]; then cd .. && node javascript/.husky/post-install.js; fi"
+    "postinstall": "bash -c 'if [[ -f .husky/post-install.js ]]; then cd .. && node javascript/.husky/post-install.js; fi'"
   },
   "engines": {
     "node": ">=18"

diff --git a/javascript/packages/orchestrator/static-configs/namespace-network-policy.yaml b/javascript/packages/orchestrator/static-configs/namespace-network-policy.yaml
@@ -13,6 +13,7 @@ spec:
           values:
           - {{namespace}}
           - gitlab
+          - arc-runners
           - loki
           - tempo
           - monitoring
-Original file line number
+Diff line change
@@ Expand Up / @@ -13,6 +13,7 @@ spec: @@
               values:
               - {{namespace}}
               - gitlab
+              - arc-runners
               - loki
               - tempo
               - monitoring
@@ Expand Down @@