Merge branch 'main' of github.com:clearlydefined/operations into move…

…-docs
yashkohli88 · Dec 17, 2024 · 6988c99 · 6988c99
2 parents 2753143 + a1cc2f5
commit 6988c99
Show file tree

Hide file tree

Showing 37 changed files with 10,571 additions and 135 deletions.
diff --git a/.github/workflows/app-build-and-deploy.yml b/.github/workflows/app-build-and-deploy.yml
@@ -85,7 +85,7 @@ jobs:
         uses: actions/[email protected]
         with:
           repository: 'clearlydefined/operations'
-          ref: 'v3.1.0'
+          ref: 'v3.2.0'
           path: 'operations'
       - name: Get version from package-lock.json
         id: get_version

diff --git a/.github/workflows/app-build-docker-image.yml b/.github/workflows/app-build-docker-image.yml
@@ -39,7 +39,7 @@ on:
 
     outputs:
       docker-image-name-with-tag: 
-        value: ${{ jobs.determine-image-name.outputs.docker-image-name-with-tag }}
+        value: ${{ jobs.build-image-names.outputs.docker-image-to-use }}
 
 jobs:
   check-deployable:
@@ -55,34 +55,39 @@ jobs:
     runs-on: ubuntu-latest
     outputs: 
       names-with-tags: ${{ env.DOCKER_IMAGE_TAGS }}
+      docker-image-to-use: ${{ env.DOCKER_IMAGE_TO_USE }}
     steps:
       - name: Checkout this repo
         uses: actions/[email protected]
         with:
           repository: 'clearlydefined/operations'
-          ref: '3.1.0'
+          ref: 'v3.2.0'
           path: 'operations'
       - name: Determine Image Name
         id: determine-image-name
         run: |
           echo "BUILD_ARGS=${{ inputs.build-args }}"
           script_log=$(./operations/scripts/app-workflows/determine-image-name.sh \
             "${{ github.event.repository.name  }}" \
-            "${{ inputs.deploy-env }}" \
-            "${{ inputs.application-version }}") || (echo "$script_log" && exit 1)
+            "${{ inputs.deploy-env }}") || (echo "$script_log" && exit 1)
           echo -e "---- script log\n$script_log\n----"; \
           image_name=$(echo "$script_log" | tail -n 1)
           echo "IMAGE_NAME=$image_name" >> $GITHUB_ENV
       - name: Add ghcr.io 
         id: add-ghcr
         run: |
-          echo "DOCKER_IMAGE_TAGS=ghcr.io/${{ github.repository_owner }}/${{ env.IMAGE_NAME }}" >> $GITHUB_ENV
+          ghcr_image_tags="ghcr.io/${{ github.repository_owner }}/${{ env.IMAGE_NAME }}:${{ inputs.application-version }}"
+          echo "DOCKER_IMAGE_TAGS=$ghcr_image_tags" >> $GITHUB_ENV
+          echo "DOCKER_IMAGE_TO_USE=$ghcr_image_tags" >> $GITHUB_ENV
       - name: add-dockerhub
         if: ${{ inputs.docker-hub-username != '' }}
         id: add-dockerhub
         run: |
-          echo "DOCKER_IMAGE_TAGS=${{ env.DOCKER_IMAGE_TAGS }},${{ inputs.docker-hub-username }}/${{ env.IMAGE_NAME }}" >> $GITHUB_ENV
-
+          docker_image_tag="${{ inputs.docker-hub-username }}/${{ env.IMAGE_NAME }}:${{ inputs.application-version }}"
+          latest_image_tag="${{ inputs.docker-hub-username }}/${{ env.IMAGE_NAME }}:latest"
+          echo "DOCKER_IMAGE_TAGS=${{ env.DOCKER_IMAGE_TAGS }}, $docker_image_tag, $latest_image_tag" >> $GITHUB_ENV
+          # use Docker Hub image if it is created by overwriting DOCKER_IMAGE_TO_USE
+          echo "DOCKER_IMAGE_TO_USE=$docker_image_tags" >> $GITHUB_ENV
 
   build-docker-image:
     name: Build Image

diff --git a/.github/workflows/app-is-deployable.yml b/.github/workflows/app-is-deployable.yml
@@ -26,7 +26,7 @@ jobs:
         uses: actions/[email protected]
         with:
           repository: 'clearlydefined/operations'
-          ref: 'v3.1.0'
+          ref: 'v3.2.0'
           path: 'operations'
       - id: confirm-dev
         shell: bash
@@ -47,7 +47,7 @@ jobs:
         uses: actions/[email protected]
         with:
           repository: 'clearlydefined/operations'
-          ref: '3.1.0'
+          ref: 'v3.2.0'
           path: 'operations'
 
       - name: Get organization ID

diff --git a/.github/workflows/cleanup-template.json b/.github/workflows/cleanup-template.json
@@ -0,0 +1,8 @@
+{
+    "$schema": "https://schema.management.azure.com/schemas/2019-04-01/deploymentTemplate.json#",
+    "contentVersion": "1.0.0.0",
+    "parameters": {},
+    "variables": {},
+    "resources": [],
+    "outputs": {}
+  }
diff --git a/.github/workflows/integration-test-custom.yml b/.github/workflows/integration-test-custom.yml
@@ -0,0 +1,260 @@
+name: Integration Test On Custom Infra
+
+on:
+  workflow_dispatch:
+    inputs:
+      baseFolderPath:
+        description: 'Base folder path for diffs'
+        required: true
+        default: 'diffs'
+      dynamicCoordinates:
+        description: 'Array of booleans for dynamic coordinates (e.g., [true, false])'
+        required: true
+        default: '[true, false]'
+      location:
+        description: 'Azure Region'
+        required: true
+        default: 'eastus'
+      vm_name:
+        description: 'VM Name'
+        required: true
+        default: 'integration-test-vm'
+      repo_refs:
+        description: 'Repo refs in format "clearlydefined/service:master,clearlydefined/crawler:master"'
+        required: true
+        default: 'clearlydefined/docker_dev_env_experiment:main,clearlydefined/service:master,clearlydefined/crawler:master,clearlydefined/website:master'
+
+
+permissions:
+  contents: read
+
+env:
+  REPOS: |
+    clearlydefined/service
+    clearlydefined/crawler
+    clearlydefined/website
+  COMPOSE_REPO: "clearlydefined/docker_dev_env_experiment"
+
+jobs:
+  unit-test:
+    runs-on: ubuntu-latest
+    # Add permissions for the GitHub token
+    permissions:
+      contents: read
+      packages: read
+    defaults:
+      run:
+        working-directory: ./tools/integration
+    steps:
+      - uses: actions/[email protected]
+
+      - uses: actions/[email protected]
+        with:
+          node-version: 18
+          cache: 'npm'
+          cache-dependency-path: './tools/integration/'
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Run tests on tools
+        run: npm test
+
+  deploy-and-run:
+    runs-on: ubuntu-latest
+    needs: unit-test
+    # Add permissions for the GitHub token
+    permissions:
+      contents: read
+      packages: read
+    outputs:
+      vm_ip: ${{ steps.create_vm.outputs.publicIpAddress }}
+    steps:
+      - uses: actions/[email protected]
+
+      - name: Azure Login
+        uses: azure/[email protected]
+        with:
+          creds: ${{ secrets.VM_INT_AZURE_CREDENTIALS }}
+
+      - name: Cleanup existing resources
+        id: cleanup
+        uses: azure/arm-deploy@v2
+        with:
+          scope: resourcegroup
+          subscriptionId: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+          resourceGroupName: integration-test-vm
+          template: ${{ github.workspace }}/.github/workflows/cleanup-template.json
+          deploymentMode: Complete
+          deploymentName: cleanup-deployment
+
+      - name: Wait for cleanup to complete
+        uses: azure/[email protected]
+        with:
+          inlineScript: |
+            echo "Waiting for cleanup deployment to complete..."
+            az deployment group wait \
+              --name cleanup-deployment \
+              --resource-group integration-test-vm \
+              --created
+
+      - name: Create VM with NSG
+        id: create_vm
+        uses: azure/arm-deploy@v2
+        with:
+          scope: resourcegroup
+          subscriptionId: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+          resourceGroupName: integration-test-vm
+          template: ${{ github.workspace }}/.github/workflows/vm-template.json
+          parameters: vmName="${{ github.event.inputs.vm_name }}" location="${{ github.event.inputs.location }}" sshPublicKey="${{ secrets.SSH_PUBLIC_KEY }}"
+
+      - name: Wait SSH to become available
+        uses: appleboy/ssh-action@master
+        with:
+          host: ${{ steps.create_vm.outputs.publicIpAddress }}
+          username: azureuser
+          key: ${{ secrets.SSH_PRIVATE_KEY }}
+          script: |
+            for i in `seq 1 12`; do
+              echo "Attempt $i: Waiting for SSH to become available"
+              if nc -zvw3 ${{ steps.create_vm.outputs.publicIpAddress }} 22; then
+                echo "SSH is available!"
+                exit 0
+              fi
+              sleep 10
+            done
+            echo "SSH is not available after 2 minutes; failing"
+            exit 1
+
+      - name: Install Docker
+        uses: appleboy/ssh-action@master
+        with:
+          host: ${{ steps.create_vm.outputs.publicIpAddress }}
+          username: azureuser
+          key: ${{ secrets.SSH_PRIVATE_KEY }}
+          script: |
+            curl -fsSL https://get.docker.com -o get-docker.sh
+            sudo sh get-docker.sh
+            sudo systemctl enable docker
+            sudo systemctl start docker
+            sudo usermod -aG docker $USER
+            sudo curl -L "https://github.com/docker/compose/releases/download/v2.20.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
+            sudo chmod +x /usr/local/bin/docker-compose
+
+      - name: Clone Repos and Setup Environment
+        uses: appleboy/ssh-action@master
+        env:
+          GITHUB_TOKEN: ${{ secrets.IT_GH_ACCESS_TOKEN }}
+          REPO_REFS: ${{ inputs.repo_refs }}
+        with:
+          host: ${{ steps.create_vm.outputs.publicIpAddress }}
+          username: azureuser
+          key: ${{ secrets.SSH_PRIVATE_KEY }}
+          envs: GITHUB_TOKEN,REPO_REFS
+          script: |
+            # Function to get ref for a repo
+            get_ref() {
+              echo "$REPO_REFS" | tr ',' '\n' | grep "^$1:" | cut -d':' -f2 || echo "master"
+            }
+            
+            # First clone the compose repo
+            COMPOSE_REPO="${{ env.COMPOSE_REPO }}"
+            COMPOSE_REF=$(get_ref "$COMPOSE_REPO")
+            echo "Cloning compose repo https://github.com/${COMPOSE_REPO}.git branch/ref: ${COMPOSE_REF}"
+            git clone "https://github.com/${COMPOSE_REPO}.git" -b "$COMPOSE_REF"
+            
+            # Navigate to compose repo
+            REPO_NAME=$(echo ${COMPOSE_REPO} | cut -d'/' -f2)
+            cd $REPO_NAME
+            
+            # Clone other repositories inside the compose repo directory
+            echo "${{ env.REPOS }}" | while read repo; do
+              if [ ! -z "$repo" ] && [ "$repo" != "$COMPOSE_REPO" ]; then
+                REF=$(get_ref "$repo")
+                echo "Cloning https://github.com/${repo}.git branch/ref: ${REF}"
+                git clone "https://github.com/${repo}.git" -b "$REF"
+              fi
+            done
+            
+            # Copy sample_env to .env and replace tokens
+            cp sample_env .env
+            
+            # Replace both GitHub tokens with the provided token
+            sed -i "s/^CRAWLER_GITHUB_TOKEN=.*/CRAWLER_GITHUB_TOKEN=${GITHUB_TOKEN}/" .env
+            sed -i "s/^CURATION_GITHUB_TOKEN=.*/CURATION_GITHUB_TOKEN=${GITHUB_TOKEN}/" .env
+            echo "" >> .env && echo "CRAWLER_SCANCODE_PARALLELISM=14" >> .env
+            
+            # Start containers
+            sudo docker-compose up -d
+
+  integration-test:
+    runs-on: ubuntu-latest
+    needs: deploy-and-run
+    env:
+      DEV_API_BASE_URL: http://${{ needs.deploy-and-run.outputs.vm_ip }}:4000
+    strategy:
+      fail-fast: false
+      matrix:
+        dynamicCoordinates: ${{ fromJson(github.event.inputs.dynamicCoordinates) }}
+    defaults:
+      run:
+        working-directory: ./tools/integration
+    steps:
+      - uses: actions/[email protected]
+
+      - uses: actions/[email protected]
+        with:
+          node-version: 18
+          cache: 'npm'
+          cache-dependency-path: './tools/integration/'
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Trigger harvest and verify completion
+        run: DEV_API_BASE_URL=${{ env.DEV_API_BASE_URL }} DYNAMIC_COORDINATES=${{ matrix.dynamicCoordinates }} npm run e2e-test-harvest
+
+      - name: Verify definition tests
+        id: verify-service-functions
+        continue-on-error: true
+        run: DEV_API_BASE_URL=${{ env.DEV_API_BASE_URL }} DYNAMIC_COORDINATES=${{ matrix.dynamicCoordinates }} npm run e2e-test-definition
+
+      - name: Generate structured diffs
+        run: DEV_API_BASE_URL=${{ env.DEV_API_BASE_URL }} DYNAMIC_COORDINATES=${{ matrix.dynamicCoordinates }} npm run definitions-diff ${{ github.event.inputs.baseFolderPath }}
+
+      - name: Upload diffs artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: diffs-${{ matrix.dynamicCoordinates == 'true' && 'dynamic' || 'static' }}
+          path: ./tools/integration/${{ github.event.inputs.baseFolderPath }}
+
+      - name: Mark build status
+        if: steps.verify-service-functions.outcome == 'failure'
+        run: exit 1
+
+  cleanup:
+    runs-on: ubuntu-latest
+    needs: integration-test
+    if: ${{ always() }}
+    # Add permissions for the GitHub token
+    permissions:
+      contents: read
+      packages: read
+    steps:
+      - uses: actions/[email protected]
+
+      - name: Azure Login
+        uses: azure/[email protected]
+        with:
+          creds: ${{ secrets.VM_INT_AZURE_CREDENTIALS }}
+
+      - name: Cleanup existing resources
+        id: cleanup
+        uses: azure/arm-deploy@v2
+        with:
+          scope: resourcegroup
+          subscriptionId: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+          resourceGroupName: integration-test-vm
+          template: ${{ github.workspace }}/.github/workflows/cleanup-template.json
+          deploymentMode: Complete
+          deploymentName: cleanup-deployment