unit-and-integration-tests-unified.yml

# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: "Unit & Integration tests CI"
on:
  push:
    paths-ignore:
      - '**/*.md'
      - 'dev/**'
      - 'distribution/bin/**'
      - 'docs/**'
      - 'examples/**/jupyter-notebooks/**'
      - 'web-console/**'
      - 'website/**'
    branches:
      - master
      - '[0-9]+.[0-9]+.[0-9]+' # release branches
      - '[0-9]+.[0-9]+.[0-9]+-[A-Za-z0-9]+' # release branches
  pull_request:
    paths-ignore:
      - '**/*.md'
      - 'dev/**'
      - 'distribution/bin/**'
      - 'docs/**'
      - 'examples/**/jupyter-notebooks/**'
      - 'web-console/**'
      - 'website/**'
    branches:
      - master
      - '[0-9]+.[0-9]+.[0-9]+' # release branches
      - '[0-9]+.[0-9]+.[0-9]+-[A-Za-z0-9]+' # release branches

concurrency:
  group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.run_id }}' # group workflows only on pull_requests and not on branch commits
  cancel-in-progress: true

env:
  MYSQL_DRIVER_CLASSNAME: com.mysql.jdbc.Driver # Used to set druid config in docker image for revised ITs
  SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5

jobs:
  set-env-var:
    name: Set env var
    runs-on: ubuntu-latest
    outputs:
      DRUID_PREVIOUS_IT_IMAGE_NAME: ${{ steps.image_name.outputs.image_name }}
      BACKWARD_COMPATIBILITY_IT_ENABLED: ${{ steps.it_enabled.outputs.enabled }}
      DRUID_PREVIOUS_VERSION: ${{ env.DRUID_PREVIOUS_VERSION }}
      DRUID_PREVIOUS_VERSION_DOWNLOAD_URL: ${{ env.DRUID_PREVIOUS_VERSION_DOWNLOAD_URL }}
    steps:
      - name: Set image name env var
        id: image_name
        run: |
          echo "::set-output name=image_name::org.apache.druid.integration-tests/test:${{ env.DRUID_PREVIOUS_VERSION }}"
      - name: Set env for enabling backward compatibility it
        id: it_enabled
        run: |
          if [ -n "${{ env.DRUID_PREVIOUS_VERSION }}" ]; then
            echo "::set-output name=enabled::true"
          else
            echo "::set-output name=enabled::false"
          fi

  build:
    needs: set-env-var
    name: "build (jdk${{ matrix.jdk }})"
    strategy:
      fail-fast: false
      matrix:
        jdk: [ '11', '17', '21' ]
    runs-on: ubuntu-latest
    steps:
      - name: Checkout branch
        uses: actions/checkout@v4

      # skip the "cache: maven" step from setup-java. We explicitly use a
      # different cache key since we cannot reuse it across commits.
      - uses: actions/setup-java@v4
        with:
          distribution: 'zulu'
          java-version: ${{ matrix.jdk }}

      # the build step produces SNAPSHOT artifacts into the local maven repository,
      # we include github.sha in the cache key to make it specific to that build/jdk
      - name: Cache Maven m2 repository
        id: maven
        uses: actions/cache@v4
        with:
          path: ~/.m2/repository
          key: maven-${{ runner.os }}-${{ matrix.jdk }}-${{ github.sha }}
          restore-keys: setup-java-Linux-maven-${{ hashFiles('**/pom.xml') }}

      - name: Cache targets
        id: target
        uses: actions/cache@v4
        with:
          path: |
            ./**/target
          key: maven-${{ runner.os }}-${{ matrix.jdk }}-targets-${{ github.sha }}

      - name: Cache image
        id: docker_container
        uses: actions/cache@v4
        with:
          key: druid-container-jdk${{ matrix.jdk }}.tar.gz-${{ github.sha }}
          path: |
            ./druid-container-jdk${{ matrix.jdk }}.tar.gz
            ./integration-tests-ex/image/target/env.sh

      - name: Cache previous version image
        id: docker_container_previous_version
        uses: actions/cache@v4
        with:
          key: druid-container-jdk${{ matrix.jdk }}-version${{ needs.set-env-var.outputs.DRUID_PREVIOUS_VERSION }}.tar.gz-${{ github.sha }}
          path: |
            ./druid-container-jdk${{ matrix.jdk }}-version${{ needs.set-env-var.outputs.DRUID_PREVIOUS_VERSION }}.tar.gz

      - name: Maven build
        id: maven_build
        run: |
          ./it.sh ci

      - name: Container build
        env:
          BACKWARD_COMPATIBILITY_IT_ENABLED: ${{ needs.set-env-var.outputs.BACKWARD_COMPATIBILITY_IT_ENABLED }}
          DRUID_PREVIOUS_VERSION: ${{ needs.set-env-var.outputs.DRUID_PREVIOUS_VERSION }}
          DRUID_PREVIOUS_VERSION_DOWNLOAD_URL: ${{ needs.set-env-var.outputs.DRUID_PREVIOUS_VERSION_DOWNLOAD_URL }}
          DRUID_PREVIOUS_IT_IMAGE_NAME: ${{ needs.set-env-var.outputs.DRUID_PREVIOUS_IT_IMAGE_NAME }}
        run: |
          ./it.sh image
          source ./integration-tests-ex/image/target/env.sh
          docker tag $DRUID_IT_IMAGE_NAME $DRUID_IT_IMAGE_NAME-jdk${{ matrix.jdk }}

      - name: Save docker container to archive
        run: |
          source ./integration-tests-ex/image/target/env.sh
          echo $DRUID_IT_IMAGE_NAME
          docker save "$DRUID_IT_IMAGE_NAME" | gzip > druid-container-jdk${{ matrix.jdk }}.tar.gz

      - name: Save previous version docker image
        if: ${{ needs.set-env-var.outputs.BACKWARD_COMPATIBILITY_IT_ENABLED == 'true' }}
        run: |
          docker tag ${{ needs.set-env-var.outputs.DRUID_PREVIOUS_IT_IMAGE_NAME }} ${{ needs.set-env-var.outputs.DRUID_PREVIOUS_IT_IMAGE_NAME }}-jdk${{ matrix.jdk }}-version${{ needs.set-env-var.outputs.DRUID_PREVIOUS_VERSION }}
          echo ${{ needs.set-env-var.outputs.DRUID_PREVIOUS_IT_IMAGE_NAME }}
          docker save "${{ needs.set-env-var.outputs.DRUID_PREVIOUS_IT_IMAGE_NAME }}" | gzip > druid-container-jdk${{ matrix.jdk }}-version${{ needs.set-env-var.outputs.DRUID_PREVIOUS_VERSION }}.tar.gz    

  # check if it is a PR and if it is approved. For approved PRs, we do not want to run tests sequentially
  check-approval:
    runs-on: ubuntu-latest
    outputs:
      approved: ${{ steps.check.outputs.isApproved }}
    steps:
      - name: Check if Triggered by a PR
        id: determine-trigger
        run: |
          if [ "${{ github.event_name }}" == "pull_request" ]; then
            echo "isPR=true" >> $GITHUB_ENV
          else
            echo "isPR=false" >> $GITHUB_ENV
          fi

      - name: Check PR Approval (if applicable)
        id: check
        if: ${{ env.isPR == 'true' }}
        uses: actions/github-script@v7
        with:
          script: |
            const reviews = await github.rest.pulls.listReviews({
              owner: context.repo.owner,
              repo: context.repo.repo,
              pull_number: context.payload.pull_request.number
            });
            const approved = reviews.data.some(review => review.state === 'APPROVED');
            core.setOutput("isApproved", approved);

      - name: Default to Approved for Branch
        id: default-check
        if: ${{ env.isPR == 'false' }}
        run: echo "::set-output name=isApproved::true"

  unit-tests-unapproved:
    name: "unit tests - PR unapproved - (jdk17)"
    uses: ./.github/workflows/ci.yml
    needs: [check-approval]
    if: ${{ needs.check-approval.outputs.approved != 'true' }}
    with:
      jdk: 17

  unit-tests-approved:
    name: "unit tests - PR approved - (jdk17)"
    uses: ./.github/workflows/ci.yml
    needs: [check-approval]
    if: ${{ needs.check-approval.outputs.approved == 'true' }}
    with:
      jdk: 17

  standard-its:
    needs: [build, unit-tests-unapproved, check-approval]
    if: ${{ always() && (needs.check-approval.outputs.approved == 'true' || needs.unit-tests-unapproved.result == 'success') }}
    uses: ./.github/workflows/standard-its.yml

  revised-its:
    needs: [build, unit-tests-unapproved, check-approval]
    if: ${{ always() && (needs.check-approval.outputs.approved == 'true' || needs.unit-tests-unapproved.result == 'success') }}
    uses: ./.github/workflows/revised-its.yml
    with:
      BACKWARD_COMPATIBILITY_IT_ENABLED: ${{ needs.set-env-var.outputs.BACKWARD_COMPATIBILITY_IT_ENABLED }}
      DRUID_PREVIOUS_VERSION: ${{ needs.set-env-var.outputs.DRUID_PREVIOUS_VERSION }}
      DRUID_PREVIOUS_VERSION_DOWNLOAD_URL: ${{ needs.set-env-var.outputs.DRUID_PREVIOUS_VERSION_DOWNLOAD_URL }}
      DRUID_PREVIOUS_IT_IMAGE_NAME: ${{ needs.set-env-var.outputs.DRUID_PREVIOUS_IT_IMAGE_NAME }}

  actions-timeline:
    needs: [build, unit-tests-approved, unit-tests-unapproved, revised-its, standard-its]
    runs-on: ubuntu-latest
    if: ${{ always() }}
    steps:
      - uses: Kesin11/actions-timeline@427ee2cf860166e404d0d69b4f2b24012bb7af4f