From 8c76169716ce036a4ae5d13cac7476cf298eaab1 Mon Sep 17 00:00:00 2001 From: Nariman Abdullin Date: Fri, 22 Dec 2023 14:06:42 +0300 Subject: [PATCH] Support checking git changes (#2891) - support detecting changed-files in build_and_test.yml --- .github/workflows/build_and_test.yml | 209 +++++++++++++++--- .github/workflows/build_and_test_reusable.yml | 15 +- 2 files changed, 198 insertions(+), 26 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 61945634e7..b401d4200b 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -14,57 +14,216 @@ concurrency: cancel-in-progress: true jobs: + calculate_build_flags: + name: Detect build flags for modules + runs-on: ubuntu-latest + outputs: + api-gateway: ${{ steps.calculate-dependencies.outputs.api-gateway }} + authentication-service: ${{ steps.calculate-dependencies.outputs.authentication-service }} + save-agent: ${{ steps.calculate-dependencies.outputs.save-agent }} + save-api: ${{ steps.calculate-dependencies.outputs.save-api }} + save-api-cli: ${{ steps.calculate-dependencies.outputs.save-api-cli }} + save-backend: ${{ steps.calculate-dependencies.outputs.save-backend }} + save-cloud-common: ${{ steps.calculate-dependencies.outputs.save-cloud-common }} + save-cosv: ${{ steps.calculate-dependencies.outputs.save-cosv }} + save-cosv-frontend: ${{ steps.calculate-dependencies.outputs.save-cosv-frontend }} + save-demo: ${{ steps.calculate-dependencies.outputs.save-demo }} + save-demo-agent: ${{ steps.calculate-dependencies.outputs.save-demo-agent }} + save-demo-cpg: ${{ steps.calculate-dependencies.outputs.save-demo-cpg }} + save-frontend: ${{ steps.calculate-dependencies.outputs.save-frontend }} + save-frontend-common: ${{ steps.calculate-dependencies.outputs.save-frontend-common }} + save-orchestrator: ${{ steps.calculate-dependencies.outputs.save-orchestrator }} + save-orchestrator-common: ${{ steps.calculate-dependencies.outputs.save-orchestrator-common }} + save-preprocessor: ${{ steps.calculate-dependencies.outputs.save-preprocessor }} + save-sandbox: ${{ steps.calculate-dependencies.outputs.save-sandbox }} + test-analysis-core: ${{ steps.calculate-dependencies.outputs.test-analysis-core }} + test-utils: ${{ steps.calculate-dependencies.outputs.test-utils }} + steps: + - id: git-changed-files + uses: tj-actions/changed-files@v40 + with: + files_yaml: | + root_gradle: + - '*.gradle.kts' + - gradle.properties + - gradle/** + - buildSrc/** + api-gateway: + - api-gateway/** + authentication-service: + - authentication-service/** + save-agent: + - save-agent/** + save-api: + - save-api/** + save-api-cli: + - save-api-cli/** + save-backend: + - save-backend/** + save-cloud-common: + - save-cloud-common/** + save-cosv: + - save-cosv/** + save-cosv-frontend: + - save-cosv-frontend/** + save-demo: + - save-demo/** + save-demo-agent: + - save-demo-agent/** + save-demo-cpg: + - save-demo-cpg/** + save-frontend: + - save-frontend/** + save-frontend-common: + - save-frontend-common/** + save-orchestrator: + - save-orchestrator/** + save-orchestrator-common: + - save-orchestrator-common/** + save-preprocessor: + - save-preprocessor/** + save-sandbox: + - save-sandbox/** + test-analysis-core: + - test-analysis-core/** + test-utils: + - test-utils/** + - id: calculate-dependencies + run: | + set -x + all=$(( ${{ steps.git-changed-files.outputs.root_gradle_all_changed_files_count }} )) + save_cloud_common=$(( $all + ${{ steps.git-changed-files.outputs.save-cloud-common_all_changed_files_count }} )) + save_frontend_common=$(( $save_cloud_common + ${{ steps.git-changed-files.outputs.save-frontend-common_all_changed_files_count }} )) + save_orchestrator_common=$(( $save_cloud_common + ${{ steps.git-changed-files.outputs.save-orchestrator-common_all_changed_files_count }} )) + + authentication_service=$(( $save_cloud_common + ${{ steps.git-changed-files.outputs.authentication-service_all_changed_files_count }} )) + save_api=$(( $save_cloud_common + ${{ steps.git-changed-files.outputs.save-api_all_changed_files_count }} )) + save_cosv=$(( $save_cloud_common + ${{ steps.git-changed-files.outputs.save-cosv_all_changed_files_count }} )) + test_analysis_core=$(( $save_cloud_common + ${{ steps.git-changed-files.outputs.test-analysis-core_all_changed_files_count }} )) + test_utils=$(( ${{ steps.git-changed-files.outputs.test-utils_all_changed_files_count }} )) + + api_gateway=$(( $save_cloud_common + ${{ steps.git-changed-files.outputs.api-gateway_all_changed_files_count }} )) + save_agent=$(( $save_cloud_common + ${{ steps.git-changed-files.outputs.save-agent_all_changed_files_count }} )) + save_api_cli=$(( $save_cloud_common + $save_api + ${{ steps.git-changed-files.outputs.save-api-cli_all_changed_files_count }} )) + save_backend=$(( $save_cloud_common + $authentication_service + $test_analysis_core + $save_cosv + $test_utils + ${{ steps.git-changed-files.outputs.save-backend_all_changed_files_count }} )) + save_cosv_frontend=$(( $save_cloud_common + $save_frontend_common + ${{ steps.git-changed-files.outputs.save-cosv-frontend_all_changed_files_count }} )) + save_demo=$(( $save_cloud_common + ${{ steps.git-changed-files.outputs.save-demo_all_changed_files_count }} )) + save_demo_agent=$(( $save_cloud_common + ${{ steps.git-changed-files.outputs.save-demo-agent_all_changed_files_count }} )) + save_demo_cpg=$(( $save_cloud_common + ${{ steps.git-changed-files.outputs.save-demo-cpg_all_changed_files_count }} )) + save_frontend=$(( $save_cloud_common + $save_frontend_common + ${{ steps.git-changed-files.outputs.save-frontend_all_changed_files_count }} )) + save_orchestrator=$(( $save_cloud_common + $save_orchestrator_common + $test_utils + ${{ steps.git-changed-files.outputs.save-orchestrator_all_changed_files_count }} )) + save_preprocessor=$(( $save_cloud_common + $test_utils + ${{ steps.git-changed-files.outputs.save-preprocessor_all_changed_files_count }} )) + save_sandbox=$(( $save_cloud_common + $authentication_service + $test_utils + ${{ steps.git-changed-files.outputs.save-sandbox_all_changed_files_count }} )) + + echo "api-gateway=$api_gateway" >> "$GITHUB_OUTPUT" + echo "authentication-service=$authentication_service" >> "$GITHUB_OUTPUT" + echo "save-agent=$save_agent" >> "$GITHUB_OUTPUT" + echo "save-api=$save_api" >> "$GITHUB_OUTPUT" + echo "save-api-cli=$save_api_cli" >> "$GITHUB_OUTPUT" + echo "save-backend=$save_backend" >> "$GITHUB_OUTPUT" + echo "save-cloud-common=$save_cloud_common" >> "$GITHUB_OUTPUT" + echo "save-cosv=$save_cosv" >> "$GITHUB_OUTPUT" + echo "save-cosv-frontend=$save_cosv_frontend" >> "$GITHUB_OUTPUT" + echo "save-demo=$save_demo" >> "$GITHUB_OUTPUT" + echo "save-demo-agent=$save_demo_agent" >> "$GITHUB_OUTPUT" + echo "save-demo-cpg=$save_demo_cpg" >> "$GITHUB_OUTPUT" + echo "save-frontend=$save_frontend" >> "$GITHUB_OUTPUT" + echo "save-frontend-common=$save_frontend_common" >> "$GITHUB_OUTPUT" + echo "save-orchestrator=$save_orchestrator" >> "$GITHUB_OUTPUT" + echo "save-orchestrator-common=$save_orchestrator_common" >> "$GITHUB_OUTPUT" + echo "save-preprocessor=$save_preprocessor" >> "$GITHUB_OUTPUT" + echo "save-sandbox=$save_sandbox" >> "$GITHUB_OUTPUT" + echo "test-analysis-core=$test_analysis_core" >> "$GITHUB_OUTPUT" + echo "test-utils=$test_utils" >> "$GITHUB_OUTPUT" build_save-cloud-common: - name: Build and test save-cloud-common + name: 'Build and test (save-cloud-common)' + needs: [ calculate_build_flags ] uses: ./.github/workflows/build_and_test_reusable.yml with: + do-build: ${{ github.event_name == 'push' || needs.calculate_build_flags.outputs.save-cloud-common > 0 }} module: save-cloud-common gradle-cache-read-only: ${{ github.ref != 'refs/heads/master' && github.event_name != 'pull_request' }} build_save-orchestrator-common: - name: Build and test save-orchestrator-common - needs: [ build_save-cloud-common ] + name: 'Build and test (save-orchestrator-common)' + needs: [ calculate_build_flags, build_save-cloud-common ] uses: ./.github/workflows/build_and_test_reusable.yml with: + do-build: ${{ github.event_name == 'push' || needs.calculate_build_flags.outputs.save-orchestrator-common > 0 }} module: save-orchestrator-common gradle-cache-read-only: ${{ github.ref != 'refs/heads/master' && github.event_name != 'pull_request' }} - build_save-agent: - name: Build and test save-agent - needs: [ build_save-orchestrator-common ] + build_save-frontend-common: + name: 'Build and test (save-frontend-common)' + needs: [ calculate_build_flags, build_save-orchestrator-common ] uses: ./.github/workflows/build_and_test_reusable.yml with: - module: save-agent - gradle-cache-read-only: ${{ github.ref != 'refs/heads/master' && github.event_name != 'pull_request' }} - build_save-demo-agent: - name: Build and test save-demo-agent - needs: [ build_save-agent ] - uses: ./.github/workflows/build_and_test_reusable.yml - with: - module: save-demo-agent + do-build: ${{ github.event_name == 'push' || needs.calculate_build_flags.outputs.save-frontend-common > 0 }} + module: save-frontend-common gradle-cache-read-only: ${{ github.ref != 'refs/heads/master' && github.event_name != 'pull_request' }} build_all: - name: Build and test left modules - needs: [ build_save-demo-agent ] + name: 'Build and test' + needs: [ calculate_build_flags, build_save-frontend-common ] strategy: fail-fast: false matrix: module: [ 'api-gateway', - 'save-backend', - 'save-orchestrator', - 'save-frontend', - 'save-cosv-frontend', - 'save-preprocessor', - 'test-utils', + 'authentication-service', + 'save-agent', 'save-api', 'save-api-cli', - 'save-sandbox', - 'authentication-service', + 'save-backend', + 'save-cosv', + 'save-cosv-frontend', 'save-demo', + 'save-demo-agent', 'save-demo-cpg', - 'test-analysis-core' + 'save-frontend', + 'save-orchestrator', + 'save-preprocessor', + 'save-sandbox', + 'test-analysis-core', + 'test-utils' ] + include: + - module: api-gateway + do-build: ${{ github.event_name == 'push' || needs.calculate_build_flags.outputs.api-gateway > 0 }} + - module: authentication-service + do-build: ${{ github.event_name == 'push' || needs.calculate_build_flags.outputs.authentication-service > 0 }} + - module: save-agent + do-build: ${{ github.event_name == 'push' || needs.calculate_build_flags.outputs.save-agent > 0 }} + - module: save-api + do-build: ${{ github.event_name == 'push' || needs.calculate_build_flags.outputs.save-api > 0 }} + - module: save-api-cli + do-build: ${{ github.event_name == 'push' || needs.calculate_build_flags.outputs.save-api-cli > 0 }} + - module: save-demo-cpg + do-build: ${{ github.event_name == 'push' || needs.calculate_build_flags.outputs.save-demo-cpg > 0 }} + - module: save-backend + do-build: ${{ github.event_name == 'push' || needs.calculate_build_flags.outputs.save-backend > 0 }} + - module: save-cosv + do-build: ${{ github.event_name == 'push' || needs.calculate_build_flags.outputs.save-cosv > 0 }} + - module: save-cosv-frontend + do-build: ${{ github.event_name == 'push' || needs.calculate_build_flags.outputs.save-cosv-frontend > 0 }} + - module: save-demo + do-build: ${{ github.event_name == 'push' || needs.calculate_build_flags.outputs.save-demo > 0 }} + - module: save-demo-agent + do-build: ${{ github.event_name == 'push' || needs.calculate_build_flags.outputs.save-demo-agent > 0 }} + - module: save-demo-cpg + do-build: ${{ github.event_name == 'push' || needs.calculate_build_flags.outputs.save-demo-cpg > 0 }} + - module: save-frontend + do-build: ${{ github.event_name == 'push' || needs.calculate_build_flags.outputs.save-frontend > 0 }} + - module: save-orchestrator + do-build: ${{ github.event_name == 'push' || needs.calculate_build_flags.outputs.save-orchestrator > 0 }} + - module: save-preprocessor + do-build: ${{ github.event_name == 'push' || needs.calculate_build_flags.outputs.save-preprocessor > 0 }} + - module: save-sandbox + do-build: ${{ github.event_name == 'push' || needs.calculate_build_flags.outputs.save-sandbox > 0 }} + - module: test-analysis-core + do-build: ${{ github.event_name == 'push' || needs.calculate_build_flags.outputs.test-analysis-core > 0 }} + - module: test-utils + do-build: ${{ github.event_name == 'push' || needs.calculate_build_flags.outputs.test-utils > 0 }} uses: ./.github/workflows/build_and_test_reusable.yml with: + do-build: ${{ matrix.do-build }} module: ${{ matrix.module }} gradle-cache-read-only: ${{ github.ref != 'refs/heads/master' }} diff --git a/.github/workflows/build_and_test_reusable.yml b/.github/workflows/build_and_test_reusable.yml index 0624e04c4a..367a94651c 100644 --- a/.github/workflows/build_and_test_reusable.yml +++ b/.github/workflows/build_and_test_reusable.yml @@ -3,6 +3,9 @@ name: Reusable build and test on: workflow_call: inputs: + do-build: + required: true + type: boolean module: required: true type: string @@ -13,7 +16,8 @@ on: jobs: build_module: - name: Build ${{ inputs.module }} + name: Build + if: inputs.do-build runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -68,3 +72,12 @@ jobs: name: save-frontend path: 'save-frontend/build/libs/save-frontend-*-distribution.jar' retention-days: 1 + + build_module_skipped: + name: Build + if: ${{ !inputs.do-build }} + runs-on: ubuntu-latest + steps: + - name: Do nothing + run: | + echo "Build of ${{ inputs.module }} is skipped" \ No newline at end of file