diff --git a/.azure-pipelines/bazel.yml b/.azure-pipelines/bazel.yml index 3c440cdef934..d68973ab0ee4 100644 --- a/.azure-pipelines/bazel.yml +++ b/.azure-pipelines/bazel.yml @@ -1,85 +1,85 @@ parameters: - - name: ciTarget - displayName: "CI target" - type: string - default: bazel.release - - name: artifactSuffix - displayName: "Suffix of artifact" - type: string - default: "" - - name: rbe - displayName: "Enable RBE" - type: boolean - default: true - - name: managedAgent - type: boolean - default: true - - name: bazelBuildExtraOptions - type: string - default: "--flaky_test_attempts=2" +- name: ciTarget + displayName: "CI target" + type: string + default: bazel.release +- name: artifactSuffix + displayName: "Suffix of artifact" + type: string + default: "" +- name: rbe + displayName: "Enable RBE" + type: boolean + default: true +- name: managedAgent + type: boolean + default: true +- name: bazelBuildExtraOptions + type: string + default: "--flaky_test_attempts=2" steps: - - task: Cache@2 - inputs: - key: '"${{ parameters.ciTarget }}" | ./WORKSPACE | **/*.bzl' - path: $(Build.StagingDirectory)/repository_cache - continueOnError: true +- task: Cache@2 + inputs: + key: '"${{ parameters.ciTarget }}" | ./WORKSPACE | **/*.bzl' + path: $(Build.StagingDirectory)/repository_cache + continueOnError: true - - bash: .azure-pipelines/cleanup.sh - displayName: "Removing tools from agent" - condition: ${{ parameters.managedAgent }} +- bash: .azure-pipelines/cleanup.sh + displayName: "Removing tools from agent" + condition: ${{ parameters.managedAgent }} - - bash: | - echo "disk space at beginning of build:" - df -h - displayName: "Check disk space at beginning" +- bash: | + echo "disk space at beginning of build:" + df -h + displayName: "Check disk space at beginning" - - bash: | - sudo mkdir -p /etc/docker - echo '{ - "ipv6": true, - "fixed-cidr-v6": "2001:db8:1::/64" - }' | sudo tee /etc/docker/daemon.json - sudo service docker restart - displayName: "Enable IPv6" - condition: ${{ parameters.managedAgent }} +- bash: | + sudo mkdir -p /etc/docker + echo '{ + "ipv6": true, + "fixed-cidr-v6": "2001:db8:1::/64" + }' | sudo tee /etc/docker/daemon.json + sudo service docker restart + displayName: "Enable IPv6" + condition: ${{ parameters.managedAgent }} - - script: ci/run_envoy_docker.sh 'ci/do_ci.sh ${{ parameters.ciTarget }}' - workingDirectory: $(Build.SourcesDirectory) - env: - ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) - SLACK_TOKEN: $(SLACK_TOKEN) - REPO_URI: $(Build.Repository.Uri) - BUILD_URI: $(Build.BuildUri) - ${{ if parameters.rbe }}: - ENVOY_RBE: "1" - BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --jobs=$(RbeJobs) ${{ parameters.bazelBuildExtraOptions }}" - BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com - BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance - GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) - ${{ if eq(parameters.rbe, false) }}: - BAZEL_BUILD_EXTRA_OPTIONS: "${{ parameters.bazelBuildExtraOptions }}" - BAZEL_REMOTE_CACHE: $(LocalBuildCache) +- script: ci/run_envoy_docker.sh 'ci/do_ci.sh ${{ parameters.ciTarget }}' + workingDirectory: $(Build.SourcesDirectory) + env: + ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) + SLACK_TOKEN: $(SLACK_TOKEN) + REPO_URI: $(Build.Repository.Uri) + BUILD_URI: $(Build.BuildUri) + ${{ if parameters.rbe }}: + ENVOY_RBE: "1" + BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --jobs=$(RbeJobs) ${{ parameters.bazelBuildExtraOptions }}" + BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com + BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance + GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) + ${{ if eq(parameters.rbe, false) }}: + BAZEL_BUILD_EXTRA_OPTIONS: "${{ parameters.bazelBuildExtraOptions }}" + BAZEL_REMOTE_CACHE: $(LocalBuildCache) - displayName: "Run CI script" + displayName: "Run CI script" - - bash: | - echo "disk space at end of build:" - df -h - # Cleanup offending files with unicode names - rm -rf $(Build.StagingDirectory)/tmp/*/*/external/go_sdk/test/fixedbugs - displayName: "Check disk space at end" - condition: always() +- bash: | + echo "disk space at end of build:" + df -h + # Cleanup offending files with unicode names + rm -rf $(Build.StagingDirectory)/tmp/*/*/external/go_sdk/test/fixedbugs + displayName: "Check disk space at end" + condition: always() - - task: PublishTestResults@2 - inputs: - testResultsFiles: "**/bazel-out/**/testlogs/**/test.xml" - testRunTitle: "${{ parameters.ciTarget }}" - searchFolder: $(Build.StagingDirectory)/tmp - condition: always() +- task: PublishTestResults@2 + inputs: + testResultsFiles: "**/bazel-out/**/testlogs/**/test.xml" + testRunTitle: "${{ parameters.ciTarget }}" + searchFolder: $(Build.StagingDirectory)/tmp + condition: always() - - task: PublishBuildArtifacts@1 - inputs: - pathtoPublish: "$(Build.StagingDirectory)/envoy" - artifactName: ${{ parameters.ciTarget }}${{ parameters.artifactSuffix }} - condition: always() +- task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: "$(Build.StagingDirectory)/envoy" + artifactName: ${{ parameters.ciTarget }}${{ parameters.artifactSuffix }} + condition: always() diff --git a/.azure-pipelines/cve_scan.yml b/.azure-pipelines/cve_scan.yml index 322adae2bb71..7c951df66f68 100644 --- a/.azure-pipelines/cve_scan.yml +++ b/.azure-pipelines/cve_scan.yml @@ -15,8 +15,8 @@ schedules: pool: vmImage: "ubuntu-18.04" steps: - - script: ci/run_envoy_docker.sh 'ci/do_ci.sh cve_scan' - workingDirectory: $(Build.SourcesDirectory) - env: - ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) - displayName: "Scan for CVEs in dependencies" +- script: ci/run_envoy_docker.sh 'ci/do_ci.sh cve_scan' + workingDirectory: $(Build.SourcesDirectory) + env: + ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) + displayName: "Scan for CVEs in dependencies" diff --git a/.azure-pipelines/pipelines.yml b/.azure-pipelines/pipelines.yml index bac100bfca96..e52b2f7f9406 100644 --- a/.azure-pipelines/pipelines.yml +++ b/.azure-pipelines/pipelines.yml @@ -1,540 +1,540 @@ trigger: branches: include: - - "main" - - "release/v*" + - "main" + - "release/v*" tags: include: - - "v*" + - "v*" # PR build config is manually overridden in Azure pipelines UI with different secrets pr: none stages: - - stage: precheck - jobs: - - job: format_pre - dependsOn: [] - pool: - vmImage: "ubuntu-18.04" - steps: - - script: ci/run_envoy_docker.sh 'ci/do_ci.sh format_pre' - workingDirectory: $(Build.SourcesDirectory) - env: - ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) - BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com - BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance - GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) - displayName: "Run format pre-checks" - - task: PublishBuildArtifacts@1 - inputs: - pathtoPublish: "$(Build.StagingDirectory)/fix_format_pre.diff" - artifactName: format - # not all have fixes so improve condition/handling - condition: failed() - - - job: tooling - dependsOn: [] - pool: - vmImage: "ubuntu-18.04" - steps: - - script: ci/run_envoy_docker.sh 'ci/do_ci.sh tooling' - workingDirectory: $(Build.SourcesDirectory) - env: - ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) - BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com - BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance - GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) - displayName: "Run tooling checks" - - - script: | - ci/run_envoy_docker.sh 'ci/upload_gcs_artifact.sh /source/generated/tooling tooling' - displayName: "Upload tooling coverage to GCS" - env: - ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) - GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) - GCS_ARTIFACT_BUCKET: $(GcsArtifactBucket) - - - job: format - dependsOn: ["format_pre"] - pool: - vmImage: "ubuntu-18.04" - steps: - - task: Cache@2 - inputs: - key: "format | ./WORKSPACE | **/*.bzl" - path: $(Build.StagingDirectory)/repository_cache - continueOnError: true - - - script: ci/run_envoy_docker.sh 'ci/check_and_fix_format.sh' - workingDirectory: $(Build.SourcesDirectory) - env: - ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) - BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com - BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance - GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) - displayName: "Run check format scripts" - - - task: PublishBuildArtifacts@1 - inputs: - pathtoPublish: "$(Build.StagingDirectory)/fix_format.diff" - artifactName: format - condition: failed() - - - job: docs - dependsOn: [] # this removes the implicit dependency on previous stage and causes this to run in parallel. - condition: ne(variables['PostSubmit'], true) - pool: - vmImage: "ubuntu-18.04" - steps: - - task: Cache@2 - inputs: - key: "docs | ./WORKSPACE | **/*.bzl" - path: $(Build.StagingDirectory)/repository_cache - continueOnError: true - - - script: ci/run_envoy_docker.sh 'ci/do_ci.sh docs' - workingDirectory: $(Build.SourcesDirectory) - env: - AZP_BRANCH: $(Build.SourceBranch) - ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) - BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com - BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance - GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) - displayName: "Generate docs" - - - script: | - ci/run_envoy_docker.sh 'ci/upload_gcs_artifact.sh /source/generated/docs docs' - displayName: "Upload Docs to GCS" - env: - ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) - GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) - GCS_ARTIFACT_BUCKET: $(GcsArtifactBucket) - - - job: dependencies - dependsOn: [] # this removes the implicit dependency on previous stage and causes this to run in parallel. - pool: - vmImage: "ubuntu-18.04" - steps: - - script: ci/run_envoy_docker.sh 'ci/do_ci.sh deps' - workingDirectory: $(Build.SourcesDirectory) - env: - ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) - BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com - BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance - GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) - GITHUB_TOKEN: $(GitHubPublicRepoOnlyAccessToken) - displayName: "Verify dependency information" - - - stage: sync - condition: and(succeeded(), eq(variables['PostSubmit'], true), ne(variables['NoSync'], true)) +- stage: precheck + jobs: + - job: format_pre dependsOn: [] - jobs: - - job: filter_example - dependsOn: [] - pool: - vmImage: "ubuntu-18.04" - steps: - - task: InstallSSHKey@0 - inputs: - hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==" - sshPublicKey: "$(FilterExamplePublicKey)" - sshPassphrase: "$(SshDeployKeyPassphrase)" - sshKeySecureFile: "$(FilterExamplePrivateKey)" - - - bash: ci/filter_example_mirror.sh - displayName: "Sync envoy-filter-example" - workingDirectory: $(Build.SourcesDirectory) - env: - AZP_BRANCH: $(Build.SourceBranch) - - - job: data_plane_api - dependsOn: [] - pool: - vmImage: "ubuntu-18.04" - steps: - - task: InstallSSHKey@0 - inputs: - hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==" - sshPublicKey: "$(DataPlaneApiPublicKey)" - sshPassphrase: "$(SshDeployKeyPassphrase)" - sshKeySecureFile: "$(DataPlaneApiPrivateKey)" - - - bash: ci/api_mirror.sh - displayName: "Sync data-plane-api" - workingDirectory: $(Build.SourcesDirectory) - env: - AZP_BRANCH: $(Build.SourceBranch) - - - job: go_control_plane - dependsOn: [] - pool: - vmImage: "ubuntu-18.04" - steps: - - task: InstallSSHKey@0 - inputs: - hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==" - sshPublicKey: "$(GoControlPlanePublicKey)" - sshPassphrase: "$(SshDeployKeyPassphrase)" - sshKeySecureFile: "$(GoControlPlanePrivateKey)" - - - bash: | - cp -a ~/.ssh $(Build.StagingDirectory)/ - ci/run_envoy_docker.sh 'ci/go_mirror.sh' - displayName: "Sync go-control-plane" - workingDirectory: $(Build.SourcesDirectory) - env: - ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) - BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com - BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance - GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) - AZP_BRANCH: $(Build.SourceBranch) - - - stage: linux_x64 - dependsOn: ["precheck"] - # For post-submit builds, continue even if precheck fails - condition: and(not(canceled()), or(succeeded(), eq(variables['PostSubmit'], true))) - jobs: - - job: release - timeoutInMinutes: 120 - pool: - vmImage: "ubuntu-18.04" - steps: - - template: bazel.yml - parameters: - ciTarget: bazel.release - - - stage: linux_arm64 - dependsOn: ["precheck"] - # For post-submit builds, continue even if precheck fails - condition: and(not(canceled()), or(succeeded(), eq(variables['PostSubmit'], true))) - jobs: - - job: release - timeoutInMinutes: 120 - pool: "arm-large" - steps: - - template: bazel.yml - parameters: - managedAgent: false - ciTarget: bazel.release - rbe: false - artifactSuffix: ".arm64" - bazelBuildExtraOptions: "--sandbox_base=/tmp/sandbox_base" - - - stage: check - dependsOn: ["linux_x64"] - jobs: - - job: bazel - displayName: "linux_x64" - dependsOn: [] - strategy: - maxParallel: 3 - matrix: - api: - CI_TARGET: "bazel.api" - gcc: - CI_TARGET: "bazel.gcc" - clang_tidy: - CI_TARGET: "bazel.clang_tidy" - asan: - CI_TARGET: "bazel.asan" - tsan: - CI_TARGET: "bazel.tsan" - compile_time_options: - CI_TARGET: "bazel.compile_time_options" - timeoutInMinutes: 120 - pool: - vmImage: "ubuntu-18.04" - steps: - - template: bazel.yml - parameters: - ciTarget: $(CI_TARGET) - - - job: coverage - displayName: "linux_x64" - dependsOn: [] - timeoutInMinutes: 120 - pool: "x64-large" - strategy: - maxParallel: 2 - matrix: - coverage: - CI_TARGET: "coverage" - fuzz_coverage: - CI_TARGET: "fuzz_coverage" - steps: - - template: bazel.yml - parameters: - managedAgent: false - ciTarget: bazel.$(CI_TARGET) - rbe: false - # /tmp/sandbox_base is a tmpfs in CI environment to optimize large I/O for coverage traces - bazelBuildExtraOptions: "--define=no_debug_info=1 --linkopt=-Wl,-s --test_env=ENVOY_IP_TEST_VERSIONS=v4only --sandbox_base=/tmp/sandbox_base" - - - script: ci/run_envoy_docker.sh 'ci/upload_gcs_artifact.sh /source/generated/$(CI_TARGET) $(CI_TARGET)' - displayName: "Upload $(CI_TARGET) Report to GCS" - env: - ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) - GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) - GCS_ARTIFACT_BUCKET: $(GcsArtifactBucket) - condition: always() - - - stage: docker - dependsOn: ["linux_x64", "linux_arm64"] - jobs: - - job: docker - displayName: "linux multiarch" - pool: - vmImage: "ubuntu-18.04" - steps: - - bash: .azure-pipelines/cleanup.sh - displayName: "Removing tools from agent" - - bash: | - echo "disk space at beginning of build:" - df -h - displayName: "Check disk space at beginning" - - task: DownloadBuildArtifacts@0 - inputs: - buildType: current - artifactName: "bazel.release" - itemPattern: "bazel.release/envoy_binary.tar.gz" - downloadType: single - targetPath: $(Build.StagingDirectory) - - task: DownloadBuildArtifacts@0 - inputs: - buildType: current - artifactName: "bazel.release.arm64" - itemPattern: "bazel.release.arm64/envoy_binary.tar.gz" - downloadType: single - targetPath: $(Build.StagingDirectory) - - bash: | - set -e - mkdir -p linux/amd64 && tar zxf $(Build.StagingDirectory)/bazel.release/envoy_binary.tar.gz -C ./linux/amd64 - mkdir -p linux/arm64 && tar zxf $(Build.StagingDirectory)/bazel.release.arm64/envoy_binary.tar.gz -C ./linux/arm64 - ci/docker_ci.sh - workingDirectory: $(Build.SourcesDirectory) - env: - AZP_BRANCH: $(Build.SourceBranch) - AZP_SHA1: $(Build.SourceVersion) - DOCKERHUB_USERNAME: $(DockerUsername) - DOCKERHUB_PASSWORD: $(DockerPassword) - - bash: | - echo "disk space at end of build:" - df -h - displayName: "Check disk space at end" - condition: always() - - task: PublishBuildArtifacts@1 - inputs: - pathtoPublish: "$(Build.StagingDirectory)/build_images" - artifactName: docker - condition: always() - - - stage: docs - dependsOn: ["docker"] - condition: and(succeeded(), eq(variables['PostSubmit'], true), ne(variables['NoSync'], true)) - jobs: - - job: publish - pool: - vmImage: "ubuntu-18.04" - steps: - - task: Cache@2 - inputs: - key: "docs | ./WORKSPACE | **/*.bzl" - path: $(Build.StagingDirectory)/repository_cache - continueOnError: true - - - script: ci/run_envoy_docker.sh 'ci/do_ci.sh docs' - workingDirectory: $(Build.SourcesDirectory) - env: - AZP_BRANCH: $(Build.SourceBranch) - ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) - BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com - BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance - GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) - displayName: "Generate docs" - - - task: InstallSSHKey@0 - inputs: - hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==" - sshPublicKey: "$(DocsPublicKey)" - sshPassphrase: "$(SshDeployKeyPassphrase)" - sshKeySecureFile: "$(DocsPrivateKey)" - - - script: docs/publish.sh - displayName: "Publish to GitHub" - workingDirectory: $(Build.SourcesDirectory) - env: - AZP_BRANCH: $(Build.SourceBranch) - - - stage: verify - dependsOn: ["docker"] - jobs: - - job: examples - pool: - vmImage: "ubuntu-18.04" - steps: - - task: DownloadBuildArtifacts@0 - inputs: - buildType: current - artifactName: "docker" - itemPattern: "docker/envoy-docker-images.tar.xz" - downloadType: single - targetPath: $(Build.StagingDirectory) - - bash: ./ci/do_ci.sh verify_examples - env: - ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) - NO_BUILD_SETUP: 1 - - job: examples_build - dependsOn: [] - pool: - vmImage: "ubuntu-18.04" - steps: - - task: DownloadBuildArtifacts@0 - inputs: - buildType: current - artifactName: "docker" - itemPattern: "docker/envoy-docker-images.tar.xz" - downloadType: single - targetPath: $(Build.StagingDirectory) - - bash: ./ci/do_ci.sh verify_build_examples - env: - ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) - NO_BUILD_SETUP: 1 - - - stage: macos - dependsOn: ["precheck"] - jobs: - - job: test - timeoutInMinutes: 180 - pool: - vmImage: "macos-latest" - steps: - - script: ./ci/mac_ci_setup.sh - displayName: "Install dependencies" - - - script: ./ci/mac_ci_steps.sh - displayName: "Run Mac CI" - env: - BAZEL_BUILD_EXTRA_OPTIONS: "--remote_download_toplevel --flaky_test_attempts=2" - BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com - BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance - GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) - - - task: PublishTestResults@2 - inputs: - testResultsFiles: "**/bazel-testlogs/**/test.xml" - testRunTitle: "macOS" - condition: always() - - - script: ./ci/flaky_test/run_process_xml.sh - displayName: "Process Test Results" - env: - TEST_TMPDIR: $(Build.SourcesDirectory) - SLACK_TOKEN: $(SLACK_TOKEN) - CI_TARGET: "MacOS" - REPO_URI: $(Build.Repository.Uri) - BUILD_URI: $(Build.BuildUri) - - - stage: windows - dependsOn: ["precheck"] - jobs: - - job: release - timeoutInMinutes: 120 - pool: - vmImage: "windows-latest" - steps: - - task: Cache@2 - inputs: - key: '"windows.release" | ./WORKSPACE | **/*.bzl' - path: $(Build.StagingDirectory)/repository_cache - continueOnError: true - - bash: ci/run_envoy_docker.sh ci/windows_ci_steps.sh - displayName: "Run Windows msvc-cl CI" - env: - CI_TARGET: "windows" - ENVOY_DOCKER_BUILD_DIR: "$(Build.StagingDirectory)" - SLACK_TOKEN: $(SLACK_TOKEN) - REPO_URI: $(Build.Repository.Uri) - BUILD_URI: $(Build.BuildUri) - ENVOY_RBE: "true" - BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --config=remote-msvc-cl --jobs=$(RbeJobs) --flaky_test_attempts=2" - BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com - BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance - GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) - - task: PublishTestResults@2 - inputs: - testResultsFiles: "**/bazel-out/**/testlogs/**/test.xml" - testRunTitle: "windows" - searchFolder: $(Build.StagingDirectory)/tmp - condition: always() - - task: PublishBuildArtifacts@1 - inputs: - pathtoPublish: "$(Build.StagingDirectory)/envoy" - artifactName: windows.release - condition: always() - - - job: clang_cl - timeoutInMinutes: 120 - pool: - vmImage: "windows-latest" - steps: - - task: Cache@2 - inputs: - key: '"windows.release" | ./WORKSPACE | **/*.bzl' - path: $(Build.StagingDirectory)/repository_cache - continueOnError: true - - bash: ci/run_envoy_docker.sh ci/windows_ci_steps.sh - displayName: "Run Windows clang-cl CI" - env: - CI_TARGET: "windows" - ENVOY_DOCKER_BUILD_DIR: "$(Build.StagingDirectory)" - SLACK_TOKEN: $(SLACK_TOKEN) - REPO_URI: $(Build.Repository.Uri) - BUILD_URI: $(Build.BuildUri) - ENVOY_RBE: "true" - BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --config=remote-clang-cl --jobs=$(RbeJobs) --flaky_test_attempts=2" - BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com - BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance - GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) - - task: PublishTestResults@2 - inputs: - testResultsFiles: "**/bazel-out/**/testlogs/**/test.xml" - testRunTitle: "clang-cl" - searchFolder: $(Build.StagingDirectory)/tmp - condition: always() - - task: PublishBuildArtifacts@1 - inputs: - pathtoPublish: "$(Build.StagingDirectory)/envoy" - artifactName: windows.clang-cl - condition: always() - - - job: docker - dependsOn: ["release"] - timeoutInMinutes: 120 - pool: - vmImage: "windows-latest" - steps: - - task: DownloadBuildArtifacts@0 - inputs: - buildType: current - artifactName: "windows.release" - itemPattern: "windows.release/envoy_binary.tar.gz" - downloadType: single - targetPath: $(Build.StagingDirectory) - - bash: | - set -e - # Convert to Unix-style path so tar doesn't think drive letter is a hostname - STAGING_DIR="/$(echo '$(Build.StagingDirectory)' | tr -d ':' | tr '\\' '/')" - mkdir -p windows/amd64 && tar zxf "${STAGING_DIR}/windows.release/envoy_binary.tar.gz" -C ./windows/amd64 - ci/docker_ci.sh - workingDirectory: $(Build.SourcesDirectory) - env: - AZP_BRANCH: $(Build.SourceBranch) - AZP_SHA1: $(Build.SourceVersion) - DOCKERHUB_USERNAME: $(DockerUsername) - DOCKERHUB_PASSWORD: $(DockerPassword) - - task: PublishBuildArtifacts@1 - inputs: - pathtoPublish: "$(Build.StagingDirectory)/build_images" - artifactName: docker_windows - condition: always() + pool: + vmImage: "ubuntu-18.04" + steps: + - script: ci/run_envoy_docker.sh 'ci/do_ci.sh format_pre' + workingDirectory: $(Build.SourcesDirectory) + env: + ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) + BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com + BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance + GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) + displayName: "Run format pre-checks" + - task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: "$(Build.StagingDirectory)/fix_format_pre.diff" + artifactName: format + # not all have fixes so improve condition/handling + condition: failed() + + - job: tooling + dependsOn: [] + pool: + vmImage: "ubuntu-18.04" + steps: + - script: ci/run_envoy_docker.sh 'ci/do_ci.sh tooling' + workingDirectory: $(Build.SourcesDirectory) + env: + ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) + BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com + BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance + GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) + displayName: "Run tooling checks" + + - script: | + ci/run_envoy_docker.sh 'ci/upload_gcs_artifact.sh /source/generated/tooling tooling' + displayName: "Upload tooling coverage to GCS" + env: + ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) + GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) + GCS_ARTIFACT_BUCKET: $(GcsArtifactBucket) + + - job: format + dependsOn: ["format_pre"] + pool: + vmImage: "ubuntu-18.04" + steps: + - task: Cache@2 + inputs: + key: "format | ./WORKSPACE | **/*.bzl" + path: $(Build.StagingDirectory)/repository_cache + continueOnError: true + + - script: ci/run_envoy_docker.sh 'ci/check_and_fix_format.sh' + workingDirectory: $(Build.SourcesDirectory) + env: + ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) + BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com + BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance + GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) + displayName: "Run check format scripts" + + - task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: "$(Build.StagingDirectory)/fix_format.diff" + artifactName: format + condition: failed() + + - job: docs + dependsOn: [] # this removes the implicit dependency on previous stage and causes this to run in parallel. + condition: ne(variables['PostSubmit'], true) + pool: + vmImage: "ubuntu-18.04" + steps: + - task: Cache@2 + inputs: + key: "docs | ./WORKSPACE | **/*.bzl" + path: $(Build.StagingDirectory)/repository_cache + continueOnError: true + + - script: ci/run_envoy_docker.sh 'ci/do_ci.sh docs' + workingDirectory: $(Build.SourcesDirectory) + env: + AZP_BRANCH: $(Build.SourceBranch) + ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) + BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com + BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance + GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) + displayName: "Generate docs" + + - script: | + ci/run_envoy_docker.sh 'ci/upload_gcs_artifact.sh /source/generated/docs docs' + displayName: "Upload Docs to GCS" + env: + ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) + GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) + GCS_ARTIFACT_BUCKET: $(GcsArtifactBucket) + + - job: dependencies + dependsOn: [] # this removes the implicit dependency on previous stage and causes this to run in parallel. + pool: + vmImage: "ubuntu-18.04" + steps: + - script: ci/run_envoy_docker.sh 'ci/do_ci.sh deps' + workingDirectory: $(Build.SourcesDirectory) + env: + ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) + BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com + BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance + GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) + GITHUB_TOKEN: $(GitHubPublicRepoOnlyAccessToken) + displayName: "Verify dependency information" + +- stage: sync + condition: and(succeeded(), eq(variables['PostSubmit'], true), ne(variables['NoSync'], true)) + dependsOn: [] + jobs: + - job: filter_example + dependsOn: [] + pool: + vmImage: "ubuntu-18.04" + steps: + - task: InstallSSHKey@0 + inputs: + hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==" + sshPublicKey: "$(FilterExamplePublicKey)" + sshPassphrase: "$(SshDeployKeyPassphrase)" + sshKeySecureFile: "$(FilterExamplePrivateKey)" + + - bash: ci/filter_example_mirror.sh + displayName: "Sync envoy-filter-example" + workingDirectory: $(Build.SourcesDirectory) + env: + AZP_BRANCH: $(Build.SourceBranch) + + - job: data_plane_api + dependsOn: [] + pool: + vmImage: "ubuntu-18.04" + steps: + - task: InstallSSHKey@0 + inputs: + hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==" + sshPublicKey: "$(DataPlaneApiPublicKey)" + sshPassphrase: "$(SshDeployKeyPassphrase)" + sshKeySecureFile: "$(DataPlaneApiPrivateKey)" + + - bash: ci/api_mirror.sh + displayName: "Sync data-plane-api" + workingDirectory: $(Build.SourcesDirectory) + env: + AZP_BRANCH: $(Build.SourceBranch) + + - job: go_control_plane + dependsOn: [] + pool: + vmImage: "ubuntu-18.04" + steps: + - task: InstallSSHKey@0 + inputs: + hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==" + sshPublicKey: "$(GoControlPlanePublicKey)" + sshPassphrase: "$(SshDeployKeyPassphrase)" + sshKeySecureFile: "$(GoControlPlanePrivateKey)" + + - bash: | + cp -a ~/.ssh $(Build.StagingDirectory)/ + ci/run_envoy_docker.sh 'ci/go_mirror.sh' + displayName: "Sync go-control-plane" + workingDirectory: $(Build.SourcesDirectory) + env: + ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) + BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com + BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance + GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) + AZP_BRANCH: $(Build.SourceBranch) + +- stage: linux_x64 + dependsOn: ["precheck"] + # For post-submit builds, continue even if precheck fails + condition: and(not(canceled()), or(succeeded(), eq(variables['PostSubmit'], true))) + jobs: + - job: release + timeoutInMinutes: 120 + pool: + vmImage: "ubuntu-18.04" + steps: + - template: bazel.yml + parameters: + ciTarget: bazel.release + +- stage: linux_arm64 + dependsOn: ["precheck"] + # For post-submit builds, continue even if precheck fails + condition: and(not(canceled()), or(succeeded(), eq(variables['PostSubmit'], true))) + jobs: + - job: release + timeoutInMinutes: 120 + pool: "arm-large" + steps: + - template: bazel.yml + parameters: + managedAgent: false + ciTarget: bazel.release + rbe: false + artifactSuffix: ".arm64" + bazelBuildExtraOptions: "--sandbox_base=/tmp/sandbox_base" + +- stage: check + dependsOn: ["linux_x64"] + jobs: + - job: bazel + displayName: "linux_x64" + dependsOn: [] + strategy: + maxParallel: 3 + matrix: + api: + CI_TARGET: "bazel.api" + gcc: + CI_TARGET: "bazel.gcc" + clang_tidy: + CI_TARGET: "bazel.clang_tidy" + asan: + CI_TARGET: "bazel.asan" + tsan: + CI_TARGET: "bazel.tsan" + compile_time_options: + CI_TARGET: "bazel.compile_time_options" + timeoutInMinutes: 120 + pool: + vmImage: "ubuntu-18.04" + steps: + - template: bazel.yml + parameters: + ciTarget: $(CI_TARGET) + + - job: coverage + displayName: "linux_x64" + dependsOn: [] + timeoutInMinutes: 120 + pool: "x64-large" + strategy: + maxParallel: 2 + matrix: + coverage: + CI_TARGET: "coverage" + fuzz_coverage: + CI_TARGET: "fuzz_coverage" + steps: + - template: bazel.yml + parameters: + managedAgent: false + ciTarget: bazel.$(CI_TARGET) + rbe: false + # /tmp/sandbox_base is a tmpfs in CI environment to optimize large I/O for coverage traces + bazelBuildExtraOptions: "--define=no_debug_info=1 --linkopt=-Wl,-s --test_env=ENVOY_IP_TEST_VERSIONS=v4only --sandbox_base=/tmp/sandbox_base" + + - script: ci/run_envoy_docker.sh 'ci/upload_gcs_artifact.sh /source/generated/$(CI_TARGET) $(CI_TARGET)' + displayName: "Upload $(CI_TARGET) Report to GCS" + env: + ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) + GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) + GCS_ARTIFACT_BUCKET: $(GcsArtifactBucket) + condition: always() + +- stage: docker + dependsOn: ["linux_x64", "linux_arm64"] + jobs: + - job: docker + displayName: "linux multiarch" + pool: + vmImage: "ubuntu-18.04" + steps: + - bash: .azure-pipelines/cleanup.sh + displayName: "Removing tools from agent" + - bash: | + echo "disk space at beginning of build:" + df -h + displayName: "Check disk space at beginning" + - task: DownloadBuildArtifacts@0 + inputs: + buildType: current + artifactName: "bazel.release" + itemPattern: "bazel.release/envoy_binary.tar.gz" + downloadType: single + targetPath: $(Build.StagingDirectory) + - task: DownloadBuildArtifacts@0 + inputs: + buildType: current + artifactName: "bazel.release.arm64" + itemPattern: "bazel.release.arm64/envoy_binary.tar.gz" + downloadType: single + targetPath: $(Build.StagingDirectory) + - bash: | + set -e + mkdir -p linux/amd64 && tar zxf $(Build.StagingDirectory)/bazel.release/envoy_binary.tar.gz -C ./linux/amd64 + mkdir -p linux/arm64 && tar zxf $(Build.StagingDirectory)/bazel.release.arm64/envoy_binary.tar.gz -C ./linux/arm64 + ci/docker_ci.sh + workingDirectory: $(Build.SourcesDirectory) + env: + AZP_BRANCH: $(Build.SourceBranch) + AZP_SHA1: $(Build.SourceVersion) + DOCKERHUB_USERNAME: $(DockerUsername) + DOCKERHUB_PASSWORD: $(DockerPassword) + - bash: | + echo "disk space at end of build:" + df -h + displayName: "Check disk space at end" + condition: always() + - task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: "$(Build.StagingDirectory)/build_images" + artifactName: docker + condition: always() + +- stage: docs + dependsOn: ["docker"] + condition: and(succeeded(), eq(variables['PostSubmit'], true), ne(variables['NoSync'], true)) + jobs: + - job: publish + pool: + vmImage: "ubuntu-18.04" + steps: + - task: Cache@2 + inputs: + key: "docs | ./WORKSPACE | **/*.bzl" + path: $(Build.StagingDirectory)/repository_cache + continueOnError: true + + - script: ci/run_envoy_docker.sh 'ci/do_ci.sh docs' + workingDirectory: $(Build.SourcesDirectory) + env: + AZP_BRANCH: $(Build.SourceBranch) + ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) + BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com + BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance + GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) + displayName: "Generate docs" + + - task: InstallSSHKey@0 + inputs: + hostName: "github.com ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEAq2A7hRGmdnm9tUDbO9IDSwBK6TbQa+PXYPCPy6rbTrTtw7PHkccKrpp0yVhp5HdEIcKr6pLlVDBfOLX9QUsyCOV0wzfjIJNlGEYsdlLJizHhbn2mUjvSAHQqZETYP81eFzLQNnPHt4EVVUh7VfDESU84KezmD5QlWpXLmvU31/yMf+Se8xhHTvKSCZIFImWwoG6mbUoWf9nzpIoaSjB+weqqUUmpaaasXVal72J+UX2B+2RPW3RcT0eOzQgqlJL3RKrTJvdsjE3JEAvGq3lGHSZXy28G3skua2SmVi/w4yCE6gbODqnTWlg7+wC604ydGXA8VJiS5ap43JXiUFFAaQ==" + sshPublicKey: "$(DocsPublicKey)" + sshPassphrase: "$(SshDeployKeyPassphrase)" + sshKeySecureFile: "$(DocsPrivateKey)" + + - script: docs/publish.sh + displayName: "Publish to GitHub" + workingDirectory: $(Build.SourcesDirectory) + env: + AZP_BRANCH: $(Build.SourceBranch) + +- stage: verify + dependsOn: ["docker"] + jobs: + - job: examples + pool: + vmImage: "ubuntu-18.04" + steps: + - task: DownloadBuildArtifacts@0 + inputs: + buildType: current + artifactName: "docker" + itemPattern: "docker/envoy-docker-images.tar.xz" + downloadType: single + targetPath: $(Build.StagingDirectory) + - bash: ./ci/do_ci.sh verify_examples + env: + ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) + NO_BUILD_SETUP: 1 + - job: examples_build + dependsOn: [] + pool: + vmImage: "ubuntu-18.04" + steps: + - task: DownloadBuildArtifacts@0 + inputs: + buildType: current + artifactName: "docker" + itemPattern: "docker/envoy-docker-images.tar.xz" + downloadType: single + targetPath: $(Build.StagingDirectory) + - bash: ./ci/do_ci.sh verify_build_examples + env: + ENVOY_DOCKER_BUILD_DIR: $(Build.StagingDirectory) + NO_BUILD_SETUP: 1 + +- stage: macos + dependsOn: ["precheck"] + jobs: + - job: test + timeoutInMinutes: 180 + pool: + vmImage: "macos-latest" + steps: + - script: ./ci/mac_ci_setup.sh + displayName: "Install dependencies" + + - script: ./ci/mac_ci_steps.sh + displayName: "Run Mac CI" + env: + BAZEL_BUILD_EXTRA_OPTIONS: "--remote_download_toplevel --flaky_test_attempts=2" + BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com + BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance + GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) + + - task: PublishTestResults@2 + inputs: + testResultsFiles: "**/bazel-testlogs/**/test.xml" + testRunTitle: "macOS" + condition: always() + + - script: ./ci/flaky_test/run_process_xml.sh + displayName: "Process Test Results" + env: + TEST_TMPDIR: $(Build.SourcesDirectory) + SLACK_TOKEN: $(SLACK_TOKEN) + CI_TARGET: "MacOS" + REPO_URI: $(Build.Repository.Uri) + BUILD_URI: $(Build.BuildUri) + +- stage: windows + dependsOn: ["precheck"] + jobs: + - job: release + timeoutInMinutes: 120 + pool: + vmImage: "windows-latest" + steps: + - task: Cache@2 + inputs: + key: '"windows.release" | ./WORKSPACE | **/*.bzl' + path: $(Build.StagingDirectory)/repository_cache + continueOnError: true + - bash: ci/run_envoy_docker.sh ci/windows_ci_steps.sh + displayName: "Run Windows msvc-cl CI" + env: + CI_TARGET: "windows" + ENVOY_DOCKER_BUILD_DIR: "$(Build.StagingDirectory)" + SLACK_TOKEN: $(SLACK_TOKEN) + REPO_URI: $(Build.Repository.Uri) + BUILD_URI: $(Build.BuildUri) + ENVOY_RBE: "true" + BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --config=remote-msvc-cl --jobs=$(RbeJobs) --flaky_test_attempts=2" + BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com + BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance + GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) + - task: PublishTestResults@2 + inputs: + testResultsFiles: "**/bazel-out/**/testlogs/**/test.xml" + testRunTitle: "windows" + searchFolder: $(Build.StagingDirectory)/tmp + condition: always() + - task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: "$(Build.StagingDirectory)/envoy" + artifactName: windows.release + condition: always() + + - job: clang_cl + timeoutInMinutes: 120 + pool: + vmImage: "windows-latest" + steps: + - task: Cache@2 + inputs: + key: '"windows.release" | ./WORKSPACE | **/*.bzl' + path: $(Build.StagingDirectory)/repository_cache + continueOnError: true + - bash: ci/run_envoy_docker.sh ci/windows_ci_steps.sh + displayName: "Run Windows clang-cl CI" + env: + CI_TARGET: "windows" + ENVOY_DOCKER_BUILD_DIR: "$(Build.StagingDirectory)" + SLACK_TOKEN: $(SLACK_TOKEN) + REPO_URI: $(Build.Repository.Uri) + BUILD_URI: $(Build.BuildUri) + ENVOY_RBE: "true" + BAZEL_BUILD_EXTRA_OPTIONS: "--config=remote-ci --config=remote-clang-cl --jobs=$(RbeJobs) --flaky_test_attempts=2" + BAZEL_REMOTE_CACHE: grpcs://remotebuildexecution.googleapis.com + BAZEL_REMOTE_INSTANCE: projects/envoy-ci/instances/default_instance + GCP_SERVICE_ACCOUNT_KEY: $(GcpServiceAccountKey) + - task: PublishTestResults@2 + inputs: + testResultsFiles: "**/bazel-out/**/testlogs/**/test.xml" + testRunTitle: "clang-cl" + searchFolder: $(Build.StagingDirectory)/tmp + condition: always() + - task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: "$(Build.StagingDirectory)/envoy" + artifactName: windows.clang-cl + condition: always() + + - job: docker + dependsOn: ["release"] + timeoutInMinutes: 120 + pool: + vmImage: "windows-latest" + steps: + - task: DownloadBuildArtifacts@0 + inputs: + buildType: current + artifactName: "windows.release" + itemPattern: "windows.release/envoy_binary.tar.gz" + downloadType: single + targetPath: $(Build.StagingDirectory) + - bash: | + set -e + # Convert to Unix-style path so tar doesn't think drive letter is a hostname + STAGING_DIR="/$(echo '$(Build.StagingDirectory)' | tr -d ':' | tr '\\' '/')" + mkdir -p windows/amd64 && tar zxf "${STAGING_DIR}/windows.release/envoy_binary.tar.gz" -C ./windows/amd64 + ci/docker_ci.sh + workingDirectory: $(Build.SourcesDirectory) + env: + AZP_BRANCH: $(Build.SourceBranch) + AZP_SHA1: $(Build.SourceVersion) + DOCKERHUB_USERNAME: $(DockerUsername) + DOCKERHUB_PASSWORD: $(DockerPassword) + - task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: "$(Build.StagingDirectory)/build_images" + artifactName: docker_windows + condition: always() diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index befd00065cbd..e11da6611401 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -4,34 +4,34 @@ tasks: name: "RBE" platform: ubuntu1804 test_targets: - - "//test/common/common/..." - - "//test/integration/..." - - "//test/exe/..." + - "//test/common/common/..." + - "//test/integration/..." + - "//test/exe/..." test_flags: - - "--config=remote-clang-libc++" - - "--config=remote-ci" - - "--define=wasm=disabled" - - "--jobs=75" + - "--config=remote-clang-libc++" + - "--config=remote-ci" + - "--define=wasm=disabled" + - "--jobs=75" coverage: name: "Coverage" platform: ubuntu1804 shell_commands: - - "bazel/setup_clang.sh /usr/lib/llvm-10" + - "bazel/setup_clang.sh /usr/lib/llvm-10" test_targets: - - "//test/common/common/..." - - "//test/integration/..." - - "//test/exe/..." + - "//test/common/common/..." + - "//test/integration/..." + - "//test/exe/..." test_flags: - - "--config=coverage" - - "--config=clang" + - "--config=coverage" + - "--config=clang" fuzz_coverage: name: "Fuzz-Coverage" platform: ubuntu1804 shell_commands: - - "bazel/setup_clang.sh /usr/lib/llvm-10" + - "bazel/setup_clang.sh /usr/lib/llvm-10" test_targets: - - "//test/server:server_fuzz_test" + - "//test/server:server_fuzz_test" test_flags: - - "--config=fuzz-coverage" - - "--config=coverage" - - "--config=clang" + - "--config=fuzz-coverage" + - "--config=coverage" + - "--config=clang" diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index cf659d48fe60..e15c5a08f654 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,5 +1,5 @@ blank_issues_enabled: false contact_links: - - name: "Crash bug" - url: https://github.com/envoyproxy/envoy/security/policy - about: "Please file any crash bug with envoy-security@googlegroups.com." +- name: "Crash bug" + url: https://github.com/envoyproxy/envoy/security/policy + about: "Please file any crash bug with envoy-security@googlegroups.com." diff --git a/.github/dependabot.yml b/.github/dependabot.yml index c2f9b29a47f3..f2f567dddaa3 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,98 +1,97 @@ version: 2 updates: - - package-ecosystem: "pip" - directory: "/.github/actions/pr_notifier" - schedule: - interval: "daily" - - - package-ecosystem: "pip" - directory: "/test/extensions/filters/network/thrift_proxy" - schedule: - interval: "daily" - - - package-ecosystem: "pip" - directory: "/source/extensions/filters/network/kafka" - schedule: - interval: "daily" - - - package-ecosystem: "pip" - directory: "/examples/grpc-bridge/client" - schedule: - interval: "daily" - - - package-ecosystem: "pip" - directory: "/docs" - schedule: - interval: "daily" - - - package-ecosystem: "pip" - directory: "/tools/docs" - schedule: - interval: "daily" - - - package-ecosystem: "pip" - directory: "/tools/github" - schedule: - interval: "daily" - - - package-ecosystem: "pip" - directory: "/tools/config_validation" - schedule: - interval: "daily" - - - package-ecosystem: "pip" - directory: "/tools/dependency" - schedule: - interval: "daily" - - - package-ecosystem: "pip" - directory: "/tools/deprecate_version" - schedule: - interval: "daily" - - - package-ecosystem: "pip" - directory: "/tools/protodoc" - schedule: - interval: "daily" - - - package-ecosystem: "pip" - directory: "/tools/deprecate_features" - schedule: - interval: "daily" - - - package-ecosystem: "pip" - directory: "/tools/code_format" - schedule: - interval: "daily" - - - package-ecosystem: "pip" - directory: "/tools/envoy_headersplit" - schedule: - interval: "daily" - - - package-ecosystem: "pip" - directory: "/ci/flaky_test" - schedule: - interval: "daily" - - - package-ecosystem: "pip" - directory: "/configs" - schedule: - interval: "daily" - - - package-ecosystem: "pip" - directory: "/tools/testing" - schedule: - interval: "daily" - - - package-ecosystem: "docker" - directory: "/ci" - schedule: - interval: daily - - - package-ecosystem: "docker" - directory: "/.devcontainer" - schedule: - interval: daily - +- package-ecosystem: "pip" + directory: "/.github/actions/pr_notifier" + schedule: + interval: "daily" + +- package-ecosystem: "pip" + directory: "/test/extensions/filters/network/thrift_proxy" + schedule: + interval: "daily" + +- package-ecosystem: "pip" + directory: "/source/extensions/filters/network/kafka" + schedule: + interval: "daily" + +- package-ecosystem: "pip" + directory: "/examples/grpc-bridge/client" + schedule: + interval: "daily" + +- package-ecosystem: "pip" + directory: "/docs" + schedule: + interval: "daily" + +- package-ecosystem: "pip" + directory: "/tools/docs" + schedule: + interval: "daily" + +- package-ecosystem: "pip" + directory: "/tools/github" + schedule: + interval: "daily" + +- package-ecosystem: "pip" + directory: "/tools/config_validation" + schedule: + interval: "daily" + +- package-ecosystem: "pip" + directory: "/tools/dependency" + schedule: + interval: "daily" + +- package-ecosystem: "pip" + directory: "/tools/deprecate_version" + schedule: + interval: "daily" + +- package-ecosystem: "pip" + directory: "/tools/protodoc" + schedule: + interval: "daily" + +- package-ecosystem: "pip" + directory: "/tools/deprecate_features" + schedule: + interval: "daily" + +- package-ecosystem: "pip" + directory: "/tools/code_format" + schedule: + interval: "daily" + +- package-ecosystem: "pip" + directory: "/tools/envoy_headersplit" + schedule: + interval: "daily" + +- package-ecosystem: "pip" + directory: "/ci/flaky_test" + schedule: + interval: "daily" + +- package-ecosystem: "pip" + directory: "/configs" + schedule: + interval: "daily" + +- package-ecosystem: "pip" + directory: "/tools/testing" + schedule: + interval: "daily" + +- package-ecosystem: "docker" + directory: "/ci" + schedule: + interval: daily + +- package-ecosystem: "docker" + directory: "/.devcontainer" + schedule: + interval: daily diff --git a/.github/workflows/codeql-daily.yml b/.github/workflows/codeql-daily.yml index 00707a85782a..d266b56ccc5d 100644 --- a/.github/workflows/codeql-daily.yml +++ b/.github/workflows/codeql-daily.yml @@ -1,6 +1,6 @@ on: schedule: - - cron: '0 12 * * 4' + - cron: '0 12 * * 4' jobs: CodeQL-Build: @@ -29,7 +29,7 @@ jobs: uses: github/codeql-action/init@v1 # Override language selection by uncommenting this and choosing your languages with: - languages: cpp + languages: cpp - name: Install deps shell: bash diff --git a/.github/workflows/codeql-push.yml b/.github/workflows/codeql-push.yml index 56254401e418..84e8e3a008e6 100644 --- a/.github/workflows/codeql-push.yml +++ b/.github/workflows/codeql-push.yml @@ -37,7 +37,7 @@ jobs: uses: github/codeql-action/init@v1 # Override language selection by uncommenting this and choosing your languages with: - languages: cpp + languages: cpp - name: Install deps shell: bash diff --git a/.github/workflows/pr_notifier.yml b/.github/workflows/pr_notifier.yml index 1ce61c693697..1ccc9695e3dd 100644 --- a/.github/workflows/pr_notifier.yml +++ b/.github/workflows/pr_notifier.yml @@ -1,7 +1,7 @@ on: workflow_dispatch: schedule: - - cron: '0 5 * * 1,2,3,4,5' + - cron: '0 5 * * 1,2,3,4,5' jobs: pr_notifier: @@ -9,17 +9,17 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Set up Python 3.8 - uses: actions/setup-python@v2 - with: - python-version: '3.8' - architecture: 'x64' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r ./.github/actions/pr_notifier/requirements.txt - - name: Notify about PRs - run: python ./.github/actions/pr_notifier/pr_notifier.py - env: - SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + - uses: actions/checkout@v2 + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: '3.8' + architecture: 'x64' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r ./.github/actions/pr_notifier/requirements.txt + - name: Notify about PRs + run: python ./.github/actions/pr_notifier/pr_notifier.py + env: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index f3ce371f510e..0b82b48b088e 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -1,7 +1,7 @@ on: workflow_dispatch: schedule: - - cron: '0 */4 * * *' + - cron: '0 */4 * * *' jobs: prune_stale: diff --git a/.zuul/playbooks/envoy-build/run.yaml b/.zuul/playbooks/envoy-build/run.yaml index 0087029e4a4f..e4fc75422d3e 100644 --- a/.zuul/playbooks/envoy-build/run.yaml +++ b/.zuul/playbooks/envoy-build/run.yaml @@ -1,17 +1,17 @@ - hosts: all become: yes roles: - - role: config-gcc - gcc_version: 7 - - role: config-bazel - bazel_version: 0.28.1 + - role: config-gcc + gcc_version: 7 + - role: config-bazel + bazel_version: 0.28.1 tasks: - - name: Build envoy - shell: - cmd: | - apt update - apt-get update - apt-get install -y \ + - name: Build envoy + shell: + cmd: | + apt update + apt-get update + apt-get install -y \ libtool \ cmake \ automake \ @@ -22,9 +22,9 @@ unzip \ virtualenv - bazel build //source/exe:envoy-static | tee $LOGS_PATH//bazel.txt + bazel build //source/exe:envoy-static | tee $LOGS_PATH//bazel.txt - cp -r ./bazel-bin $RESULTS_PATH - chdir: '{{ zuul.project.src_dir }}' - executable: /bin/bash - environment: '{{ global_env }}' + cp -r ./bazel-bin $RESULTS_PATH + chdir: '{{ zuul.project.src_dir }}' + executable: /bin/bash + environment: '{{ global_env }}'