Update common Docker engineering infrastructure with latest

eng/common/templates/1es-official.yml

@@ -46,9 +46,13 @@ extends:
         ignoreDirectories: $(Build.SourcesDirectory)/versions
         whatIf: ${{ parameters.cgDryRun }}
         showAlertLink: true
+      policheck:
+        enabled: true
       sourceRepositoriesToScan:
         exclude:
         - repository: InternalVersionsRepo
         - repository: PublicVersionsRepo
       sourceAnalysisPool: ${{ parameters.sourceAnalysisPool }}
+      tsa:
+        enabled: true
     stages: ${{ parameters.stages }}

eng/common/templates/1es-unofficial.yml

@@ -45,16 +45,20 @@ extends:
   parameters:
     pool: ${{ parameters.pool }}
     sdl:
-      enableAllTools: ${{ not(parameters.disableSDL) }}
       componentgovernance:
         ignoreDirectories: $(Build.SourcesDirectory)/versions
         whatIf: true
         showAlertLink: true
+      enableAllTools: ${{ not(parameters.disableSDL) }}
+      policheck:
+        enabled: true
       sbom:
         enabled: true
       sourceRepositoriesToScan:
         exclude:
         - repository: InternalVersionsRepo
         - repository: PublicVersionsRepo
       sourceAnalysisPool: ${{ parameters.sourceAnalysisPool }}
+      tsa:
+        enabled: true
     stages: ${{ parameters.stages }}

eng/common/templates/jobs/build-images.yml

@@ -4,18 +4,19 @@ parameters:
   matrix: {}
   dockerClientOS: null
   buildJobTimeout: 60
+  commonInitStepsForMatrixAndBuild: []
   customInitSteps: []
   noCache: false
   internalProjectName: null
   publicProjectName: null
-  internalVersionsRepoRef: null
-  publicVersionsRepoRef: null
+  isInternalServicingValidation: false
 
 jobs:
 - job: ${{ parameters.name }}
-  condition: and(${{ parameters.matrix }}, not(canceled()), in(dependencies.PreBuildValidation.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'))
+  condition: and(${{ parameters.matrix }}, not(canceled()), or(in(dependencies.PreBuildValidation.result, 'Succeeded', 'SucceededWithIssues', 'Skipped'), eq(${{ parameters.isInternalServicingValidation }}, 'true')))
   dependsOn:
-  - PreBuildValidation
+  - ${{ if eq(parameters.isInternalServicingValidation, 'false') }}:
+    - PreBuildValidation
   - CopyBaseImages
   - GenerateBuildMatrix
   pool: ${{ parameters.pool }}
@@ -24,59 +25,11 @@ jobs:
   timeoutInMinutes: ${{ parameters.buildJobTimeout }}
   variables:
     imageBuilderDockerRunExtraOptions: $(build.imageBuilderDockerRunExtraOptions)
-    versionsRepoPath: versions
     sbomDirectory: $(Build.ArtifactStagingDirectory)/sbom
     imageInfoHostDir: $(Build.ArtifactStagingDirectory)/imageInfo
     imageInfoContainerDir: $(artifactsPath)/imageInfo
-    ${{ if eq(parameters.noCache, false) }}:
-      versionsBasePath: $(versionsRepoPath)/
-      pipelineDisabledCache: false
-    ${{ if eq(parameters.noCache, true) }}:
-      versionsBasePath: ""
-      pipelineDisabledCache: true
   steps:
-  - checkout: self
-  - ${{ if and(eq(variables['System.TeamProject'], parameters.publicProjectName), eq(parameters.noCache, false)) }}:
-    - checkout: ${{ parameters.publicVersionsRepoRef }}
-      path: s/$(versionsRepoPath)
-  - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), eq(parameters.noCache, false)) }}:
-    - checkout: ${{ parameters.internalVersionsRepoRef }}
-      path: s/$(versionsRepoPath)
-  - ${{ if eq(parameters.noCache, false) }}:
-    - powershell: |
-        $pathSeparatorIndex = "$(Build.Repository.Name)".IndexOf("/")
-        if ($pathSeparatorIndex -ge 0) {
-          $buildRepoName = "$(Build.Repository.Name)".Substring($pathSeparatorIndex + 1)
-        }
-        else {
-          $buildRepoName = "$(Build.Repository.Name)"
-        }
-
-        $engCommonPath = "$(Build.Repository.LocalPath)/$buildRepoName/$(engCommonRelativePath)"
-        $engPath = "$(Build.Repository.LocalPath)/$buildRepoName/eng"
-        $manifest = "$buildRepoName/$(manifest)"
-        $testResultsDirectory = "$buildRepoName/$testResultsDirectory"
-
-        if ("$(testScriptPath)") {
-          $testScriptPath = "$buildRepoName/$(testScriptPath)"
-        }
-
-        echo "##vso[task.setvariable variable=buildRepoName]$buildRepoName"
-        echo "##vso[task.setvariable variable=manifest]$manifest"
-        echo "##vso[task.setvariable variable=engCommonPath]$engCommonPath"
-        echo "##vso[task.setvariable variable=engPath]$engPath"
-        echo "##vso[task.setvariable variable=testScriptPath]$testScriptPath"
-        echo "##vso[task.setvariable variable=testResultsDirectory]$testResultsDirectory"
-      displayName: Override Common Paths
-  - powershell: |
-      if ("${{ parameters.noCache }}" -eq "false") {
-        $baseContainerRepoPath = "/repo/$(buildRepoName)"
-      }
-      else {
-        $baseContainerRepoPath = "/repo"
-      }
-      echo "##vso[task.setvariable variable=baseContainerRepoPath]$baseContainerRepoPath"
-    displayName: Set Base Container Repo Path
+  - ${{ parameters.commonInitStepsForMatrixAndBuild }}
   - template: /eng/common/templates/jobs/${{ format('../steps/init-docker-{0}.yml', parameters.dockerClientOS) }}@self
     parameters:
       cleanupDocker: true
@@ -96,13 +49,9 @@ jobs:
       # to escape the single quotes that are in the string which would need to be done outside the context of PowerShell. Since
       # all we need is for that value to be in a PowerShell variable, we can get that by the fact that AzDO automatically creates
       # the environment variable for us.
-      $imageBuilderBuildArgs = "$env:IMAGEBUILDERBUILDARGS $(imageBuilder.queueArgs) --image-info-output-path $(imageInfoContainerDir)/$(legName)-image-info.json"
-      if ($env:SYSTEM_TEAMPROJECT -eq "${{ parameters.internalProjectName }}" -and $env:BUILD_REASON -ne "PullRequest") {
-        $imageBuilderBuildArgs = "$imageBuilderBuildArgs --registry-override $(acr-staging.server) --repo-prefix $(stagingRepoPrefix) --source-repo-prefix $(mirrorRepoPrefix) --push"
-      }
-
-      if ($env:SYSTEM_TEAMPROJECT -eq "${{ parameters.publicProjectName }}" -and ${env:PUBLIC-MIRROR_SERVER} -ne "") {
-        $imageBuilderBuildArgs = "$imageBuilderBuildArgs --base-override-regex '^(?!mcr\.microsoft\.com)' --base-override-sub '$(public-mirror.server)/'"
+      $imageBuilderBuildArgs = "$env:IMAGEBUILDERBUILDARGS $(imageBuilder.queueArgs) --image-info-output-path $(imageInfoContainerDir)/$(legName)-image-info.json $(commonMatrixAndBuildOptions)"
+      if ($env:SYSTEM_TEAMPROJECT -eq "${{ parameters.internalProjectName }}" -and $env:BUILD_REASON -ne "PullRequest" -and "${{ parameters.isInternalServicingValidation }}" -ne "true") {
+        $imageBuilderBuildArgs = "$imageBuilderBuildArgs --repo-prefix $(stagingRepoPrefix) --push"
       }
 
       # If the pipeline isn't configured to disable the cache and a build variable hasn't been set to disable the cache
@@ -128,7 +77,6 @@ jobs:
         --os-type $(osType)
         --architecture $(architecture)
         --retry
-        --source-repo $(publicGitRepoUri)
         --digests-out-var 'builtImages'
         --acr-subscription '$(acr-staging.subscription)'
         --acr-resource-group '$(acr-staging.resourceGroup)'
@@ -141,7 +89,7 @@ jobs:
       displayName: Publish Image Info File Artifact
       internalProjectName: ${{ parameters.internalProjectName }}
       publicProjectName: ${{ parameters.publicProjectName }}
-  - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}:
+  - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.isInternalServicingValidation, 'false')) }}:
     # The following task depends on the SBOM Manifest Generator task installed on the agent.
     # This task is auto-injected by 1ES Pipeline Templates so we don't need to install it ourselves.
     - powershell: |
@@ -193,11 +141,11 @@ jobs:
         }
       displayName: Generate SBOMs
       condition: and(succeeded(), ne(variables['BuildImages.builtImages'], ''))
-  - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
+  - ${{ if or(eq(variables['Build.Reason'], 'PullRequest'), eq(parameters.isInternalServicingValidation, 'true')) }}:
     - template: /eng/common/templates/jobs/${{ format('../steps/test-images-{0}-client.yml', parameters.dockerClientOS) }}@self
       parameters:
         condition: ne(variables.testScriptPath, '')
-  - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}:
+  - ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), eq(parameters.isInternalServicingValidation, 'false')) }}:
     - template: /eng/common/templates/steps/publish-artifact.yml@self
       parameters:
         path: $(sbomDirectory)

eng/common/templates/jobs/generate-matrix.yml

@@ -5,28 +5,43 @@ parameters:
   customBuildLegGroupArgs: ""
   isTestStage: false
   internalProjectName: null
+  noCache: false
+  commonInitStepsForMatrixAndBuild: []
 
 jobs:
 - job: ${{ parameters.name }}
   pool: ${{ parameters.pool }}
   steps:
+  - ${{ parameters.commonInitStepsForMatrixAndBuild }}
   - template: /eng/common/templates/steps/retain-build.yml@self
   - template: /eng/common/templates/steps/init-docker-linux.yml@self
   - template: /eng/common/templates/steps/validate-branch.yml@self
     parameters:
       internalProjectName: ${{ parameters.internalProjectName }}
+  - template: /eng/common/templates/steps/set-image-info-path-var.yml
+    parameters:
+      publicSourceBranch: $(publicSourceBranch)
   - ${{ if eq(parameters.isTestStage, true) }}:
     - template: /eng/common/templates/steps/download-build-artifact.yml@self
       parameters:
         targetPath: $(Build.ArtifactStagingDirectory)
         artifactName: image-info
-    - script: echo "##vso[task.setvariable variable=additionalGenerateBuildMatrixOptions]--image-info $(artifactsPath)/image-info.json"
-      displayName: Set GenerateBuildMatrix Variables
-  - ${{ if eq(parameters.isTestStage, false) }}:
-    - script: echo "##vso[task.setvariable variable=additionalGenerateBuildMatrixOptions]"
-      displayName: Set GenerateBuildMatrix Variables
+  - powershell: |
+      $additionalGenerateBuildMatrixOptions = "$(additionalGenerateBuildMatrixOptions)"
+      
+      if ("${{ parameters.isTestStage}}" -eq "true") {
+        $additionalGenerateBuildMatrixOptions = "$additionalGenerateBuildMatrixOptions --image-info $(artifactsPath)/image-info.json"
+      }
+      elseif ("$(pipelineDisabledCache)" -ne "true" -and $env:NOCACHE -ne "true" -and "$(trimCachedImagesForMatrix)" -eq "true") {
+        # If the pipeline isn't configured to disable the cache and a build variable hasn't been set to disable the cache
+        $additionalGenerateBuildMatrixOptions = "$additionalGenerateBuildMatrixOptions --image-info $(versionsBasePath)$(imageInfoVersionsPath) --trim-cached-images"
+      }
+
+      echo "##vso[task.setvariable variable=additionalGenerateBuildMatrixOptions]$additionalGenerateBuildMatrixOptions"
+    displayName: Set GenerateBuildMatrix Variables
   - script: >
-      $(runImageBuilderCmd) generateBuildMatrix
+      echo "##vso[task.setvariable variable=generateBuildMatrixCommand]
+      generateBuildMatrix
       --manifest $(manifest)
       --type ${{ parameters.matrixType }}
       --os-type '*'
@@ -35,6 +50,13 @@ jobs:
       ${{ parameters.customBuildLegGroupArgs }}
       $(imageBuilder.pathArgs)
       $(manifestVariables)
-      $(additionalGenerateBuildMatrixOptions)
-    displayName: Generate ${{ parameters.matrixType }} Matrix
-    name: matrix
+      $(commonMatrixAndBuildOptions)
+      $(additionalGenerateBuildMatrixOptions)"
+    displayName: Set GenerateBuildMatrix Command
+  - template: /eng/common/templates/steps/run-imagebuilder.yml@self
+    parameters:
+      name: matrix
+      displayName: Generate ${{ parameters.matrixType }} Matrix
+      serviceConnection: $(build.serviceConnectionName)
+      internalProjectName: internal
+      args: $(generateBuildMatrixCommand)

eng/common/templates/jobs/post-build.yml

@@ -52,22 +52,32 @@ jobs:
           }
     displayName: Prune Publish Artifacts
   - powershell: |
+      $imageInfoFiles = Get-ChildItem "$(imageInfosHostDir)"
+      if ($imageInfoFiles.Count -eq 0) {
+        echo "No image info files found."
+        echo "##vso[task.setvariable variable=noImageInfos;isOutput=true]true"
+        exit 0
+      }
+
       New-Item -ItemType Directory -Path $(imageInfosHostDir)$(imageInfosOutputSubDir) -Force
       $(runImageBuilderCmd) mergeImageInfo `
         --manifest $(manifest) `
         $(imageInfosContainerDir) `
         $(imageInfosContainerDir)$(imageInfosOutputSubDir)/image-info.json `
         $(manifestVariables)
+    name: MergeImageInfoFiles
     displayName: Merge Image Info Files
   - template: /eng/common/templates/steps/publish-artifact.yml@self
     parameters:
+      condition: and(succeeded(), ne(variables['MergeImageInfoFiles.noImageInfos'], 'true'))
       path: $(sbomOutputDir)
       artifactName: sboms
       displayName: Publish SBOM Artifact
       internalProjectName: ${{ parameters.internalProjectName }}
       publicProjectName: ${{ parameters.publicProjectName }}
   - template: /eng/common/templates/steps/publish-artifact.yml@self
     parameters:
+      condition: and(succeeded(), ne(variables['MergeImageInfoFiles.noImageInfos'], 'true'))
       path: $(imageInfosHostDir)$(imageInfosOutputSubDir)
       artifactName: image-info
       displayName: Publish Image Info File Artifact