From beed292595035108f09e345a095c62dfd9696652 Mon Sep 17 00:00:00 2001 From: Davis Goodin Date: Wed, 21 Aug 2019 15:44:58 -0500 Subject: [PATCH 1/2] Split blob publish into stages run before Arcade * Split blob publish into stages run before Arcade The Arcade publish stages contain the job that promotes the build, causing auto-update PRs to be generated. This means that running installer publish after Arcade can cause a temporarily broken update PR. To fix this, publish blobs before the Arcade stages. Leave dotnet/versions publish and build finalization until after the Arcade stages, so that they still only happen after the nupkgs are published. * Changes per feedback Remove "Custom" verbiage, not useful and in some cases now incorrect. Order publish.yml stages by dependency to help readability. (cherry picked from commit 54ae67a366f3dcc374b487be2bb04dba267bb275) --- azure-pipelines.yml | 22 +-- eng/jobs/run-publish-project.yml | 120 ++++++++++++++++ eng/stages/custom-publish.yml | 129 ------------------ eng/stages/publish.yml | 48 +++++++ publish/publish-blobs.proj | 75 ++++++++++ ...custom-publish.proj => publish-final.proj} | 64 --------- 6 files changed, 248 insertions(+), 210 deletions(-) create mode 100644 eng/jobs/run-publish-project.yml delete mode 100644 eng/stages/custom-publish.yml create mode 100644 eng/stages/publish.yml create mode 100644 publish/publish-blobs.proj rename publish/{custom-publish.proj => publish-final.proj} (54%) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index e239b1291b..a4ea4c4b2e 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -171,24 +171,12 @@ stages: name: NetCoreInternal-Pool queue: buildpool.windows.10.amd64.vs2017 - # Stages-based publishing entry point - - template: eng\common\templates\post-build\post-build.yml + - template: /eng/stages/publish.yml parameters: - dependsOn: [PrepareForPublish] - # Symbol validation is not ready yet. https://github.com/dotnet/arcade/issues/2871 - enableSymbolValidation: false - # Doesn't work yet, not fully configured. https://github.com/dotnet/core-setup/issues/5254 - enableSigningValidation: false - # SourceLink validation doesn't work in dev builds: tries to pull from GitHub. https://github.com/dotnet/arcade/issues/3604 - enableSourceLinkValidation: false - # Allow symbol publish to emit expected warnings without failing the build. Include single - # quotes inside the string so that it passes through to MSBuild without script interference. - symbolPublishingAdditionalParameters: "'-warnAsError:$false'" - - - template: /eng/stages/custom-publish.yml - parameters: - # Custom publish stage dependency configuration. - # - dependsOn: + # Publish channel configuration. The publish template wraps Arcade publishing and adds some + # extras that aren't implemented in Arcade yet. + # + # - dependsOn: # channel: # name: # bar: diff --git a/eng/jobs/run-publish-project.yml b/eng/jobs/run-publish-project.yml new file mode 100644 index 0000000000..734cd31e9d --- /dev/null +++ b/eng/jobs/run-publish-project.yml @@ -0,0 +1,120 @@ +parameters: + projectName: '' + dependency: null + +jobs: + +- template: /eng/common/templates/post-build/setup-maestro-vars.yml + +- job: CustomPublish + displayName: Custom Publish + dependsOn: setupMaestroVars + # Only run this job if setup-maestro-vars says the current branch publishes to this channel. + # Logic copied from ../common/templates/post-build/channels/netcore-dev-5.yml + condition: contains( + dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], + format('[{0}]', variables['${{ parameters.dependency.channel.bar }}'])) + pool: + name: Hosted VS2017 + # Double the default timeout. + timeoutInMinutes: 120 + workspace: + clean: all + + variables: + # Only get the secret variable groups if the def has the official name. Reduce dev build def risk. + - ${{ if eq(variables['Build.DefinitionName'], 'dotnet-core-setup') }}: + # Used for publishing individual leg assets to azure blob storage + - group: DotNet-DotNetCli-Storage + # Used for dotnet/versions update + - group: DotNet-Versions-Publish + + # Blob storage publish (installers and checksums) + - name: _AzureAccountName + value: $[ coalesce(variables.AzureAccountName, 'dotnetcli') ] + - name: _ContainerName + value: $[ coalesce(variables.ContainerName, 'dotnet') ] + - name: _AzureAccessToken + value: $[ coalesce(variables.AzureAccessToken, '$(dotnetcli-storage-key)') ] + - name: _ChecksumAzureAccountName + value: $[ coalesce(variables.ChecksumAzureAccountName, 'dotnetclichecksums') ] + - name: _ChecksumContainerName + value: $[ coalesce(variables.ChecksumContainerName, 'dotnet') ] + - name: _ChecksumAzureAccessToken + value: $[ coalesce(variables.ChecksumAzureAccessToken, '$(dotnetclichecksums-storage-key)') ] + - name: _CommonPublishArgs + value: >- + /p:AzureAccountName=$(_AzureAccountName) + /p:ContainerName=$(_ContainerName) + /p:AzureAccessToken=$(_AzureAccessToken) + /p:ChecksumAzureAccountName=$(_ChecksumAzureAccountName) + /p:ChecksumContainerName=$(_ChecksumContainerName) + /p:ChecksumAzureAccessToken=$(_ChecksumAzureAccessToken) + + # dotnet/versions update + - name: _GitHubUser + value: $[ coalesce(variables.GitHubUser, 'dotnet-build-bot') ] + - name: _GitHubEmail + value: $[ coalesce(variables.GitHubEmail, 'dotnet-build-bot@microsoft.com') ] + - name: _GitHubAuthToken + value: $[ coalesce(variables.GitHubAuthToken, '$(AccessToken-dotnet-build-bot-public-repo)') ] + - name: _VersionsRepoOwner + value: $[ coalesce(variables.VersionsRepoOwner, 'dotnet') ] + - name: _VersionsRepo + value: $[ coalesce(variables.VersionsRepo, 'versions') ] + - name: _DotNetVersionsArgs + value: >- + /p:GitHubUser=$(_GitHubUser) + /p:GitHubEmail=$(_GitHubEmail) + /p:GitHubAuthToken=$(_GitHubAuthToken) + /p:VersionsRepoOwner=$(_VersionsRepoOwner) + /p:VersionsRepo=$(_VersionsRepo) + /p:VersionsRepoPath=build-info/dotnet/core-setup/$(FullBranchName) + + steps: + + - task: DownloadBuildArtifacts@0 + displayName: Download Artifacts + inputs: + artifactName: PreparedArtifacts + downloadPath: $(Build.SourcesDirectory)\artifacts\PackageDownload + + - powershell: | + $prefix = "refs/heads/" + $branch = "$(Build.SourceBranch)" + $branchName = $branch + if ($branchName.StartsWith($prefix)) + { + $branchName = $branchName.Substring($prefix.Length) + } + Write-Host "For Build.SourceBranch $branch, FullBranchName is $branchName" + Write-Host "##vso[task.setvariable variable=FullBranchName;]$branchName" + displayName: Find true SourceBranchName + + - script: >- + build.cmd -ci + -projects $(Build.SourcesDirectory)\publish\${{ parameters.projectName }}.proj + /p:Channel=${{ parameters.dependency.channel.storage }} + /p:Configuration=Release + $(_CommonPublishArgs) + $(_DotNetVersionsArgs) + /bl:$(Build.SourcesDirectory)\${{ parameters.projectName }}.binlog + displayName: Publish to custom locations + + - task: CopyFiles@2 + displayName: Copy Files to $(Build.StagingDirectory)\BuildLogs + inputs: + SourceFolder: '$(Build.SourcesDirectory)' + Contents: | + **/*.log + **/*.binlog + TargetFolder: '$(Build.StagingDirectory)\BuildLogs' + continueOnError: true + condition: succeededOrFailed() + + - task: PublishBuildArtifacts@1 + displayName: Publish Artifact BuildLogs + inputs: + PathtoPublish: '$(Build.StagingDirectory)\BuildLogs' + ArtifactName: Logs-CustomPublish-${{ parameters.dependency.dependsOn }}-${{ parameters.projectName }} + condition: succeededOrFailed() diff --git a/eng/stages/custom-publish.yml b/eng/stages/custom-publish.yml deleted file mode 100644 index 952469077a..0000000000 --- a/eng/stages/custom-publish.yml +++ /dev/null @@ -1,129 +0,0 @@ -parameters: - dependsOnPublishStages: [] - -stages: -# Create one stage per pipeline that needs some extra publish steps. -- ${{ each dependency in parameters.dependsOnPublishStages }}: - - stage: CustomPublish_${{ dependency.dependsOn }} - displayName: 'Custom: ${{ dependency.channel.name }}' - dependsOn: - - ${{ dependency.dependsOn }} - variables: - - template: /eng/common/templates/post-build/common-variables.yml - - jobs: - - - template: /eng/common/templates/post-build/setup-maestro-vars.yml - - - job: CustomPublish - displayName: Custom Publish - dependsOn: setupMaestroVars - # Only run this job if setup-maestro-vars says the current branch publishes to this channel. - # Logic copied from ../common/templates/post-build/channels/netcore-dev-5.yml - condition: contains( - dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], - format('[{0}]', variables['${{ dependency.channel.bar }}'])) - pool: - name: Hosted VS2017 - # Double the default timeout. - timeoutInMinutes: 120 - workspace: - clean: all - - variables: - # Only get the secret variable groups if the def has the official name. Reduce dev build def risk. - - ${{ if eq(variables['Build.DefinitionName'], 'dotnet-core-setup') }}: - # Used for publishing individual leg assets to azure blob storage - - group: DotNet-DotNetCli-Storage - # Used for dotnet/versions update - - group: DotNet-Versions-Publish - - # Blob storage publish (installers and checksums) - - name: _AzureAccountName - value: $[ coalesce(variables.AzureAccountName, 'dotnetcli') ] - - name: _ContainerName - value: $[ coalesce(variables.ContainerName, 'dotnet') ] - - name: _AzureAccessToken - value: $[ coalesce(variables.AzureAccessToken, '$(dotnetcli-storage-key)') ] - - name: _ChecksumAzureAccountName - value: $[ coalesce(variables.ChecksumAzureAccountName, 'dotnetclichecksums') ] - - name: _ChecksumContainerName - value: $[ coalesce(variables.ChecksumContainerName, 'dotnet') ] - - name: _ChecksumAzureAccessToken - value: $[ coalesce(variables.ChecksumAzureAccessToken, '$(dotnetclichecksums-storage-key)') ] - - name: _CommonPublishArgs - value: >- - /p:AzureAccountName=$(_AzureAccountName) - /p:ContainerName=$(_ContainerName) - /p:AzureAccessToken=$(_AzureAccessToken) - /p:ChecksumAzureAccountName=$(_ChecksumAzureAccountName) - /p:ChecksumContainerName=$(_ChecksumContainerName) - /p:ChecksumAzureAccessToken=$(_ChecksumAzureAccessToken) - - # dotnet/versions update - - name: _GitHubUser - value: $[ coalesce(variables.GitHubUser, 'dotnet-build-bot') ] - - name: _GitHubEmail - value: $[ coalesce(variables.GitHubEmail, 'dotnet-build-bot@microsoft.com') ] - - name: _GitHubAuthToken - value: $[ coalesce(variables.GitHubAuthToken, '$(AccessToken-dotnet-build-bot-public-repo)') ] - - name: _VersionsRepoOwner - value: $[ coalesce(variables.VersionsRepoOwner, 'dotnet') ] - - name: _VersionsRepo - value: $[ coalesce(variables.VersionsRepo, 'versions') ] - - name: _DotNetVersionsArgs - value: >- - /p:GitHubUser=$(_GitHubUser) - /p:GitHubEmail=$(_GitHubEmail) - /p:GitHubAuthToken=$(_GitHubAuthToken) - /p:VersionsRepoOwner=$(_VersionsRepoOwner) - /p:VersionsRepo=$(_VersionsRepo) - /p:VersionsRepoPath=build-info/dotnet/core-setup/$(FullBranchName) - - steps: - - - task: DownloadBuildArtifacts@0 - displayName: Download Artifacts - inputs: - artifactName: PreparedArtifacts - downloadPath: $(Build.SourcesDirectory)\artifacts\PackageDownload - - - powershell: | - $prefix = "refs/heads/" - $branch = "$(Build.SourceBranch)" - $branchName = $branch - if ($branchName.StartsWith($prefix)) - { - $branchName = $branchName.Substring($prefix.Length) - } - Write-Host "For Build.SourceBranch $branch, FullBranchName is $branchName" - Write-Host "##vso[task.setvariable variable=FullBranchName;]$branchName" - displayName: Find true SourceBranchName - - - script: >- - build.cmd -ci - -projects $(Build.SourcesDirectory)\publish\custom-publish.proj - /p:Channel=${{ dependency.channel.storage }} - /p:Configuration=Release - $(_CommonPublishArgs) - $(_DotNetVersionsArgs) - /bl:$(Build.SourcesDirectory)\custom-publish.binlog - displayName: Publish to custom locations - - - task: CopyFiles@2 - displayName: Copy Files to $(Build.StagingDirectory)\BuildLogs - inputs: - SourceFolder: '$(Build.SourcesDirectory)' - Contents: | - **/*.log - **/*.binlog - TargetFolder: '$(Build.StagingDirectory)\BuildLogs' - continueOnError: true - condition: succeededOrFailed() - - - task: PublishBuildArtifacts@1 - displayName: Publish Artifact BuildLogs - inputs: - PathtoPublish: '$(Build.StagingDirectory)\BuildLogs' - ArtifactName: Logs-CustomPublish-${{ dependency.dependsOn }} - condition: succeededOrFailed() diff --git a/eng/stages/publish.yml b/eng/stages/publish.yml new file mode 100644 index 0000000000..a059d57dcf --- /dev/null +++ b/eng/stages/publish.yml @@ -0,0 +1,48 @@ +parameters: + dependsOnPublishStages: [] + +stages: + +# Create extra stage per BAR channel that needs extra publish steps. +- ${{ each dependency in parameters.dependsOnPublishStages }}: + - stage: PublishBlob_${{ dependency.dependsOn }} + displayName: '${{ dependency.channel.name }} Blob Publish' + dependsOn: PrepareForPublish + variables: + - template: /eng/common/templates/post-build/common-variables.yml + jobs: + - template: /eng/jobs/run-publish-project.yml + parameters: + projectName: publish-blobs + dependency: ${{ dependency }} + +# Stages-based publishing entry point +- template: /eng/common/templates/post-build/post-build.yml + parameters: + dependsOn: + - ${{ each dependency in parameters.dependsOnPublishStages }}: + - PublishBlob_${{ dependency.dependsOn }} + # Symbol validation is not ready yet. https://github.com/dotnet/arcade/issues/2871 + enableSymbolValidation: false + # Doesn't work yet, not fully configured. https://github.com/dotnet/core-setup/issues/5254 + enableSigningValidation: false + # SourceLink validation doesn't work in dev builds: tries to pull from GitHub. https://github.com/dotnet/arcade/issues/3604 + enableSourceLinkValidation: false + # Allow symbol publish to emit expected warnings without failing the build. Include single + # quotes inside the string so that it passes through to MSBuild without script interference. + symbolPublishingAdditionalParameters: "'-warnAsError:$false'" + +# Create extra stage per BAR channel that needs extra publish steps. These run after the Arcade +# stages because they depend on Arcade's NuGet package publish being complete. +- ${{ each dependency in parameters.dependsOnPublishStages }}: + - stage: PublishFinal_${{ dependency.dependsOn }} + displayName: '${{ dependency.channel.name }} Finalize' + dependsOn: + - ${{ dependency.dependsOn }} + variables: + - template: /eng/common/templates/post-build/common-variables.yml + jobs: + - template: /eng/jobs/run-publish-project.yml + parameters: + projectName: publish-final + dependency: ${{ dependency }} diff --git a/publish/publish-blobs.proj b/publish/publish-blobs.proj new file mode 100644 index 0000000000..ed3b2d67d4 --- /dev/null +++ b/publish/publish-blobs.proj @@ -0,0 +1,75 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/publish/custom-publish.proj b/publish/publish-final.proj similarity index 54% rename from publish/custom-publish.proj rename to publish/publish-final.proj index 5ffaa36c35..ce0089bd6b 100644 --- a/publish/custom-publish.proj +++ b/publish/publish-final.proj @@ -1,11 +1,7 @@ - - - - From 9759caf3969337362729ff21e2602eaa311391ab Mon Sep 17 00:00:00 2001 From: Matt Mitchell Date: Wed, 21 Aug 2019 17:12:41 -0700 Subject: [PATCH 2/2] Enable publish using pipelines --- azure-pipelines.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index a4ea4c4b2e..224c1a9dc1 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -166,6 +166,7 @@ stages: # Publish to Build Asset Registry in order to generate the ReleaseConfigs artifact. - template: /eng/common/templates/job/publish-build-assets.yml parameters: + publishUsingPipelines: true dependsOn: PrepareSignedArtifacts pool: name: NetCoreInternal-Pool