Skip to content

Commit

Permalink
Merge pull request #1245 from microsoft/dev/dagood/port-pme-1.22
Browse files Browse the repository at this point in the history
[microsoft/release-branch.go1.22] Port symbol publish and changes necessary to publish to PME
karianna authored Jun 10, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature.
2 parents c7beadf + 044d5b5 commit 9d90c61
Showing 7 changed files with 305 additions and 77 deletions.
51 changes: 50 additions & 1 deletion eng/_core/cmd/build/build.go
Original file line number Diff line number Diff line change
@@ -45,6 +45,7 @@ func main() {
flag.BoolVar(&o.JSON, "json", false, "Runs tests with -json flag to emit verbose results in JSON format. For use in CI.")
flag.BoolVar(&o.PackBuild, "packbuild", false, "Enable creating an archive of this build using upstream 'distpack' and placing it in eng/artifacts/bin.")
flag.BoolVar(&o.PackSource, "packsource", false, "Enable creating a source archive using upstream 'distpack' and placing it in eng/artifacts/bin.")
flag.BoolVar(&o.CreatePDB, "pdb", false, "Create PDB files for all the PE binaries in the bin and tool directories. The PE files are modified in place and PDBs are placed in eng/artifacts/symbols.")

flag.BoolVar(
&o.Refresh, "refresh", false,
@@ -82,6 +83,7 @@ type options struct {
JSON bool
PackBuild bool
PackSource bool
CreatePDB bool
Refresh bool
Experiment string

@@ -237,8 +239,55 @@ func build(o *options) error {
}
}

goRootDir := filepath.Join(rootDir, "go")
if o.CreatePDB {
if _, err := exec.LookPath("gopdb"); err != nil {
return fmt.Errorf("gopdb not found in PATH: %v", err)
}
// Print the version of gopdb to the console.
cmd := exec.Command("gopdb", "-version")
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if err := runCmd(cmd); err != nil {
return fmt.Errorf("gopdb failed: %v", err)
}

// Traverse the bin and tool directories to find all the binaries to generate PDBs for.
binDir := filepath.Join(goRootDir, "bin")
toolsDir := filepath.Join(goRootDir, "pkg", "tool", targetOS+"_"+targetArch)
artifactsPDBDir := filepath.Join(rootDir, "eng", "artifacts", "symbols")

if err := os.MkdirAll(artifactsPDBDir, os.ModePerm); err != nil {
return err
}

var bins []string
for _, dir := range []string{binDir, toolsDir} {
entries, err := os.ReadDir(dir)
if err != nil {
return err
}
for _, entry := range entries {
if !entry.Type().IsRegular() {
continue
}
bins = append(bins, filepath.Join(dir, entry.Name()))
}
}

// Generate PDBs for all the binaries.
for _, bin := range bins {
out := filepath.Join(artifactsPDBDir, filepath.Base(bin)+"."+targetOS+"-"+targetArch+".pdb")
cmd := exec.Command("gopdb", "-o", out, bin)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if err := runCmd(cmd); err != nil {
return fmt.Errorf("gopdb failed: %v", err)
}
}
}

if o.PackBuild || o.PackSource {
goRootDir := filepath.Join(rootDir, "go")
// Find the host version of distpack. (Not the target version, which might not run.)
toolsDir := filepath.Join(goRootDir, "pkg", "tool", runtime.GOOS+"_"+runtime.GOARCH)
// distpack needs a VERSION file to run. If we're on the main branch, we don't have one, so
28 changes: 8 additions & 20 deletions eng/pipeline/rolling-internal-pipeline.yml
Original file line number Diff line number Diff line change
@@ -24,6 +24,11 @@ parameters:
type: boolean
default: false

- name: publishExistingRunID
displayName: 'For debugging publish steps: skip building, and instead publish the artifacts from an existing run. Leave "nil" otherwise.'
type: string
default: nil

variables:
- template: variables/pool-providers.yml
# MicroBuild configuration.
@@ -60,24 +65,7 @@ extends:
buildandpack: true
official: true
createSourceArchive: true
createSymbols: true
publish: true
publishExistingRunID: ${{ parameters.publishExistingRunID }}
releaseVersion: ${{ parameters.releaseVersion }}

- ${{ if not(startsWith(variables['Build.SourceBranch'], 'refs/heads/internal/')) }}:
- template: stages/pool.yml
parameters:
inner:
template: publish-stage.yml
parameters:
# This is not a builder, but provide partial builder info for agent selection.
builder: { os: linux, arch: amd64 }
official: true
public: true

- template: stages/pool.yml
parameters:
inner:
template: publish-stage.yml
parameters:
builder: { os: linux, arch: amd64 }
official: true
public: false
74 changes: 55 additions & 19 deletions eng/pipeline/stages/builders-to-stages.yml
Original file line number Diff line number Diff line change
@@ -7,40 +7,76 @@
parameters:
# [] of { id, os, arch, hostarch, config, distro?, experiment? }
builders: []
# If true, publish build artifacts to blob storage.
publish: false
# If changed to specify an existing pipeline run, skip build/sign and publish the existing run.
publishExistingRunID: 'nil'
# If true, include a signing stage+job that depends on all 'buildandpack' builder jobs finishing.
# 'official' is passed through into run-stage.yml, where it has other effects.
official: false
# If true, generate source archive tarballs.
createSourceArchive: false
# If true, generate and publish symbols (aka PDBs).
createSymbols: false
releaseVersion: 'nil'

stages:
- ${{ each builder in parameters.builders }}:
- template: pool.yml
parameters:
inner:
template: run-stage.yml
parameters:
builder: ${{ builder }}
createSourceArchive: ${{ parameters.createSourceArchive }}
releaseVersion: ${{ parameters.releaseVersion }}
official: ${{ parameters.official }}
# Attempt to retry the build on Windows to mitigate flakiness:
# "Access Denied" during EXE copying and general flakiness during tests.
${{ if eq(builder.os, 'windows') }}:
retryAttempts: [1, 2, 3, 4, "FINAL"]
- ${{ if eq(parameters.publishExistingRunID, 'nil') }}:
- ${{ each builder in parameters.builders }}:
- template: pool.yml
parameters:
inner:
template: run-stage.yml
parameters:
builder: ${{ builder }}
createSourceArchive: ${{ parameters.createSourceArchive }}
releaseVersion: ${{ parameters.releaseVersion }}
official: ${{ parameters.official }}
createSymbols: ${{ parameters.createSymbols }}
# Attempt to retry the build on Windows to mitigate flakiness:
# "Access Denied" during EXE copying and general flakiness during tests.
${{ if eq(builder.os, 'windows') }}:
retryAttempts: [1, 2, 3, 4, "FINAL"]

- ${{ if eq(parameters.official, true) }}:
- template: pool.yml
parameters:
inner:
template: sign-stage.yml
parameters:
# This is not a builder, but provide partial builder info for agent selection.
builder: { os: windows, arch: amd64 }
official: ${{ parameters.official }}
# The list of builders to depend on and grab artifacts from.
builders:
- ${{ each builder in parameters.builders }}:
- ${{ if eq(builder.config, 'buildandpack') }}:
- ${{ builder }}

- ${{ if eq(parameters.publish, true) }}:
- ${{ if not(startsWith(variables['Build.SourceBranch'], 'refs/heads/internal/')) }}:
- template: pool.yml
parameters:
inner:
template: publish-stage.yml
parameters:
# This is not a builder, but provide partial builder info for agent selection.
builder: { os: windows, arch: amd64 }
official: true
public: true
publishExistingRunID: ${{ parameters.publishExistingRunID }}

- ${{ if eq(parameters.official, true) }}:
- template: pool.yml
parameters:
inner:
template: sign-stage.yml
template: publish-stage.yml
parameters:
# This is not a builder, but provide partial builder info for agent selection.
builder: { os: windows, arch: amd64 }
official: ${{ parameters.official }}
# The list of builders to depend on and grab artifacts from.
official: true
public: false
builders:
- ${{ each builder in parameters.builders }}:
- ${{ if eq(builder.config, 'buildandpack') }}:
- ${{ builder }}
publishSymbols: ${{ parameters.createSymbols }}
publishExistingRunID: ${{ parameters.publishExistingRunID }}
12 changes: 12 additions & 0 deletions eng/pipeline/stages/go-builder-matrix-stages.yml
Original file line number Diff line number Diff line change
@@ -28,20 +28,32 @@ parameters:
- name: official
type: boolean
default: false
- name: publish
type: boolean
default: false
- name: createSourceArchive
type: boolean
default: false
- name: createSymbols
type: boolean
default: false
- name: releaseVersion
type: string
default: 'nil'
- name: publishExistingRunID
type: string
default: 'nil'

stages:
- template: shorthand-builders-to-builders.yml
parameters:
jobsTemplate: builders-to-stages.yml
jobsParameters:
official: ${{ parameters.official }}
publish: ${{ parameters.publish }}
publishExistingRunID: ${{ parameters.publishExistingRunID }}
createSourceArchive: ${{ parameters.createSourceArchive }}
createSymbols: ${{ parameters.createSymbols }}
releaseVersion: ${{ parameters.releaseVersion }}
shorthandBuilders:
# Individually enable buildandpack.
160 changes: 124 additions & 36 deletions eng/pipeline/stages/publish-stage.yml
Original file line number Diff line number Diff line change
@@ -11,37 +11,47 @@ parameters:
- name: pool
type: object

- name: publishExistingRunID
type: string
default: 'nil'

# Unused. Declared so pool selection doesn't fail when trying to pass them.
- name: builder
type: object
- name: official
type: boolean
- name: builders
type: object
default: []
- name: publishSymbols
type: boolean
default: false

stages:
- stage: Publish${{ parameters.public }}
${{ if parameters.public }}:
displayName: Publish Public
${{ else }}:
displayName: Publish Internal
dependsOn: Sign
${{ if eq(parameters.publishExistingRunID, 'nil') }}:
dependsOn: Sign
${{ else }}:
dependsOn: []
jobs:
- job: Publish
pool: ${{ parameters.pool }}

variables:
- ${{ if parameters.public }}:
- name: blobContainer
- name: blobContainer
${{ if parameters.public }}:
value: 'https://dotnetbuildoutput.blob.core.windows.net/golang/microsoft'
- name: blobSASArg
value: --sas-token '$(dotnetbuildoutput-golang-write-sas-query)'
- ${{ else }}:
- name: blobContainer
${{ else }}:
value: 'https://golangartifacts.blob.core.windows.net/microsoft'
- name: blobSASArg
value: '' # golangartifacts is set up with service connection auth.

- name: blobPrefix
value: '$(PublishBranchAlias)/$(Build.BuildNumber)'
- name: blobDestinationUrl
value: '$(blobContainer)/$(PublishBranchAlias)/$(Build.BuildNumber)'
value: '$(blobContainer)/$(blobPrefix)'

- group: go-storage

@@ -60,10 +70,16 @@ stages:
artifact: BuildAssets
${{ else }}:
artifact: BuildAssetsInternal
- ${{ if parameters.publishSymbols }}:
- output: pipelineArtifact
path: $(Pipeline.Workspace)/Symbols
${{ if parameters.public }}:
artifact: Symbols
${{ else }}:
artifact: SymbolsInternal

steps:
- template: ../steps/checkout-unix-task.yml
- template: ../steps/init-pwsh-task.yml
- template: ../steps/checkout-windows-task.yml
- template: ../steps/init-submodule-task.yml

- pwsh: |
@@ -89,11 +105,25 @@ stages:
Write-Host "##vso[task.setvariable variable=PublishBranchAlias;]$branch"
displayName: Find publish branch alias
- download: current
artifact: Binaries Signed
# Filter out manifests added by 1ES pipeline template.
patterns: '!_manifest/**'
displayName: 'Download: Binaries Signed'
- ${{ if eq(parameters.publishExistingRunID, 'nil') }}:
- download: current
artifact: Binaries Signed
# Filter out manifests added by 1ES pipeline template.
patterns: '!_manifest/**'
displayName: 'Download: Binaries Signed'
- ${{ else }}:
- task: DownloadPipelineArtifact@2
displayName: 'Download: Binaries Signed (Specific)'
inputs:
buildType: specific
project: $(System.TeamProject)
definition: $(System.DefinitionId)
runVersion: 'specific'
runId: ${{ parameters.publishExistingRunID }}
artifact: Binaries Signed
# Filter out manifests added by 1ES pipeline template.
patterns: '!_manifest/**'
targetPath: '$(Pipeline.Workspace)/Binaries Signed'

- pwsh: |
eng/run.ps1 createbuildassetjson `
@@ -104,23 +134,81 @@ stages:
-o '$(Pipeline.Workspace)/Binaries Signed/assets.json'
displayName: 'Create build asset JSON'
- task: AzureCLI@2
displayName: Upload to blob storage
inputs:
azureSubscription: GoLang
scriptType: bash
scriptLocation: inlineScript
# Send literal '*' to az: it handles the wildcard itself. Az copy only accepts one
# "from" argument, so we can't use the shell's wildcard expansion.
inlineScript: |
az storage copy -s '*' -d '$(blobDestinationUrl)' $(blobSASArg)
workingDirectory: '$(Pipeline.Workspace)/Binaries Signed/'

- script: |
echo 'Generated links to artifacts in blob storage:'
echo ''
for f in *; do
echo "$(blobDestinationUrl)/$f"
done
displayName: Show uploaded URLs
- ${{ if parameters.public }}:
- task: AzureCLI@2
displayName: Upload to blob storage
inputs:
azureSubscription: GoLang
scriptType: bash
scriptLocation: inlineScript
# Send literal '*' to az: it handles the wildcard itself. Az copy only accepts one
# "from" argument, so we can't use the shell's wildcard expansion.
inlineScript: |
az storage copy -s '*' -d '$(blobDestinationUrl)' --sas-token '$(dotnetbuildoutput-golang-write-sas-query)'
workingDirectory: '$(Pipeline.Workspace)/Binaries Signed/'
- ${{ else }}:
- task: AzureFileCopy@6
displayName: Upload to blob storage
inputs:
Destination: AzureBlob
azureSubscription: golang-pme-storage
storage: golangartifactsbackup
ContainerName: microsoft
SourcePath: '$(Pipeline.Workspace)/Binaries Signed/*'
BlobPrefix: $(blobPrefix)

- pwsh: |
Write-Host 'Generated links to artifacts in blob storage:'
Write-Host ''
Get-ChildItem -File -Path '.' | %{
Write-Host "$(blobDestinationUrl)/$($_.Name)"
}
displayName: Show expected uploaded URLs
workingDirectory: '$(Pipeline.Workspace)/Binaries Signed/'
- ${{ if eq(parameters.publishSymbols, true) }}:
- ${{ each builder in parameters.builders }}:
- ${{ if eq(parameters.publishExistingRunID, 'nil') }}:
- download: current
artifact: Symbols ${{ builder.id }}
# Filter out manifests added by 1ES pipeline template.
patterns: '!_manifest/**'
displayName: 'Download: Symbols ${{ builder.id }}'
- ${{ else }}:
- task: DownloadPipelineArtifact@2
displayName: 'Download: Symbols ${{ builder.id }} (Specific)'
inputs:
buildType: specific
project: $(System.TeamProject)
definition: $(System.DefinitionId)
runVersion: 'specific'
runId: ${{ parameters.publishExistingRunID }}
artifact: Symbols ${{ builder.id }}
# Filter out manifests added by 1ES pipeline template.
patterns: '!_manifest/**'
targetPath: '$(Pipeline.Workspace)/Symbols ${{ builder.id }}'

- pwsh: |
$flatDir = "$(Pipeline.Workspace)/Symbols"
New-Item $flatDir -ItemType Directory -ErrorAction Ignore
Get-ChildItem -Recurse -File -Path @(
'Symbols ${{ builder.id }}'
) | %{
if (Test-Path "$flatDir\$($_.Name)") {
throw "Duplicate filename, unable to flatten: $($_.FullName)"
}
Copy-Item $_.FullName $flatDir
}
displayName: 'Flatten: Symbols ${{ builder.id }}'
workingDirectory: '$(Pipeline.Workspace)'
- task: PublishSymbols@2
inputs:
SymbolsFolder: $(Pipeline.Workspace)/Symbols
SearchPattern: '*.pdb'
SymbolServerType: TeamServices
# Source indexing doesn't work for us. It needs the source files to be available
# in the AzDO repo, but we pull them at build time using a git submodule.
# See https://github.com/microsoft/go-lab/issues/67.
IndexSources: false
displayName: Publish symbols
21 changes: 20 additions & 1 deletion eng/pipeline/stages/run-stage.yml
Original file line number Diff line number Diff line change
@@ -13,6 +13,10 @@ parameters:
type: boolean
default: false

- name: createSymbols
type: boolean
default: false

- name: releaseVersion
type: string
default: 'nil'
@@ -64,6 +68,8 @@ stages:

variables:
- group: go-cmdscan-rules
- name: createPDB
value: ${{ and(eq(parameters.createSymbols, true), eq(parameters.builder.config, 'buildandpack'), eq(parameters.builder.os, 'windows')) }} # Only create PDBs on Windows

${{ if and(parameters.official, eq(parameters.builder.config, 'buildandpack')) }}:
templateContext:
@@ -72,6 +78,9 @@ stages:
- output: pipelineArtifact
path: eng/artifacts/bin
artifact: Binaries ${{ parameters.builder.id }}
- output: pipelineArtifact
path: eng/artifacts/symbols
artifact: Symbols ${{ parameters.builder.id }}

steps:
- ${{ if eq(parameters.builder.os, 'linux') }}:
@@ -88,6 +97,11 @@ stages:
- template: ../steps/checkout-unix-task.yml
- template: ../steps/init-pwsh-task.yml

- pwsh: |
New-Item eng/artifacts/bin -ItemType Directory -ErrorAction Ignore
New-Item eng/artifacts/symbols -ItemType Directory -ErrorAction Ignore
displayName: Create artifact directories
- ${{ if eq(parameters.builder.os, 'windows') }}:
- template: ../steps/checkout-windows-task.yml
@@ -121,16 +135,21 @@ stages:
- pwsh: Write-Host "##vso[task.setvariable variable=GOARM]6"
displayName: Set GOARM for cross-compile

- ${{ if eq(variables.createPDB, true) }}:
- template: ../steps/install-gopdb.yml

# Use build script directly for "buildandpack". If we used run-builder, we would need to
# download its external module dependencies.
- ${{ if eq(parameters.builder.config, 'buildandpack' ) }}:
- pwsh: |
eng/run.ps1 cmdscan -envprefix GO_CMDSCAN_RULE_ -- `
pwsh eng/run.ps1 build -packbuild $env:PACK_SOURCE_ARG
pwsh eng/run.ps1 build -packbuild $env:PACK_SOURCE_ARG $env:CREATE_PDB_ARG
env:
# Generate the source archive on one job only. The os choice is arbitrary.
${{ if and(eq(parameters.createSourceArchive, true), eq(parameters.builder.config, 'buildandpack'), eq(parameters.builder.os, 'linux'), eq(parameters.builder.arch, 'amd64')) }}:
PACK_SOURCE_ARG: '-packsource'
${{ if eq(variables.createPDB, true) }}:
CREATE_PDB_ARG: '-pdb'
displayName: Build and Pack
# We want to create a checksum as early as possible, but Windows signing involves
36 changes: 36 additions & 0 deletions eng/pipeline/steps/install-gopdb.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
# Copyright (c) Microsoft Corporation.
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.

# go-pdb is experimental and not available publicly.

parameters:
- name: version
type: string
default: 0.2.1

steps:
- pwsh: |
$pdbPath = "$(System.ArtifactsDirectory)/gopdb"
New-Item $pdbPath -ItemType Directory -ErrorAction Ignore
Write-Host "##vso[task.setvariable variable=pdbPath;]$pdbPath"
Write-Host "##vso[task.prependpath]$pdbPath"
displayName: Set up gopdb path
# Clone the go-pdb repo from the AzDO mirror, the bot doesn't have access to the GitHub repo.
# Use "git clone" instead of "go install", the later doesn't work because the module name points to GitHub, not AzDO.
- pwsh: |
git -c http.extraheader="AUTHORIZATION: bearer $(System.AccessToken)" `
clone --depth 1 --branch v${{ parameters.version }} https://dev.azure.com/dnceng/internal/_git/microsoft-go-pdb go-pdb
displayName: Clone gopdb
- pwsh: |
. eng/utilities.ps1
$gobin = Get-Stage0GoRoot # Make sure we have a Go toolchain available
cd go-pdb
& $gobin/bin/go.exe build -o $(pdbPath)/gopdb.exe ./cmd/gopdb
displayName: Install gopdb
- pwsh: |
Remove-Item -Path go-pdb -Force -Recurse
displayName: Cleanup gopdb

0 comments on commit 9d90c61

Please sign in to comment.