[microsoft/release-branch.go1.22] Port symbol publish and changes necessary to publish to PME

eng/_core/cmd/build/build.go

@@ -45,6 +45,7 @@ func main() {
 	flag.BoolVar(&o.JSON, "json", false, "Runs tests with -json flag to emit verbose results in JSON format. For use in CI.")
 	flag.BoolVar(&o.PackBuild, "packbuild", false, "Enable creating an archive of this build using upstream 'distpack' and placing it in eng/artifacts/bin.")
 	flag.BoolVar(&o.PackSource, "packsource", false, "Enable creating a source archive using upstream 'distpack' and placing it in eng/artifacts/bin.")
+	flag.BoolVar(&o.CreatePDB, "pdb", false, "Create PDB files for all the PE binaries in the bin and tool directories. The PE files are modified in place and PDBs are placed in eng/artifacts/symbols.")
 
 	flag.BoolVar(
 		&o.Refresh, "refresh", false,
@@ -82,6 +83,7 @@ type options struct {
 	JSON       bool
 	PackBuild  bool
 	PackSource bool
+	CreatePDB  bool
 	Refresh    bool
 	Experiment string
 
@@ -237,8 +239,55 @@ func build(o *options) error {
 		}
 	}
 
+	goRootDir := filepath.Join(rootDir, "go")
+	if o.CreatePDB {
+		if _, err := exec.LookPath("gopdb"); err != nil {
+			return fmt.Errorf("gopdb not found in PATH: %v", err)
+		}
+		// Print the version of gopdb to the console.
+		cmd := exec.Command("gopdb", "-version")
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		if err := runCmd(cmd); err != nil {
+			return fmt.Errorf("gopdb failed: %v", err)
+		}
+
+		// Traverse the bin and tool directories to find all the binaries to generate PDBs for.
+		binDir := filepath.Join(goRootDir, "bin")
+		toolsDir := filepath.Join(goRootDir, "pkg", "tool", targetOS+"_"+targetArch)
+		artifactsPDBDir := filepath.Join(rootDir, "eng", "artifacts", "symbols")
+
+		if err := os.MkdirAll(artifactsPDBDir, os.ModePerm); err != nil {
+			return err
+		}
+
+		var bins []string
+		for _, dir := range []string{binDir, toolsDir} {
+			entries, err := os.ReadDir(dir)
+			if err != nil {
+				return err
+			}
+			for _, entry := range entries {
+				if !entry.Type().IsRegular() {
+					continue
+				}
+				bins = append(bins, filepath.Join(dir, entry.Name()))
+			}
+		}
+
+		// Generate PDBs for all the binaries.
+		for _, bin := range bins {
+			out := filepath.Join(artifactsPDBDir, filepath.Base(bin)+"."+targetOS+"-"+targetArch+".pdb")
+			cmd := exec.Command("gopdb", "-o", out, bin)
+			cmd.Stdout = os.Stdout
+			cmd.Stderr = os.Stderr
+			if err := runCmd(cmd); err != nil {
+				return fmt.Errorf("gopdb failed: %v", err)
+			}
+		}
+	}
+
 	if o.PackBuild || o.PackSource {
-		goRootDir := filepath.Join(rootDir, "go")
 		// Find the host version of distpack. (Not the target version, which might not run.)
 		toolsDir := filepath.Join(goRootDir, "pkg", "tool", runtime.GOOS+"_"+runtime.GOARCH)
 		// distpack needs a VERSION file to run. If we're on the main branch, we don't have one, so

eng/pipeline/rolling-internal-pipeline.yml

@@ -24,6 +24,11 @@ parameters:
     type: boolean
     default: false
 
+  - name: publishExistingRunID
+    displayName: 'For debugging publish steps: skip building, and instead publish the artifacts from an existing run. Leave "nil" otherwise.'
+    type: string
+    default: nil
+
 variables:
   - template: variables/pool-providers.yml
   # MicroBuild configuration.
@@ -60,24 +65,7 @@ extends:
           buildandpack: true
           official: true
           createSourceArchive: true
+          createSymbols: true
+          publish: true
+          publishExistingRunID: ${{ parameters.publishExistingRunID }}
           releaseVersion: ${{ parameters.releaseVersion }}
-
-      - ${{ if not(startsWith(variables['Build.SourceBranch'], 'refs/heads/internal/')) }}:
-        - template: stages/pool.yml
-          parameters:
-            inner:
-              template: publish-stage.yml
-              parameters:
-                # This is not a builder, but provide partial builder info for agent selection.
-                builder: { os: linux, arch: amd64 }
-                official: true
-                public: true
-
-      - template: stages/pool.yml
-        parameters:
-          inner:
-            template: publish-stage.yml
-            parameters:
-              builder: { os: linux, arch: amd64 }
-              official: true
-              public: false

eng/pipeline/stages/builders-to-stages.yml

@@ -7,40 +7,76 @@
 parameters:
   # [] of { id, os, arch, hostarch, config, distro?, experiment? }
   builders: []
+  # If true, publish build artifacts to blob storage.
+  publish: false
+  # If changed to specify an existing pipeline run, skip build/sign and publish the existing run.
+  publishExistingRunID: 'nil'
   # If true, include a signing stage+job that depends on all 'buildandpack' builder jobs finishing.
   # 'official' is passed through into run-stage.yml, where it has other effects.
   official: false
   # If true, generate source archive tarballs.
   createSourceArchive: false
+  # If true, generate and publish symbols (aka PDBs).
+  createSymbols: false
   releaseVersion: 'nil'
 
 stages:
-  - ${{ each builder in parameters.builders }}:
-    - template: pool.yml
-      parameters:
-        inner:
-          template: run-stage.yml
-          parameters:
-            builder: ${{ builder }}
-            createSourceArchive: ${{ parameters.createSourceArchive }}
-            releaseVersion: ${{ parameters.releaseVersion }}
-            official: ${{ parameters.official }}
-            # Attempt to retry the build on Windows to mitigate flakiness:
-            # "Access Denied" during EXE copying and general flakiness during tests.
-            ${{ if eq(builder.os, 'windows') }}:
-              retryAttempts: [1, 2, 3, 4, "FINAL"]
+  - ${{ if eq(parameters.publishExistingRunID, 'nil') }}:
+    - ${{ each builder in parameters.builders }}:
+      - template: pool.yml
+        parameters:
+          inner:
+            template: run-stage.yml
+            parameters:
+              builder: ${{ builder }}
+              createSourceArchive: ${{ parameters.createSourceArchive }}
+              releaseVersion: ${{ parameters.releaseVersion }}
+              official: ${{ parameters.official }}
+              createSymbols: ${{ parameters.createSymbols }}
+              # Attempt to retry the build on Windows to mitigate flakiness:
+              # "Access Denied" during EXE copying and general flakiness during tests.
+              ${{ if eq(builder.os, 'windows') }}:
+                retryAttempts: [1, 2, 3, 4, "FINAL"]
+
+    - ${{ if eq(parameters.official, true) }}:
+      - template: pool.yml
+        parameters:
+          inner:
+            template: sign-stage.yml
+            parameters:
+              # This is not a builder, but provide partial builder info for agent selection.
+              builder: { os: windows, arch: amd64 }
+              official: ${{ parameters.official }}
+              # The list of builders to depend on and grab artifacts from.
+              builders:
+                - ${{ each builder in parameters.builders }}:
+                  - ${{ if eq(builder.config, 'buildandpack') }}:
+                    - ${{ builder }}
+
+  - ${{ if eq(parameters.publish, true) }}:
+    - ${{ if not(startsWith(variables['Build.SourceBranch'], 'refs/heads/internal/')) }}:
+      - template: pool.yml
+        parameters:
+          inner:
+            template: publish-stage.yml
+            parameters:
+              # This is not a builder, but provide partial builder info for agent selection.
+              builder: { os: windows, arch: amd64 }
+              official: true
+              public: true
+              publishExistingRunID: ${{ parameters.publishExistingRunID }}
 
-  - ${{ if eq(parameters.official, true) }}:
     - template: pool.yml
       parameters:
         inner:
-          template: sign-stage.yml
+          template: publish-stage.yml
           parameters:
-            # This is not a builder, but provide partial builder info for agent selection.
             builder: { os: windows, arch: amd64 }
-            official: ${{ parameters.official }}
-            # The list of builders to depend on and grab artifacts from.
+            official: true
+            public: false
             builders:
               - ${{ each builder in parameters.builders }}:
                 - ${{ if eq(builder.config, 'buildandpack') }}:
                   - ${{ builder }}
+            publishSymbols: ${{ parameters.createSymbols }}
+            publishExistingRunID: ${{ parameters.publishExistingRunID }}

eng/pipeline/stages/go-builder-matrix-stages.yml

@@ -28,20 +28,32 @@ parameters:
   - name: official
     type: boolean
     default: false
+  - name: publish
+    type: boolean
+    default: false
   - name: createSourceArchive
     type: boolean
     default: false
+  - name: createSymbols
+    type: boolean
+    default: false
   - name: releaseVersion
     type: string
     default: 'nil'
+  - name: publishExistingRunID
+    type: string
+    default: 'nil'
 
 stages:
   - template: shorthand-builders-to-builders.yml
     parameters:
       jobsTemplate: builders-to-stages.yml
       jobsParameters:
         official: ${{ parameters.official }}
+        publish: ${{ parameters.publish }}
+        publishExistingRunID: ${{ parameters.publishExistingRunID }}
         createSourceArchive: ${{ parameters.createSourceArchive }}
+        createSymbols: ${{ parameters.createSymbols }}
         releaseVersion: ${{ parameters.releaseVersion }}
       shorthandBuilders:
         # Individually enable buildandpack.