internal/task: generate release artifacts in vscode-go prerelease

Use RunCustomSteps method to trigger a few steps to build vscode-go release artifacts: - download go binary. - clone vscode-go repo and checkout specific commit. - execute npm ci and go run to build artifacts. - copy generated package extension and logs to gcs. The steps after this will use the returned CloudBuild struct (containing cloud build ID, project, gcs url) to fetch the artifacts and upload as github release assets. A local relui screenshot is at golang/vscode-go#3500 (comment) For golang/vscode-go#3500 Change-Id: Ia2987e1367cd6d76e019b4e576b6cc3e878ac751 Reviewed-on: https://go-review.googlesource.com/c/build/+/611945 LUCI-TryBot-Result: Go LUCI <[email protected]> Reviewed-by: Dmitri Shuralyov <[email protected]> Reviewed-by: Hyang-Ah Hana Kim <[email protected]>
golang · Sep 16, 2024 · d68cbf0 · d68cbf0
1 parent e0ba883
commit d68cbf0
Show file tree

Hide file tree

Showing 4 changed files with 194 additions and 13 deletions.
diff --git a/internal/task/cloudbuild.go b/internal/task/cloudbuild.go
@@ -78,25 +78,26 @@ func (c *RealCloudBuildClient) RunBuildTrigger(ctx context.Context, project, tri
 	return CloudBuild{Project: project, ID: meta.Build.Id}, nil
 }
 
-func (c *RealCloudBuildClient) RunScript(ctx context.Context, script string, gerritProject string, outputs []string) (CloudBuild, error) {
-	const downloadGoScript = `#!/usr/bin/env bash
+const cloudBuildClientScriptPrefix = `#!/usr/bin/env bash
+set -eux
+set -o pipefail
+export PATH=/workspace/released_go/bin:$PATH
+`
+
+const cloudBuildClientDownloadGoScript = `#!/usr/bin/env bash
 set -eux
 archive=$(wget -qO - 'https://go.dev/dl/?mode=json' | grep -Eo 'go.*linux-amd64.tar.gz' | head -n 1)
 wget -qO - https://go.dev/dl/${archive} | tar -xz
 mv go /workspace/released_go
-`
-	const scriptPrefix = `#!/usr/bin/env bash
-set -eux
-set -o pipefail
-export PATH=/workspace/released_go/bin:$PATH
 `
 
+func (c *RealCloudBuildClient) RunScript(ctx context.Context, script string, gerritProject string, outputs []string) (CloudBuild, error) {
 	steps := func(resultURL string) []*cloudbuildpb.BuildStep {
 		// Cloud build loses directory structure when it saves artifacts, which is
 		// a problem since (e.g.) we have multiple files named go.mod in the
 		// tagging tasks. It's not very complicated, so reimplement it ourselves.
 		var saveOutputsScript strings.Builder
-		saveOutputsScript.WriteString(scriptPrefix)
+		saveOutputsScript.WriteString(cloudBuildClientScriptPrefix)
 		for _, out := range outputs {
 			saveOutputsScript.WriteString(fmt.Sprintf("gsutil cp %q %q\n", out, resultURL+"/"+strings.TrimPrefix(out, "./")))
 		}
@@ -113,11 +114,11 @@ export PATH=/workspace/released_go/bin:$PATH
 		steps = append(steps,
 			&cloudbuildpb.BuildStep{
 				Name:   "bash",
-				Script: downloadGoScript,
+				Script: cloudBuildClientDownloadGoScript,
 			},
 			&cloudbuildpb.BuildStep{
 				Name:   "gcr.io/cloud-builders/gsutil",
-				Script: scriptPrefix + script,
+				Script: cloudBuildClientScriptPrefix + script,
 				Dir:    dir,
 			},
 			&cloudbuildpb.BuildStep{

diff --git a/internal/task/fakes.go b/internal/task/fakes.go
@@ -632,9 +632,9 @@ case "$1" in
 esac
 `
 
-const fakeChown = `
+const fakeBinary = `
 #!/bin/bash -eux
-echo "chown change owner successful"
+echo "this binary will always exit without any error"
 exit 0
 `
 
@@ -646,7 +646,10 @@ func NewFakeCloudBuild(t *testing.T, gerrit *FakeGerrit, project string, allowed
 	if err := os.WriteFile(filepath.Join(toolDir, "gsutil"), []byte(fakeGsutil), 0777); err != nil {
 		t.Fatal(err)
 	}
-	if err := os.WriteFile(filepath.Join(toolDir, "chown"), []byte(fakeChown), 0777); err != nil {
+	if err := os.WriteFile(filepath.Join(toolDir, "chown"), []byte(fakeBinary), 0777); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(toolDir, "npm"), []byte(fakeBinary), 0777); err != nil {
 		t.Fatal(err)
 	}
 	return &FakeCloudBuild{
@@ -756,6 +759,14 @@ func (cb *FakeCloudBuild) RunCustomSteps(ctx context.Context, steps func(resultU
 	var gerritProject, fullScript string
 	resultURL := "file://" + cb.t.TempDir()
 	for _, step := range steps(resultURL) {
+		// Cloud Build support docker hub images like "bash". See more details:
+		// https://cloud.google.com/build/docs/interacting-with-dockerhub-images
+		// Currently, the Bash script is solely for downloading the Go binary.
+		// The RunScripts mock implementation provides the Go binary, allowing us
+		// to bypass the Bash script for now.
+		if step.Name == "bash" && step.Script == cloudBuildClientDownloadGoScript {
+			continue
+		}
 		tool, found := strings.CutPrefix(step.Name, "gcr.io/cloud-builders/")
 		if !found {
 			return CloudBuild{}, fmt.Errorf("does not support custom image: %s", step.Name)

diff --git a/internal/task/releasevscodego.go b/internal/task/releasevscodego.go
@@ -141,6 +141,8 @@ func (r *ReleaseVSCodeGoTasks) NewPrereleaseDefinition() *wf.Definition {
 
 	_ = wf.Task1(wd, "create release milestone and issue", r.createReleaseMilestoneAndIssue, release, wf.After(verified))
 	branched := wf.Action2(wd, "create release branch", r.createReleaseBranch, release, prerelease, wf.After(verified))
+	_ = wf.Task3(wd, "generate package extension (.vsix) for release candidate", r.generatePackageExtension, release, prerelease, revision, wf.After(verified))
+
 	_ = wf.Action3(wd, "tag release candidate", r.tag, revision, release, prerelease, wf.After(branched))
 
 	return wd
@@ -291,6 +293,71 @@ func (r *ReleaseVSCodeGoTasks) createReleaseBranch(ctx *wf.TaskContext, release
 	return nil
 }
 
+func (r *ReleaseVSCodeGoTasks) generatePackageExtension(ctx *wf.TaskContext, release releaseVersion, prerelease, revision string) (CloudBuild, error) {
+	steps := func(resultURL string) []*cloudbuildpb.BuildStep {
+		const packageScriptFmt = cloudBuildClientScriptPrefix + `
+export TAG_NAME=%s
+
+npm ci &> npm-output.log
+go run tools/release/release.go package &> go-output.log
+cat npm-output.log
+cat go-output.log
+`
+
+		versionString := release.String()
+		if prerelease != "" {
+			versionString += "-" + prerelease
+		}
+		// The version inside of vsix does not have prefix "v".
+		vsix := fmt.Sprintf("go-%s.vsix", versionString[1:])
+
+		saveScript := cloudBuildClientScriptPrefix
+		for _, file := range []string{"npm-output.log", "go-output.log", vsix} {
+			saveScript += fmt.Sprintf("gsutil cp %s %s/%s\n", file, resultURL, file)
+		}
+		return []*cloudbuildpb.BuildStep{
+			{
+				Name:   "bash",
+				Script: cloudBuildClientDownloadGoScript,
+			},
+			{
+				Name: "gcr.io/cloud-builders/git",
+				Args: []string{"clone", "https://go.googlesource.com/vscode-go", "vscode-go"},
+			},
+			{
+				Name: "gcr.io/cloud-builders/git",
+				Args: []string{"checkout", revision},
+				Dir:  "vscode-go",
+			},
+			{
+				Name:   "gcr.io/cloud-builders/npm",
+				Script: fmt.Sprintf(packageScriptFmt, versionString),
+				Dir:    "vscode-go/extension",
+			},
+			{
+				Name:   "gcr.io/cloud-builders/gsutil",
+				Script: saveScript,
+				Dir:    "vscode-go/extension",
+			},
+		}
+	}
+
+	build, err := r.CloudBuild.RunCustomSteps(ctx, steps)
+	if err != nil {
+		return CloudBuild{}, err
+	}
+
+	outputs, err := buildToOutputs(ctx, r.CloudBuild, build)
+	if err != nil {
+		return CloudBuild{}, err
+	}
+
+	ctx.Printf("the output from npm ci:\n%s\n", outputs["npm-output.log"])
+	ctx.Printf("the output from package generation:\n%s\n", outputs["go-output.log"])
+
+	return build, nil
+}
+
 // determineReleaseVersion determines the release version for the upcoming
 // stable release of vscode-go by examining all existing tags in the repository.
 //

diff --git a/internal/task/releasevscodego_test.go b/internal/task/releasevscodego_test.go
@@ -7,6 +7,7 @@ package task
 import (
 	"context"
 	"fmt"
+	"io"
 	"testing"
 
 	"github.com/google/go-github/v48/github"
@@ -425,6 +426,107 @@ esac
 	}
 }
 
+func TestGeneratePackageExtension(t *testing.T) {
+	mustHaveShell(t)
+	testcases := []struct {
+		name       string
+		release    releaseVersion
+		prerelease string
+		rc         int
+		wantErr    bool
+	}{
+		{
+			name:       "test failed, return error",
+			release:    releaseVersion{0, 1, 0},
+			prerelease: "rc-1",
+			rc:         1,
+			wantErr:    true,
+		},
+		{
+			name:       "test passed, return nil",
+			release:    releaseVersion{0, 2, 3},
+			prerelease: "",
+			rc:         0,
+			wantErr:    false,
+		},
+	}
+
+	for _, tc := range testcases {
+		t.Run(tc.name, func(t *testing.T) {
+			vscodego := NewFakeRepo(t, "vscode-go")
+			commit := vscodego.Commit(map[string]string{
+				"go.mod":                             "module github.com/golang/vscode-go\n",
+				"go.sum":                             "\n",
+				"extension/tools/release/release.go": "foo",
+			})
+
+			gerrit := NewFakeGerrit(t, vscodego)
+			ctx := &workflow.TaskContext{
+				Context: context.Background(),
+				Logger:  &testLogger{t, ""},
+			}
+
+			version := tc.release.String()[1:]
+			if tc.prerelease != "" {
+				version += tc.prerelease
+			}
+			// fakeGo write "bar" content to go-${version}.vsix file and "foo" content
+			// to README.md when executed go run tools/release/release.go.
+			var fakeGo = fmt.Sprintf(`#!/bin/bash -exu
+
+case "$1" in
+"run")
+	echo "writing content to vsix and README.md"
+	echo -n "bar" > go-%s.vsix
+	echo -n "foo" > README.md
+	exit %v
+	;;
+*)
+	echo unexpected command $@
+	exit 1
+	;;
+esac
+`, version, tc.rc)
+
+			cloudbuild := NewFakeCloudBuild(t, gerrit, "vscode-go", nil, fakeGo)
+			tasks := &ReleaseVSCodeGoTasks{
+				Gerrit:     gerrit,
+				CloudBuild: cloudbuild,
+			}
+
+			cb, err := tasks.generatePackageExtension(ctx, tc.release, tc.prerelease, commit)
+			if tc.wantErr && err == nil {
+				t.Errorf("generateArtifacts(%s, %s, %s) should return error but return nil", tc.release, tc.prerelease, commit)
+			} else if !tc.wantErr && err != nil {
+				t.Errorf("generateArtifacts(%s, %s, %s) should return nil but return err: %v", tc.release, tc.prerelease, commit, err)
+			}
+
+			if !tc.wantErr {
+				path := fmt.Sprintf("go-%s.vsix", version)
+				resultFS, err := cloudbuild.ResultFS(ctx, cb)
+				if err != nil {
+					t.Fatal(err)
+				}
+
+				f, err := resultFS.Open(path)
+				if err != nil {
+					t.Fatal(err)
+				}
+				defer f.Close()
+
+				got, err := io.ReadAll(f)
+				if err != nil {
+					t.Fatal(err)
+				}
+
+				if string(got) != "bar" {
+					t.Errorf("generateArtifacts(%s, %s, %s) write content %q to %s, want %q", tc.release, tc.prerelease, commit, got, path, "bar")
+				}
+			}
+		})
+	}
+}
+
 func TestDetermineInsiderVersion(t *testing.T) {
 	testcases := []struct {
 		name         string