Merge pull request #1287 from microsoft/dev/dagood/go-simpler-eng

Merge _core into _util, adding dependency tests

.github/dependabot.yml

@@ -9,11 +9,6 @@ updates:
   schedule:
     interval: daily
   open-pull-requests-limit: 10
-- package-ecosystem: gomod
-  directory: "eng/_core"
-  schedule:
-    interval: daily
-  open-pull-requests-limit: 10
 - package-ecosystem: gomod
   directory: "eng/_util"
   schedule:

eng/_util/README.md

@@ -1,9 +1,40 @@
 ## `github.com/microsoft/go/_util`
 
 This module is a set of utilities Microsoft uses to build Go in Azure DevOps and
-maintain this repository. Run `eng/run.ps1` to list the available commands and
-see instructions on how to use them.
+maintain this repository. Run `eng/run.ps1 build -h` to list available build
+options, or `eng/run.ps1` to list all commands in this module.
 
-The `_util` module requires the `gotestsum` library and doesn't vendor it.
-`_util` is not strictly necessary to build Go, so it's ok if its dependencies
-are downloaded when needed. CI avoids uses the `_util` module when possible.
+### Minimal dependencies
+Some commands in this module use minimal external dependencies. This reduces the
+dependencies used to produce the signed Microsoft binaries.
+
+Commands that use more than the minimal external dependencies will panic upon
+init if `MS_GO_UTIL_ALLOW_ONLY_MINIMAL_DEPS` is set to `1`. This makes it
+possible to test our pipelines to make sure they only use the expected commands.
+
+The minimal dependencies are themselves tested by
+`TestMinimalCommandDependencies` in `testutil`. It uses `go list` to ensure that
+all commands that use more than the minimal set of dependencies include the
+conditional panic upon init.
+
+### Support for gotestsum wrapping
+The `run-builder` command implements a gotestsum wrapper around the `build`
+command. This isn't implemented in `build` itself to keep dependencies for the
+signed build low. There are some features in the build command that accommodate
+gotestsum but don't make sense as standalone features a dev would use. For
+example, JSON test output and stderr redirection to stdout.
+
+The high-level execution flow looks roughly like this when running in CI:
+
+* `eng/pipeline/jobs/run-stage.yml`  
+  runs:
+* `eng/run.ps1 run-builder -test -builder linux-amd64-test -junitfile [...]`  
+  which runs the Go function:
+* `gotestsum.Run(... eng/run.ps1 build -test -json ...)`  
+  which runs and captures the output of:
+* `eng/run.ps1 build -test -json`  
+  which runs [`cmd/build/build.go`](cmd/build/build.go) in this module.
+
+> [!NOTE]
+> This support is not currently used in our CI because this process seems to cut off some test output:
+> [microsoft/go#1114](https://github.com/microsoft/go/issues/1114).

eng/_core/cmd/build/build.go → eng/_util/cmd/build/build.go

@@ -15,9 +15,9 @@ import (
 	"runtime"
 	"strings"
 
-	"github.com/microsoft/go/_core/buildutil"
-	"github.com/microsoft/go/_core/patch"
-	"github.com/microsoft/go/_core/submodule"
+	"github.com/microsoft/go-infra/patch"
+	"github.com/microsoft/go-infra/submodule"
+	"github.com/microsoft/go/_util/buildutil"
 )
 
 const description = `
@@ -112,10 +112,14 @@ func build(o *options) error {
 	}
 
 	if o.Refresh {
-		if err := submodule.Reset(rootDir); err != nil {
+		config, err := patch.FindAncestorConfig(rootDir)
+		if err != nil {
 			return err
 		}
-		if err := patch.Apply(rootDir, patch.ApplyModeIndex); err != nil {
+		if err := submodule.Reset(rootDir, filepath.Join(config.RootDir, config.SubmoduleDir), true); err != nil {
+			return err
+		}
+		if err := patch.Apply(config, patch.ApplyModeIndex); err != nil {
 			return err
 		}
 	}
@@ -224,7 +228,7 @@ func build(o *options) error {
 		// For example, if we're running in CI, gotestsum may be capturing our output to report in a
 		// JUnit file. If gotestsum detects output in stderr, it prints it in an error message. This
 		// error message stands out, and could mislead someone trying to diagnose a failed test run.
-		// Redirecting all stderr output avoids this scenario. (See /eng/_core/README.md for more
+		// Redirecting all stderr output avoids this scenario. (See /eng/_util/README.md for more
 		// info on why we may be wrapped by gotestsum.)
 		//
 		// An example of benign stderr output is when the tests check for machine capabilities. A

eng/_util/cmd/createbuildassetjson/nonminimaldeps.go

@@ -0,0 +1,9 @@
+// Copyright (c) Microsoft Corporation.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+// This command uses non-minimal dependencies, so ensure it can't be used while in minimal mode.
+
+import _ "github.com/microsoft/go/_util/internal/depsinitpanic"

eng/_util/cmd/run-builder/nonminimaldeps.go

@@ -0,0 +1,9 @@
+// Copyright (c) Microsoft Corporation.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+// This command uses non-minimal dependencies, so ensure it can't be used while in minimal mode.
+
+import _ "github.com/microsoft/go/_util/internal/depsinitpanic"

eng/_util/cmd/run-builder/run-builder.go

@@ -13,7 +13,7 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/microsoft/go/_core/buildutil"
+	"github.com/microsoft/go/_util/buildutil"
 	gotestsumcmd "gotest.tools/gotestsum/cmd"
 )
 

eng/_util/cmd/selftest/selftest.go

@@ -0,0 +1,52 @@
+// Copyright (c) Microsoft Corporation.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/microsoft/go-infra/executil"
+)
+
+const description = `
+This command runs the _util self-tests using the stage 0 Go toolchain.
+`
+
+func main() {
+	var help = flag.Bool("h", false, "Print this help message.")
+
+	flag.Usage = func() {
+		fmt.Fprintf(flag.CommandLine.Output(), "Usage of selftest:\n")
+		flag.PrintDefaults()
+		fmt.Fprintf(flag.CommandLine.Output(), "%s\n", description)
+	}
+
+	flag.Parse()
+	if *help {
+		flag.Usage()
+		return
+	}
+
+	if err := run(); err != nil {
+		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
+		os.Exit(1)
+	}
+}
+
+func run() error {
+	stage0Goroot := os.Getenv("STAGE_0_GOROOT")
+	if stage0Goroot == "" {
+		return fmt.Errorf("STAGE_0_GOROOT not set")
+	}
+
+	return executil.Run(executil.Dir(
+		filepath.Join("eng", "_util"),
+		filepath.Join(stage0Goroot, "bin", "go"),
+		"test", "./...",
+	))
+}