From a05e8d7e53aedec59a5067f7bf750439609f73e2 Mon Sep 17 00:00:00 2001 From: Jason Hall Date: Fri, 11 Mar 2022 09:40:51 -0500 Subject: [PATCH 1/6] Support `ko build ./...` This changes `ko build` to interpret an importpath string that ends in "/..." by running `go list` and parsing its output to find `package main`s matched by that wildcard. Since `ko build` already supports passing multiple paths, this additively collects uniq matched importpaths, so this works, and produces three images for the three `package main`s in this repo: ``` ko build ./... ./cmd/... ./ ``` Matched paths are still checked if they're `package main`, so this doesn't work: ``` ko build ./... ./pkg/build ``` (because `pkg/build` isn't `package main`) --- pkg/commands/build.go | 57 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 55 insertions(+), 2 deletions(-) diff --git a/pkg/commands/build.go b/pkg/commands/build.go index dd6b0b33f7..dbc850995f 100644 --- a/pkg/commands/build.go +++ b/pkg/commands/build.go @@ -15,7 +15,15 @@ package commands import ( + "bytes" + "context" + "encoding/json" + "errors" "fmt" + "go/build" + "io" + "os/exec" + "strings" "github.com/google/ko/pkg/commands/options" "github.com/spf13/cobra" @@ -61,9 +69,28 @@ func addBuild(topLevel *cobra.Command) { if err := options.Validate(po, bo); err != nil { return fmt.Errorf("validating options: %w", err) } - ctx := cmd.Context() + uniq := map[string]struct{}{} + for _, a := range args { + if strings.HasSuffix(a, "/...") { + matches, err := importPaths(ctx, a) + if err != nil { + return fmt.Errorf("resolving import path %q: %w", a, err) + } + for _, m := range matches { + uniq[m] = struct{}{} + } + } else { + uniq[a] = struct{}{} + } + } + importpaths := make([]string, 0, len(uniq)) + for k := range uniq { + importpaths = append(importpaths, k) + } + // TODO: sort? + bo.InsecureRegistry = po.InsecureRegistry builder, err := makeBuilder(ctx, bo) if err != nil { @@ -74,7 +101,7 @@ func addBuild(topLevel *cobra.Command) { return fmt.Errorf("error creating publisher: %w", err) } defer publisher.Close() - images, err := publishImages(ctx, args, publisher, builder) + images, err := publishImages(ctx, importpaths, publisher, builder) if err != nil { return fmt.Errorf("failed to publish images: %w", err) } @@ -88,3 +115,29 @@ func addBuild(topLevel *cobra.Command) { options.AddBuildOptions(build, bo) topLevel.AddCommand(build) } + +// importPaths resolves a wildcard importpath string like "./cmd/..." or +// "./..." and returns the 'package main' packages matched by that wildcard, +// using `go list`. +func importPaths(ctx context.Context, s string) ([]string, error) { + var buf bytes.Buffer + cmd := exec.CommandContext(ctx, "go", "list", "-json", s) + cmd.Stdout = &buf + if err := cmd.Run(); err != nil { + return nil, err + } + var out []string + dec := json.NewDecoder(&buf) + for { + var p build.Package + if err := dec.Decode(&p); errors.Is(err, io.EOF) { + break + } else if err != nil { + return nil, err + } + if p.Name == "main" { + out = append(out, p.ImportPath) + } + } + return out, nil +} From 04feae6d31f36db714e76b6d685bcd891df03a3e Mon Sep 17 00:00:00 2001 From: Jason Hall Date: Fri, 11 Mar 2022 09:48:01 -0500 Subject: [PATCH 2/6] Fail if no import paths match ``` $ ko build ./pkg/... 2022/03/11 09:47:49 error during command execution: no package main packages matched ``` --- main.go | 2 +- pkg/commands/build.go | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/main.go b/main.go index bad1da750f..78b5596036 100644 --- a/main.go +++ b/main.go @@ -29,6 +29,6 @@ func main() { ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) defer stop() if err := commands.Root.ExecuteContext(ctx); err != nil { - log.Fatal("error during command execution:", err) + log.Fatal("error during command execution: ", err) } } diff --git a/pkg/commands/build.go b/pkg/commands/build.go index dbc850995f..bf9fd2b8db 100644 --- a/pkg/commands/build.go +++ b/pkg/commands/build.go @@ -89,6 +89,9 @@ func addBuild(topLevel *cobra.Command) { for k := range uniq { importpaths = append(importpaths, k) } + if len(importpaths) == 0 { + return errors.New("no package main packages matched") + } // TODO: sort? bo.InsecureRegistry = po.InsecureRegistry From d264b3f4b1244f875eb6ca02fe934e367b243680 Mon Sep 17 00:00:00 2001 From: Jason Hall Date: Fri, 11 Mar 2022 10:03:51 -0500 Subject: [PATCH 3/6] Use packages.Load instead of go list --- pkg/commands/build.go | 64 ++++++++++++------------------------------- 1 file changed, 17 insertions(+), 47 deletions(-) diff --git a/pkg/commands/build.go b/pkg/commands/build.go index bf9fd2b8db..317b15abcc 100644 --- a/pkg/commands/build.go +++ b/pkg/commands/build.go @@ -15,18 +15,12 @@ package commands import ( - "bytes" - "context" - "encoding/json" "errors" "fmt" - "go/build" - "io" - "os/exec" - "strings" "github.com/google/ko/pkg/commands/options" "github.com/spf13/cobra" + "golang.org/x/tools/go/packages" ) // addBuild augments our CLI surface with build. @@ -71,28 +65,10 @@ func addBuild(topLevel *cobra.Command) { } ctx := cmd.Context() - uniq := map[string]struct{}{} - for _, a := range args { - if strings.HasSuffix(a, "/...") { - matches, err := importPaths(ctx, a) - if err != nil { - return fmt.Errorf("resolving import path %q: %w", a, err) - } - for _, m := range matches { - uniq[m] = struct{}{} - } - } else { - uniq[a] = struct{}{} - } - } - importpaths := make([]string, 0, len(uniq)) - for k := range uniq { - importpaths = append(importpaths, k) - } - if len(importpaths) == 0 { - return errors.New("no package main packages matched") + importpaths, err := importPaths(args) + if err != nil { + return fmt.Errorf("resolving import paths: %w", err) } - // TODO: sort? bo.InsecureRegistry = po.InsecureRegistry builder, err := makeBuilder(ctx, bo) @@ -119,28 +95,22 @@ func addBuild(topLevel *cobra.Command) { topLevel.AddCommand(build) } -// importPaths resolves a wildcard importpath string like "./cmd/..." or -// "./..." and returns the 'package main' packages matched by that wildcard, -// using `go list`. -func importPaths(ctx context.Context, s string) ([]string, error) { - var buf bytes.Buffer - cmd := exec.CommandContext(ctx, "go", "list", "-json", s) - cmd.Stdout = &buf - if err := cmd.Run(); err != nil { - return nil, err +// importPaths resolves a list of importpath strings that may contain wildcards +// like "./cmd/..." or "./...", and returns the 'package main' packages matched +// by those importpaths. +func importPaths(args []string) ([]string, error) { + ps, err := packages.Load(&packages.Config{}, args...) + if err != nil { + return nil, fmt.Errorf("loading packages: %w", err) } - var out []string - dec := json.NewDecoder(&buf) - for { - var p build.Package - if err := dec.Decode(&p); errors.Is(err, io.EOF) { - break - } else if err != nil { - return nil, err - } + out := []string{} + for _, p := range ps { if p.Name == "main" { - out = append(out, p.ImportPath) + out = append(out, p.String()) } } + if len(importpaths) == 0 { + return nil, errors.New("no package main packages matched") + } return out, nil } From bb7530e254a6900557e8f77c0b1e84df58068468 Mon Sep 17 00:00:00 2001 From: Jason Hall Date: Fri, 11 Mar 2022 10:26:35 -0500 Subject: [PATCH 4/6] fix a dumb bug --- pkg/commands/build.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/commands/build.go b/pkg/commands/build.go index 317b15abcc..121b7d451c 100644 --- a/pkg/commands/build.go +++ b/pkg/commands/build.go @@ -109,7 +109,7 @@ func importPaths(args []string) ([]string, error) { out = append(out, p.String()) } } - if len(importpaths) == 0 { + if len(out) == 0 { return nil, errors.New("no package main packages matched") } return out, nil From b92dc6ead8d9cebffc0db9e555fa73b8923d367a Mon Sep 17 00:00:00 2001 From: Jason Hall Date: Fri, 11 Mar 2022 10:57:08 -0500 Subject: [PATCH 5/6] pass more stuff to packages.Load --- pkg/commands/build.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pkg/commands/build.go b/pkg/commands/build.go index 121b7d451c..8eeff2732c 100644 --- a/pkg/commands/build.go +++ b/pkg/commands/build.go @@ -17,6 +17,7 @@ package commands import ( "errors" "fmt" + "path/filepath" "github.com/google/ko/pkg/commands/options" "github.com/spf13/cobra" @@ -65,7 +66,7 @@ func addBuild(topLevel *cobra.Command) { } ctx := cmd.Context() - importpaths, err := importPaths(args) + importpaths, err := importPaths(bo.WorkingDirectory, args) if err != nil { return fmt.Errorf("resolving import paths: %w", err) } @@ -98,8 +99,12 @@ func addBuild(topLevel *cobra.Command) { // importPaths resolves a list of importpath strings that may contain wildcards // like "./cmd/..." or "./...", and returns the 'package main' packages matched // by those importpaths. -func importPaths(args []string) ([]string, error) { - ps, err := packages.Load(&packages.Config{}, args...) +func importPaths(dir string, args []string) ([]string, error) { + dir = filepath.Clean(dir) + if dir == "." { + dir = "" + } + ps, err := packages.Load(&packages.Config{Dir: dir, Mode: packages.NeedName}, args...) if err != nil { return nil, fmt.Errorf("loading packages: %w", err) } From 6271d1b7a8335566c4c1e818c928378c0f147a52 Mon Sep 17 00:00:00 2001 From: Jason Hall Date: Fri, 11 Mar 2022 11:13:11 -0500 Subject: [PATCH 6/6] fail-fast: false on modules integration tests --- .github/workflows/modules-integration-test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/modules-integration-test.yaml b/.github/workflows/modules-integration-test.yaml index ddcc425af3..b56467620e 100644 --- a/.github/workflows/modules-integration-test.yaml +++ b/.github/workflows/modules-integration-test.yaml @@ -8,6 +8,7 @@ jobs: test: name: Module Tests strategy: + fail-fast: false matrix: go-version: [1.16.x, 1.17.x] runs-on: 'ubuntu-latest'