From 983228366edc1bed1be6e6f7a45e285b4707b9ba Mon Sep 17 00:00:00 2001 From: Oliver Breitwieser Date: Sun, 1 Sep 2024 23:20:31 +0200 Subject: [PATCH] fix: build with go 1.23 by tracking modules.txt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes: #117 Since go 1.23, `vendor/modules.txt` is [required](https://github.com/golang/go/commit/38ee0c7630e999f97d96899ecf4e8d0230236cd6#diff-61fb6e44eac25bd4d6a8a64b3f38ee8a41faaefd1ef481170a011ecfc0f7c76bR344). Hence, this is a first attempt to track `modules.txt` via gomod2nix. Since, unfortunately, `module.txt` employs a bespoke custom format in which [varying](https://cs.opensource.google/go/go/+/refs/tags/go1.23.0:src/cmd/go/internal/modload/vendor.go;l=57) [numbers](https://cs.opensource.google/go/go/+/refs/tags/go1.23.0:src/cmd/go/internal/modload/vendor.go;l=103) of `#` hold meaning I deemed it more stable to not reverse engineer it. Instead, we generate it when running `gomod2nix` by executing `go mod vendor` and tracking its content in `gomod2nix.toml`. Not the most elegant solution, but it works and enables us to use gomod2nix with go v1.23. Any comments appreached… --- builder/default.nix | 6 +++++- gomod2nix.toml | 3 ++- internal/cmd/root.go | 8 ++++---- internal/generate/generate.go | 38 +++++++++++++++++++++++++++++++++-- internal/schema/schema.go | 16 +++++++++++---- 5 files changed, 59 insertions(+), 12 deletions(-) diff --git a/builder/default.nix b/builder/default.nix index d500555..b9c9eca 100644 --- a/builder/default.nix +++ b/builder/default.nix @@ -106,11 +106,13 @@ let sources = toJSON (filterAttrs (n: _: n != defaultPackage) sources); + vendorModules = modulesStruct.vendorModulesTxt or ""; + passthru = { inherit sources; }; - passAsFile = [ "json" "sources" ]; + passAsFile = [ "json" "sources" "vendorModules" ]; } ( '' @@ -122,6 +124,8 @@ let ${internal.symlink} ${concatStringsSep "\n" localReplaceCommands} + cp $vendorModulesPath vendor/modules.txt + mv vendor $out '' ); diff --git a/gomod2nix.toml b/gomod2nix.toml index 6766016..61c6efc 100644 --- a/gomod2nix.toml +++ b/gomod2nix.toml @@ -1,4 +1,5 @@ -schema = 3 +schema = 4 +vendorModulesTxt = "# github.com/BurntSushi/toml v1.3.2\n## explicit; go 1.16\ngithub.com/BurntSushi/toml\ngithub.com/BurntSushi/toml/internal\n# github.com/inconshreveable/mousetrap v1.1.0\n## explicit; go 1.18\ngithub.com/inconshreveable/mousetrap\n# github.com/nix-community/go-nix v0.0.0-20220612195009-5f5614f7ca47\n## explicit; go 1.15\ngithub.com/nix-community/go-nix/pkg/nar\ngithub.com/nix-community/go-nix/pkg/wire\n# github.com/sirupsen/logrus v1.9.3\n## explicit; go 1.13\ngithub.com/sirupsen/logrus\n# github.com/spf13/cobra v1.8.0\n## explicit; go 1.15\ngithub.com/spf13/cobra\n# github.com/spf13/pflag v1.0.5\n## explicit; go 1.12\ngithub.com/spf13/pflag\n# golang.org/x/mod v0.14.0\n## explicit; go 1.18\ngolang.org/x/mod/internal/lazyregexp\ngolang.org/x/mod/modfile\ngolang.org/x/mod/module\ngolang.org/x/mod/semver\n# golang.org/x/sys v0.14.0\n## explicit; go 1.18\ngolang.org/x/sys/execabs\ngolang.org/x/sys/unix\ngolang.org/x/sys/windows\n# golang.org/x/tools/go/vcs v0.1.0-deprecated\n## explicit; go 1.19\ngolang.org/x/tools/go/vcs\n" [mod] [mod."github.com/BurntSushi/toml"] diff --git a/internal/cmd/root.go b/internal/cmd/root.go index b435941..981d672 100644 --- a/internal/cmd/root.go +++ b/internal/cmd/root.go @@ -5,10 +5,10 @@ import ( "os" "path/filepath" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" generate "github.com/nix-community/gomod2nix/internal/generate" schema "github.com/nix-community/gomod2nix/internal/schema" + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" ) const directoryDefault = "./" @@ -62,7 +62,7 @@ func generateFunc(cmd *cobra.Command, args []string) { { goMod2NixPath := filepath.Join(outDir, "gomod2nix.toml") outFile := goMod2NixPath - pkgs, err := generate.GeneratePkgs(directory, goMod2NixPath, maxJobs) + generated, err := generate.GeneratePkgs(directory, goMod2NixPath, maxJobs) if err != nil { panic(fmt.Errorf("error generating pkgs: %v", err)) } @@ -75,7 +75,7 @@ func generateFunc(cmd *cobra.Command, args []string) { goPackagePath = tmpProj.GoPackagePath } - output, err := schema.Marshal(pkgs, goPackagePath, subPackages) + output, err := schema.Marshal(generated, goPackagePath, subPackages) if err != nil { panic(fmt.Errorf("error marshaling output: %v", err)) } diff --git a/internal/generate/generate.go b/internal/generate/generate.go index 12db73b..602795d 100644 --- a/internal/generate/generate.go +++ b/internal/generate/generate.go @@ -140,7 +140,7 @@ builtins.filterSource (name: type: baseNameOf name != ".DS_Store") ( return executor.Wait() } -func GeneratePkgs(directory string, goMod2NixPath string, numWorkers int) ([]*schema.Package, error) { +func GeneratePkgs(directory string, goMod2NixPath string, numWorkers int) (*schema.GeneratePkgsResult, error) { modDownloads, replace, err := common(directory) if err != nil { return nil, err @@ -208,10 +208,44 @@ func GeneratePkgs(directory string, goMod2NixPath string, numWorkers int) ([]*sc return nil, err } + // Dependencies is downloaded, run `go mod vendor` to obtain `vendor/modules.txt` without reverse engineering + tmpVendorEnv := filepath.Join(directory, "vendor-gomod2nix") + err = os.RemoveAll(tmpVendorEnv) + if err != nil { + return nil, err + } + defer func() { + _ = os.RemoveAll(tmpVendorEnv) + }() + + var modulesTxt string + { + log.Info("Obtaining modules.txt") + cmd := exec.Command( + "go", "mod", "vendor", "-o", tmpVendorEnv, + ) + cmd.Dir = directory + err = cmd.Run() + if err != nil { + return nil, err + } + modulesTxtBytes, err := os.ReadFile(filepath.Join(tmpVendorEnv, "modules.txt")) + if err != nil { + return nil, err + } + if len(modulesTxtBytes) == 0 { + return nil, fmt.Errorf("modules.txt has no content") + } + modulesTxt = string(modulesTxtBytes) + } + sort.Slice(packages, func(i, j int) bool { return packages[i].GoPackagePath < packages[j].GoPackagePath }) - return packages, nil + return &schema.GeneratePkgsResult{ + Packages: packages, + ModulesTxt: string(modulesTxt), + }, nil } diff --git a/internal/schema/schema.go b/internal/schema/schema.go index 2ca359f..addde7f 100644 --- a/internal/schema/schema.go +++ b/internal/schema/schema.go @@ -2,11 +2,12 @@ package types import ( "bytes" - "github.com/BurntSushi/toml" "os" + + "github.com/BurntSushi/toml" ) -const SchemaVersion = 3 +const SchemaVersion = 4 type Package struct { GoPackagePath string `toml:"-"` @@ -18,6 +19,7 @@ type Package struct { type Output struct { SchemaVersion int `toml:"schema"` Mod map[string]*Package `toml:"mod"` + VendorModules string `toml:"vendorModulesTxt"` // Packages with passed import paths trigger `go install` based on this list SubPackages []string `toml:"subPackages,omitempty"` @@ -26,15 +28,21 @@ type Output struct { GoPackagePath string `toml:"goPackagePath,omitempty"` } -func Marshal(pkgs []*Package, goPackagePath string, subPackages []string) ([]byte, error) { +type GeneratePkgsResult struct { + Packages []*Package + ModulesTxt string +} + +func Marshal(generated *GeneratePkgsResult, goPackagePath string, subPackages []string) ([]byte, error) { out := &Output{ SchemaVersion: SchemaVersion, Mod: make(map[string]*Package), SubPackages: subPackages, GoPackagePath: goPackagePath, + VendorModules: generated.ModulesTxt, } - for _, pkg := range pkgs { + for _, pkg := range generated.Packages { out.Mod[pkg.GoPackagePath] = pkg }