Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

initial support for using fetchGit #11

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

initial support for using fetchGit #11

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
207 changes: 170 additions & 37 deletions fetch/fetch.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,22 +1,28 @@
package fetch

import (
"context"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
log "github.com/sirupsen/logrus"
"github.com/tweag/gomod2nix/formats/buildgopackage"
"github.com/tweag/gomod2nix/formats/gomod2nix"
"github.com/tweag/gomod2nix/types"
"golang.org/x/mod/modfile"
"golang.org/x/mod/module"
"golang.org/x/tools/go/vcs"
"io/ioutil"
"os"
"os/exec"
"path/filepath"
"regexp"
"sort"
"strings"

"github.com/google/go-github/github"
log "github.com/sirupsen/logrus"
"github.com/tweag/gomod2nix/formats/buildgopackage"
"github.com/tweag/gomod2nix/formats/gomod2nix"
"github.com/tweag/gomod2nix/types"
"golang.org/x/mod/modfile"
"golang.org/x/mod/module"
"golang.org/x/tools/go/vcs"
)

type packageJob struct {
Expand Down Expand Up @@ -151,6 +157,74 @@ func FetchPackages(goModPath string, goSumPath string, goMod2NixPath string, dep
return pkgs, nil
}

var GithubClient *github.Client
var githubRepoRegexp *regexp.Regexp = regexp.MustCompile(`^https://github.com/([^/]+)/([^/]+)`)

func resolveFullRev(repoRoot *vcs.RepoRoot, rev string) (string, error) {
// try using github api
if GithubClient != nil {
repoParts := githubRepoRegexp.FindStringSubmatch(repoRoot.Repo)
if len(repoParts) > 0 {
owner := repoParts[1]
repo := repoParts[2]

commit, _, _ := GithubClient.Repositories.GetCommit(context.Background(), owner, repo, rev)
if commit != nil && commit.SHA != nil {
return *commit.SHA, nil
}
}
}

// fallback to cloning repo
dirTop, err := ioutil.TempDir("", "vcstop-*")
if err != nil {
return "", err
}
defer os.RemoveAll(dirTop)

dir := filepath.Join(dirTop, "repo")

err = repoRoot.VCS.CreateAtRev(dir, repoRoot.Repo, rev)
if err != nil {
return "", err
}

stdout, err := exec.Command(
"git",
"-C",
dir,
"rev-parse",
rev,
).Output()

if err != nil {
return "", err
}

if len(stdout) > 0 {
// remove ending new line
return string(stdout[:len(stdout)-1]), nil
} else {
return "", errors.New("git rev-parse output was empty")
}
}

func fetchGitExprForRev(repo, rev string) string {
return fmt.Sprintf(
"builtins.fetchGit { url = %q; allRefs = true; rev = %q; submodules = true; }",
repo,
rev,
)
}

func fetchGitExprForTag(repo, tag string) string {
return fmt.Sprintf(
"builtins.fetchGit { url = %q; ref = %q; submodules = true; }",
repo,
"refs/tags/"+tag,
)
}

func fetchPackage(caches []map[string]*types.Package, importPath string, goPackagePath string, sumVersion string) (*types.Package, error) {
repoRoot, err := vcs.RepoRootForImportPath(importPath, false)
if err != nil {
Expand All @@ -159,8 +233,19 @@ func fetchPackage(caches []map[string]*types.Package, importPath string, goPacka

commitShaRev := regexp.MustCompile(`v\d+\.\d+\.\d+-[\d+\.a-zA-Z]*?[0-9]{14}-(.*?)$`)
rev := strings.TrimSuffix(sumVersion, "+incompatible")
var makeFetchExpr func(string, string) string
if commitShaRev.MatchString(rev) {
makeFetchExpr = fetchGitExprForRev
rev = commitShaRev.FindAllStringSubmatch(rev, -1)[0][1]
rev, err = resolveFullRev(repoRoot, rev)
if err != nil {
log.WithFields(log.Fields{
"goPackagePath": goPackagePath,
}).Error("Fetching failed")
return nil, err
}
} else {
makeFetchExpr = fetchGitExprForTag
}

goPackagePathPrefix, pathMajor, _ := module.SplitPathVersion(goPackagePath)
Expand Down Expand Up @@ -194,48 +279,78 @@ func fetchPackage(caches []map[string]*types.Package, importPath string, goPacka
}

type prefetchOutput struct {
URL string `json:"url"`
Rev string `json:"rev"`
Sha256 string `json:"sha256"`
Path string `json:"path"`
// date string
// fetchSubmodules bool
// deepClone bool
// leaveDotGit bool
LastModified int `json:"lastModified"`
LastModifiedDate string `json:"lastModifiedDate"`
NarHash string `json:"narHash"`
Path string `json:"path"`
Rev string `json:"rev"`
RevCount int `json:"revCount"`
ShortRev string `json:"shortRev"`
Submodules bool `json:"submodules"`
}

log.WithFields(log.Fields{
"goPackagePath": goPackagePath,
"rev": rev,
}).Info("Cache miss, fetching")

stdout, err := exec.Command(
"nix-prefetch-git",
"--quiet",
"--fetch-submodules",
"--url", repoRoot.Repo,
"--rev", rev).Output()
"nix",
"eval",
"--impure",
"--json",
"--expr",
fmt.Sprintf(
// we need to rename the outPath attr because nix's JSON output format
// logic is such that an attrset with an outPath attr will be printed
// as a solitary JSON string consisting of the outPath.
"let r = (%s); in builtins.removeAttrs r [\"outPath\"] // { path = r.outPath; }",
makeFetchExpr(repoRoot.Repo, rev),
),
).Output()

if err != nil {
newRev := fmt.Sprintf("%s/%s", relPath, rev)
if relPath != "" {
// handle cases like cloud.google.com/go/datastore where rev is v1.1.0
// the ref to fetch is refs/tags/datastore/v1.1.0,
// not refs/tags/v1.1.0.
//
// TODO: look through go source code and briefly document how this logic
// is supposed to work precisely.
newRev := fmt.Sprintf("%s/%s", relPath, rev)

log.WithFields(log.Fields{
"goPackagePath": goPackagePath,
"rev": newRev,
}).Info("Fetching failed, retrying with different rev format")
originalErr := err
stdout, err = exec.Command(
"nix-prefetch-git",
"--quiet",
"--fetch-submodules",
"--url", repoRoot.Repo,
"--rev", newRev).Output()
if err != nil {
log.WithFields(log.Fields{
"goPackagePath": goPackagePath,
"rev": newRev,
}).Info("Fetching failed, retrying with different rev format")
originalErr := err

stdout, err = exec.Command(
"nix",
"eval",
"--impure",
"--json",
"--expr",
fmt.Sprintf(
"let r = (%s); in builtins.removeAttrs r [\"outPath\"] // { path = r.outPath; }",
makeFetchExpr(repoRoot.Repo, newRev),
),
).Output()
if err != nil {
log.WithFields(log.Fields{
"goPackagePath": goPackagePath,
}).Error("Fetching failed")
return nil, originalErr
}

rev = newRev
} else {
// no relative path to try: propagate original error
log.WithFields(log.Fields{
"goPackagePath": goPackagePath,
}).Error("Fetching failed")
return nil, originalErr
return nil, err
}

rev = newRev
}

var output *prefetchOutput
Expand All @@ -249,6 +364,24 @@ func fetchPackage(caches []map[string]*types.Package, importPath string, goPacka
vendorPath = importPath
}

// need to convert SRI sha256 hash to sha256 hex
if !strings.HasPrefix(output.NarHash, "sha256-") {
log.WithFields(log.Fields{
"goPackagePath": goPackagePath,
}).Error("Fetching failed")
return nil, fmt.Errorf("Error: NarHash didn't begin with sha256- prefix: %s", output.NarHash)
}

b, err := base64.StdEncoding.DecodeString(output.NarHash[7:])
if err != nil {
log.WithFields(log.Fields{
"goPackagePath": goPackagePath,
}).Error("Fetching failed")
return nil, err
}

sha256 := hex.EncodeToString(b)

if relPath == "" && pathMajor != "" {
p := filepath.Join(output.Path, pathMajor)
_, err := os.Stat(p)
Expand All @@ -262,7 +395,7 @@ func fetchPackage(caches []map[string]*types.Package, importPath string, goPacka
GoPackagePath: goPackagePath,
URL: repoRoot.Repo,
Rev: output.Rev,
Sha256: output.Sha256,
Sha256: sha256,
// This is used to skip fetching where the previous package path & versions are still the same
// It's also used to construct the vendor directory in the Nix build
SumVersion: sumVersion,
Expand Down
12 changes: 11 additions & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,18 @@ go 1.14

require (
github.com/BurntSushi/toml v0.3.1
github.com/golang/protobuf v1.3.2 // indirect
github.com/google/go-github v17.0.0+incompatible
github.com/google/go-querystring v1.0.0 // indirect
github.com/kr/pretty v0.1.0 // indirect
github.com/orivej/go-nix v0.0.0-20180830055821-dae45d921a44
github.com/sirupsen/logrus v1.7.0
github.com/stretchr/testify v1.4.0 // indirect
golang.org/x/mod v0.3.0
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be
golang.org/x/sys v0.0.0-20200803210538-64077c9b5642 // indirect
golang.org/x/tools v0.0.0-20200825202427-b303f430e36d
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
google.golang.org/appengine v1.1.0 // indirect
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
)
41 changes: 37 additions & 4 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,20 @@ github.com/alecthomas/repr v0.0.0-20180818092828-117648cd9897 h1:p9Sln00KOTlrYkx
github.com/alecthomas/repr v0.0.0-20180818092828-117648cd9897/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/mattn/go-isatty v0.0.3 h1:ns/ykhmWi7G9O+8a448SecJU3nSMBXJfqQkl0upE1jI=
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/orivej/e v0.0.0-20180728214217-ac3492690fda h1:fqLgbcmo9qKecZOH8lByuxi9XXoIhNYBpRJEo4rDEUQ=
Expand All @@ -24,23 +36,44 @@ github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrSV+Z2tcbze+pEc3v5W4=
golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180828065106-d99a578cf41b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200803210538-64077c9b5642 h1:B6caxRw+hozq68X2MY7jEpZh/cr4/aHLv9xU8Kkadrw=
golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e h1:aZzprAO9/8oim3qStq3wc1Xuxx4QmAGriC4VU4ojemQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200825202427-b303f430e36d h1:W07d4xkoAUSNOkOzdzXCdFGxT7o2rW4q8M34tB2i//k=
golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 h1:/atklqdjdhuosWIl6AIbOeHJjicWYPqR9bpxqxYG2pA=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/appengine v1.1.0 h1:igQkv0AAhEIvTEpD5LIpAfav2eeVO9HBTjvKHVJPRSs=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
Loading