Skip to content

Commit

Permalink
wip: git: centralize git cli operations
Browse files Browse the repository at this point in the history 
Signed-off-by: Justin Chadwell <[email protected]>
  • Loading branch information
@jedevc
jedevc committed Aug 4, 2023
1 parent b49a887 commit 1aa6885
Show file tree
Hide file tree
Showing 4 changed files with 298 additions and 99 deletions.
154 changes: 60 additions & 94 deletions source/git/gitsource.go
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package git

import (
"bytes"
"context"
"encoding/base64"
"fmt"
Expand All @@ -27,6 +26,7 @@ import (
"github.com/moby/buildkit/source"
srctypes "github.com/moby/buildkit/source/types"
"github.com/moby/buildkit/util/bklog"
"github.com/moby/buildkit/util/gitutil"
"github.com/moby/buildkit/util/progress/logs"
"github.com/moby/buildkit/util/urlutil"
"github.com/moby/locker"
Expand Down Expand Up @@ -124,16 +124,26 @@ func (gs *gitSource) mountRemote(ctx context.Context, remote string, auth []stri
}
}()

git, err := gitutil.NewGitCLI(
gitutil.WithExec(runWithStandardUmask),
gitutil.WithStreams(logStreams),
gitutil.WithDir("", dir),
gitutil.WithArgs(auth...),
)
if err != nil {
return "", nil, err
}

if initializeRepo {
// Explicitly set the Git config 'init.defaultBranch' to the
// implied default to suppress "hint:" output about not having a
// default initial branch name set which otherwise spams unit
// test logs.
if _, err := gitWithinDir(ctx, dir, "", "", "", auth, "-c", "init.defaultBranch=master", "init", "--bare"); err != nil {
if _, err := git.Run(ctx, "-c", "init.defaultBranch=master", "init", "--bare"); err != nil {
return "", nil, errors.Wrapf(err, "failed to init repo at %s", dir)
}

if _, err := gitWithinDir(ctx, dir, "", "", "", auth, "remote", "add", "origin", remote); err != nil {
if _, err := git.Run(ctx, "remote", "add", "origin", remote); err != nil {
return "", nil, errors.Wrapf(err, "failed add origin repo at %s", dir)
}

Expand Down Expand Up @@ -335,17 +345,29 @@ func (gs *gitSourceHandler) CacheKey(ctx context.Context, g session.Group, index
defer unmountKnownHosts()
}

git, err := gitutil.NewGitCLI(
gitutil.WithExec(runWithStandardUmask),
gitutil.WithStreams(logStreams),
gitutil.WithDir("", gitDir),
gitutil.WithArgs(gs.auth...),
gitutil.WithSSHAuthSock(sock),
gitutil.WithSSHKnownHosts(knownHosts),
)
if err != nil {
return "", "", nil, false, err
}

ref := gs.src.Ref
if ref == "" {
ref, err = getDefaultBranch(ctx, gitDir, "", sock, knownHosts, gs.auth, gs.src.Remote)
ref, err = getDefaultBranch(ctx, git, gs.src.Remote)
if err != nil {
return "", "", nil, false, err
}
}

// TODO: should we assume that remote tag is immutable? add a timer?

buf, err := gitWithinDir(ctx, gitDir, "", sock, knownHosts, gs.auth, "ls-remote", "origin", ref)
buf, err := git.Run(ctx, "ls-remote", "origin", ref)
if err != nil {
return "", "", nil, false, errors.Wrapf(err, "failed to fetch remote %s", urlutil.RedactCredentials(remote))
}
Expand Down Expand Up @@ -416,9 +438,21 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, g session.Group) (out
defer unmountKnownHosts()
}

git, err := gitutil.NewGitCLI(
gitutil.WithExec(runWithStandardUmask),
gitutil.WithStreams(logStreams),
gitutil.WithDir("", gitDir),
gitutil.WithArgs(gs.auth...),
gitutil.WithSSHAuthSock(sock),
gitutil.WithSSHKnownHosts(knownHosts),
)
if err != nil {
return nil, err
}

ref := gs.src.Ref
if ref == "" {
ref, err = getDefaultBranch(ctx, gitDir, "", sock, knownHosts, gs.auth, gs.src.Remote)
ref, err = getDefaultBranch(ctx, git, gs.src.Remote)
if err != nil {
return nil, err
}
Expand All @@ -427,7 +461,7 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, g session.Group) (out
doFetch := true
if isCommitSHA(ref) {
// skip fetch if commit already exists
if _, err := gitWithinDir(ctx, gitDir, "", sock, knownHosts, nil, "cat-file", "-e", ref+"^{commit}"); err == nil {
if _, err := git.Run(ctx, "cat-file", "-e", ref+"^{commit}"); err == nil {
doFetch = false
}
}
Expand All @@ -451,10 +485,10 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, g session.Group) (out
// in case the ref is a branch and it now points to a different commit sha
// TODO: is there a better way to do this?
}
if _, err := gitWithinDir(ctx, gitDir, "", sock, knownHosts, gs.auth, args...); err != nil {
if _, err := git.Run(ctx, args...); err != nil {
return nil, errors.Wrapf(err, "failed to fetch remote %s", urlutil.RedactCredentials(gs.src.Remote))
}
_, err = gitWithinDir(ctx, gitDir, "", sock, knownHosts, nil, "reflog", "expire", "--all", "--expire=now")
_, err = git.Run(ctx, "reflog", "expire", "--all", "--expire=now")
if err != nil {
return nil, errors.Wrapf(err, "failed to expire reflog for remote %s", urlutil.RedactCredentials(gs.src.Remote))
}
Expand Down Expand Up @@ -496,19 +530,20 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, g session.Group) (out
if err := os.MkdirAll(checkoutDir, 0711); err != nil {
return nil, err
}
_, err = gitWithinDir(ctx, checkoutDirGit, "", sock, knownHosts, nil, "-c", "init.defaultBranch=master", "init")
checkoutGit := git.New(gitutil.WithDir(checkoutDir, checkoutDirGit))
_, err = checkoutGit.Run(ctx, "-c", "init.defaultBranch=master", "init")
if err != nil {
return nil, err
}
// Defense-in-depth: clone using the file protocol to disable local-clone
// optimizations which can be abused on some versions of Git to copy unintended
// host files into the build context.
_, err = gitWithinDir(ctx, checkoutDirGit, "", sock, knownHosts, nil, "remote", "add", "origin", "file://"+gitDir)
_, err = checkoutGit.Run(ctx, "remote", "add", "origin", "file://"+gitDir)
if err != nil {
return nil, err
}

gitCatFileBuf, err := gitWithinDir(ctx, gitDir, "", sock, knownHosts, gs.auth, "cat-file", "-t", ref)
gitCatFileBuf, err := git.Run(ctx, "cat-file", "-t", ref)
if err != nil {
return nil, err
}
Expand All @@ -519,26 +554,26 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, g session.Group) (out
pullref += ":refs/tags/" + pullref
} else if isCommitSHA(ref) {
pullref = "refs/buildkit/" + identity.NewID()
_, err = gitWithinDir(ctx, gitDir, "", sock, knownHosts, gs.auth, "update-ref", pullref, ref)
_, err = git.Run(ctx, "update-ref", pullref, ref)
if err != nil {
return nil, err
}
} else {
pullref += ":" + pullref
}
_, err = gitWithinDir(ctx, checkoutDirGit, "", sock, knownHosts, gs.auth, "fetch", "-u", "--depth=1", "origin", pullref)
_, err = checkoutGit.Run(ctx, "fetch", "-u", "--depth=1", "origin", pullref)
if err != nil {
return nil, err
}
_, err = gitWithinDir(ctx, checkoutDirGit, checkoutDir, sock, knownHosts, nil, "checkout", "FETCH_HEAD")
_, err = checkoutGit.Run(ctx, "checkout", "FETCH_HEAD")
if err != nil {
return nil, errors.Wrapf(err, "failed to checkout remote %s", urlutil.RedactCredentials(gs.src.Remote))
}
_, err = gitWithinDir(ctx, checkoutDirGit, "", sock, knownHosts, nil, "remote", "set-url", "origin", urlutil.RedactCredentials(gs.src.Remote))
_, err = checkoutGit.Run(ctx, "remote", "set-url", "origin", urlutil.RedactCredentials(gs.src.Remote))
if err != nil {
return nil, errors.Wrapf(err, "failed to set remote origin to %s", urlutil.RedactCredentials(gs.src.Remote))
}
_, err = gitWithinDir(ctx, checkoutDirGit, "", sock, knownHosts, nil, "reflog", "expire", "--all", "--expire=now")
_, err = checkoutGit.Run(ctx, "reflog", "expire", "--all", "--expire=now")
if err != nil {
return nil, errors.Wrapf(err, "failed to expire reflog for remote %s", urlutil.RedactCredentials(gs.src.Remote))
}
Expand All @@ -554,7 +589,7 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, g session.Group) (out
return nil, errors.Wrapf(err, "failed to create temporary checkout dir")
}
}
_, err = gitWithinDir(ctx, gitDir, cd, sock, knownHosts, nil, "checkout", ref, "--", ".")
_, err = git.New(gitutil.WithDir(cd, gitDir)).Run(ctx, "checkout", ref, "--", ".")
if err != nil {
return nil, errors.Wrapf(err, "failed to checkout remote %s", urlutil.RedactCredentials(gs.src.Remote))
}
Expand Down Expand Up @@ -587,7 +622,7 @@ func (gs *gitSourceHandler) Snapshot(ctx context.Context, g session.Group) (out
}
}

_, err = gitWithinDir(ctx, gitDir, checkoutDir, sock, knownHosts, gs.auth, "submodule", "update", "--init", "--recursive", "--depth=1")
_, err = git.New(gitutil.WithDir(checkoutDir, gitDir)).Run(ctx, "submodule", "update", "--init", "--recursive", "--depth=1")
if err != nil {
return nil, errors.Wrapf(err, "failed to update submodules for %s", urlutil.RedactCredentials(gs.src.Remote))
}
Expand Down Expand Up @@ -628,79 +663,6 @@ func isCommitSHA(str string) bool {
return validHex.MatchString(str)
}

func gitWithinDir(ctx context.Context, gitDir, workDir, sshAuthSock, knownHosts string, auth []string, args ...string) (*bytes.Buffer, error) {
a := append([]string{"--git-dir", gitDir}, auth...)
if workDir != "" {
a = append(a, "--work-tree", workDir)
}
return git(ctx, workDir, sshAuthSock, knownHosts, append(a, args...)...)
}

func getGitSSHCommand(knownHosts string) string {
gitSSHCommand := "ssh -F /dev/null"
if knownHosts != "" {
gitSSHCommand += " -o UserKnownHostsFile=" + knownHosts
} else {
gitSSHCommand += " -o StrictHostKeyChecking=no"
}
return gitSSHCommand
}

func git(ctx context.Context, dir, sshAuthSock, knownHosts string, args ...string) (_ *bytes.Buffer, err error) {
for {
stdout, stderr, flush := logs.NewLogStreams(ctx, false)
defer stdout.Close()
defer stderr.Close()
defer func() {
if err != nil {
flush()
}
}()
args = append([]string{"-c", "protocol.file.allow=user"}, args...) // Block sneaky repositories from using repos from the filesystem as submodules.
cmd := exec.Command("git", args...)
cmd.Dir = dir // some commands like submodule require this
buf := bytes.NewBuffer(nil)
errbuf := bytes.NewBuffer(nil)
cmd.Stdin = nil
cmd.Stdout = io.MultiWriter(stdout, buf)
cmd.Stderr = io.MultiWriter(stderr, errbuf)
cmd.Env = []string{
"PATH=" + os.Getenv("PATH"),
"GIT_TERMINAL_PROMPT=0",
"GIT_SSH_COMMAND=" + getGitSSHCommand(knownHosts),
// "GIT_TRACE=1",
"GIT_CONFIG_NOSYSTEM=1", // Disable reading from system gitconfig.
"HOME=/dev/null", // Disable reading from user gitconfig.
"LC_ALL=C", // Ensure consistent output.
}
if sshAuthSock != "" {
cmd.Env = append(cmd.Env, "SSH_AUTH_SOCK="+sshAuthSock)
}
// remote git commands spawn helper processes that inherit FDs and don't
// handle parent death signal so exec.CommandContext can't be used
err := runWithStandardUmask(ctx, cmd)
if err != nil {
if strings.Contains(errbuf.String(), "--depth") || strings.Contains(errbuf.String(), "shallow") {
if newArgs := argsNoDepth(args); len(args) > len(newArgs) {
args = newArgs
continue
}
}
}
return buf, err
}
}

func argsNoDepth(args []string) []string {
out := make([]string, 0, len(args))
for _, a := range args {
if a != "--depth=1" {
out = append(out, a)
}
}
return out
}

func tokenScope(remote string) string {
// generally we can only use the token for fetching main remote but in case of github.com we do best effort
// to try reuse same token for all github.com remotes. This is the same behavior actions/checkout uses
Expand All @@ -713,8 +675,8 @@ func tokenScope(remote string) string {
}

// getDefaultBranch gets the default branch of a repository using ls-remote
func getDefaultBranch(ctx context.Context, gitDir, workDir, sshAuthSock, knownHosts string, auth []string, remoteURL string) (string, error) {
buf, err := gitWithinDir(ctx, gitDir, workDir, sshAuthSock, knownHosts, auth, "ls-remote", "--symref", remoteURL, "HEAD")
func getDefaultBranch(ctx context.Context, git *gitutil.GitCLI, remoteURL string) (string, error) {
buf, err := git.Run(ctx, "ls-remote", "--symref", remoteURL, "HEAD")
if err != nil {
return "", errors.Wrapf(err, "error fetching default branch for repository %s", urlutil.RedactCredentials(remoteURL))
}
Expand Down Expand Up @@ -762,3 +724,7 @@ func (md cacheRefMetadata) setGitSnapshot(key string) error {
func (md cacheRefMetadata) setGitRemote(key string) error {
return md.SetString(keyGitRemote, key, gitRemoteIndex+key)
}

func logStreams(ctx context.Context) (stdout, stderr io.WriteCloser, flush func()) {
return logs.NewLogStreams(ctx, false)
}
18 changes: 13 additions & 5 deletions source/git/gitsource_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (
"github.com/moby/buildkit/snapshot"
containerdsnapshot "github.com/moby/buildkit/snapshot/containerd"
"github.com/moby/buildkit/source"
"github.com/moby/buildkit/util/gitutil"
"github.com/moby/buildkit/util/leaseutil"
"github.com/moby/buildkit/util/progress"
"github.com/moby/buildkit/util/winlayers"
Expand Down Expand Up @@ -285,13 +286,20 @@ func testFetchByTag(t *testing.T, tag, expectedCommitSubject string, isAnnotated
}

if keepGitDir {
git, err := gitutil.NewGitCLI(
gitutil.WithExec(runWithStandardUmask),
gitutil.WithStreams(logStreams),
gitutil.WithDir(dir, ""),
)
require.NoError(t, err)

if isAnnotatedTag {
// get commit sha that the annotated tag points to
annotatedTagCommit, err := git(ctx, dir, "", "", "rev-list", "-n", "1", tag)
annotatedTagCommit, err := git.Run(ctx, "rev-list", "-n", "1", tag)
require.NoError(t, err)

// get current commit sha
headCommit, err := git(ctx, dir, "", "", "rev-parse", "HEAD")
headCommit, err := git.Run(ctx, "rev-parse", "HEAD")
require.NoError(t, err)

// HEAD should match the actual commit sha (and not the sha of the annotated tag,
Expand All @@ -302,9 +310,9 @@ func testFetchByTag(t *testing.T, tag, expectedCommitSubject string, isAnnotated
// test that we checked out the correct commit
// (in the case of an annotated tag, this message is of the commit the annotated tag points to
// and not the message of the tag)
gitLogOutput, err := git(ctx, dir, "", "", "log", "-n", "1", "--format=%s")
gitLogOutput, err := git.Run(ctx, "log", "-n", "1", "--format=%s")
require.NoError(t, err)
require.True(t, strings.Contains(strings.TrimSpace(gitLogOutput.String()), expectedCommitSubject))
require.Contains(t, strings.TrimSpace(gitLogOutput.String()), expectedCommitSubject)
}
}

Expand Down Expand Up @@ -578,7 +586,7 @@ func setupGitRepo(t *testing.T) gitRepoFixture {
"echo sbb > foo13",
"git add foo13",
"git commit -m third",
"git tag lightweight-tag",
"git tag --no-sign lightweight-tag",
"git checkout -B feature",
"echo baz > ghi",
"git add ghi",
Expand Down
11 changes: 11 additions & 0 deletions source/git/gitsource_windows.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,10 @@ package git

import (
"context"
"errors"
"os/exec"

"github.com/moby/buildkit/session/networks"
)

func runWithStandardUmask(ctx context.Context, cmd *exec.Cmd) error {
Expand All @@ -22,3 +25,11 @@ func runWithStandardUmask(ctx context.Context, cmd *exec.Cmd) error {
}()
return cmd.Wait()
}

func (s *gitCLI) initConfig(netConf *networks.Config) error {
if netConf == nil {
return nil
}

return errors.New("overriding network config is not supported on Windows")
}
Loading

0 comments on commit 1aa6885

Please sign in to comment.