Merge branch 'main' into feat/create_default_merge_type

OWNERS

@@ -1,38 +1,4 @@
 approvers:
-- abayer
-- cagiti
-- ccojocar
-- daveconde
-- dgozalo
-- eddycharly
-- garethjevans
-- hferentschik
-- jstrachan
-- macox
-- mrageh
-- pmuir
-- rawlingsj
-- warrenbailey
-- wbrefvem
-- ankitm123
-- tomhobson
-- babadofar
+- maintainers
 reviewers:
-- abayer
-- cagiti
-- ccojocar
-- daveconde
-- dgozalo
-- eddycharly
-- garethjevans
-- hferentschik
-- jstrachan
-- macox
-- mrageh
-- pmuir
-- rawlingsj
-- warrenbailey
-- wbrefvem
-- ankitm123
-- tomhobson
-- babadofar
+- maintainers

OWNERS_ALIASES

@@ -0,0 +1,3 @@
+foreignAliases:
+- name: jx-community
+  org: jenkins-x

pkg/filebrowser/fake/fake_file_browser.go

@@ -63,7 +63,7 @@ func (f *fakeFileBrowser) getPath(owner, repo, path, ref string) string {
 	return filepath.Join(f.dir, path)
 }
 
-func (f *fakeFileBrowser) WithDir(owner, repo, ref string, fc filebrowser.FetchCache, fn func(dir string) error) error {
+func (f *fakeFileBrowser) WithDir(owner, repo, ref string, fc filebrowser.FetchCache, sparseCheckoutPatterns []string, fn func(dir string) error) error {
 	dir := f.getPath(owner, repo, "", ref)
 	return fn(dir)
 }

pkg/filebrowser/git_file_browser.go

@@ -2,9 +2,11 @@ package filebrowser
 
 import (
 	"io/ioutil"
+	"os"
 	"os/exec"
 	"path/filepath"
 	"regexp"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -38,8 +40,8 @@ func NewFileBrowserFromGitClient(clientFactory git.ClientFactory) Interface {
 	}
 }
 
-func (f *gitFileBrowser) WithDir(owner, repo, ref string, fc FetchCache, fn func(dir string) error) error {
-	return f.withRepoClient(owner, repo, ref, fc, func(repoClient git.RepoClient) error {
+func (f *gitFileBrowser) WithDir(owner, repo, ref string, fc FetchCache, sparseCheckoutPatterns []string, fn func(dir string) error) error {
+	return f.withRepoClient(owner, repo, ref, fc, sparseCheckoutPatterns, func(repoClient git.RepoClient) error {
 		dir := repoClient.Directory()
 		return fn(dir)
 	})
@@ -50,7 +52,7 @@ func (f *gitFileBrowser) GetMainAndCurrentBranchRefs(_, _, eventRef string) ([]s
 }
 
 func (f *gitFileBrowser) GetFile(owner, repo, path, ref string, fc FetchCache) (answer []byte, err error) {
-	err = f.withRepoClient(owner, repo, ref, fc, func(repoClient git.RepoClient) error {
+	err = f.withRepoClient(owner, repo, ref, fc, []string{"/" + path}, func(repoClient git.RepoClient) error {
 		f := repoPath(repoClient, path)
 		exists, err := util.FileExists(f)
 		if err != nil {
@@ -67,7 +69,7 @@ func (f *gitFileBrowser) GetFile(owner, repo, path, ref string, fc FetchCache) (
 }
 
 func (f *gitFileBrowser) ListFiles(owner, repo, path, ref string, fc FetchCache) (answer []*scm.FileEntry, err error) {
-	err = f.withRepoClient(owner, repo, ref, fc, func(repoClient git.RepoClient) error {
+	err = f.withRepoClient(owner, repo, ref, fc, []string{"/" + path + "/*"}, func(repoClient git.RepoClient) error {
 		dir := repoPath(repoClient, path)
 		exists, err := util.DirExists(dir)
 		if err != nil {
@@ -112,18 +114,21 @@ func repoPath(repoClient git.RepoClient, path string) string {
 	return filepath.Join(dir, path)
 }
 
-func (f *gitFileBrowser) withRepoClient(owner, repo, ref string, fc FetchCache, fn func(repoClient git.RepoClient) error) error {
+func (f *gitFileBrowser) withRepoClient(owner, repo, ref string, fc FetchCache, sparseCheckoutPatterns []string, fn func(repoClient git.RepoClient) error) error {
 	client := f.getOrCreateClient(owner, repo)
-
-	var repoClient git.RepoClient
+	sparseCheckout, _ := strconv.ParseBool(os.Getenv("SPARSE_CHECKOUT"))
 	var err error
 	client.lock.Lock()
 	defer client.lock.Unlock()
 	if client.repoClient == nil {
-		client.repoClient, err = f.clientFactory.ClientFor(owner, repo)
+		client.repoClient, err = f.clientFactory.ClientFor(owner, repo, sparseCheckoutPatterns)
 		if err != nil {
 			return errors.Wrapf(err, "failed to create repo client")
 		}
+	} else if sparseCheckout && len(sparseCheckoutPatterns) > 0 {
+		if err := client.repoClient.SetSparseCheckoutPatterns(sparseCheckoutPatterns); err != nil {
+			return errors.Wrapf(err, "failed to set sparse checkout patterns")
+		}
 	}
 	if client.mainBranch == "" {
 		var err error
@@ -133,13 +138,12 @@ func (f *gitFileBrowser) withRepoClient(owner, repo, ref string, fc FetchCache,
 		}
 	}
 	if err == nil {
-		repoClient = client.repoClient
 		err = client.UseRef(ref, fc)
 		if err != nil {
 			err = errors.Wrapf(err, "failed to switch to ref %s", ref)
 		}
 		if err == nil {
-			err = fn(repoClient)
+			err = fn(client.repoClient)
 			if err != nil {
 				err = errors.Wrapf(err, "failed to process repo %s/%s refref %s", owner, repo, ref)
 			}
@@ -230,10 +234,10 @@ func (c *repoClientFacade) UseRef(ref string, fc FetchCache) error {
 			}
 		}
 		if !isSHA {
-			// lets pull any new changes into the main branch
-			err := c.repoClient.Pull()
+			// lets merge any new changes into the main branch
+			_, err := runCmd(c.repoClient.Directory(), "git", "merge", "FETCH_HEAD")
 			if err != nil {
-				return errors.Wrapf(err, "failed to fetch repository %s", c.fullName)
+				return errors.Wrapf(err, "failed to merge repository %s", c.fullName)
 			}
 		}
 	}

pkg/filebrowser/interface.go

@@ -14,5 +14,5 @@ type Interface interface {
 	ListFiles(owner, repo, path, ref string, fc FetchCache) ([]*scm.FileEntry, error)
 
 	// WithDir processes the given repository and reference at the given directory
-	WithDir(owner, repo, ref string, fc FetchCache, f func(dir string) error) error
+	WithDir(owner, repo, ref string, fc FetchCache, sparseCheckoutPatterns []string, f func(dir string) error) error
 }

pkg/git/git.go

@@ -25,6 +25,7 @@ type Client interface {
 	SetRemote(remote string)
 	SetCredentials(user string, tokenGenerator func() []byte)
 	Clone(repo string) (*Repo, error)
+	SparseClone(repo string, sparseCheckoutPatterns []string) (*Repo, error)
 }
 
 // client can clone repos. It keeps a local cache, so successive clones of the
@@ -150,18 +151,8 @@ func (c *client) Clone(repo string) (*Repo, error) {
 		if err := os.Mkdir(filepath.Dir(cache), os.ModePerm); err != nil && !os.IsExist(err) {
 			return nil, err
 		}
-		prefix := ""
-		repoText := repo
-		if c.gitKind == kindBitbucketServer {
-			prefix = "scm/"
-			idx := strings.Index(repo, "/")
-
-			// to clone on bitbucket we need to lower case the projectKey owner
-			if idx > 0 {
-				repoText = fmt.Sprintf("%s/%s", strings.ToLower(repo[0:idx]), repo[idx+1:])
-			}
-		}
-		remote := fmt.Sprintf("%s/%s%s", base, prefix, repoText)
+		remote := getRemote(repo, c, base)
+
 		if b, err := retryCmd(c.logger, "", c.git, "clone", "--mirror", remote, cache); err != nil {
 			return nil, fmt.Errorf("git cache clone error: %v. output: %s", err, string(b))
 		}
@@ -193,6 +184,69 @@ func (c *client) Clone(repo string) (*Repo, error) {
 	}, nil
 }
 
+func (c *client) SparseClone(repo string, sparseCheckoutPatterns []string) (*Repo, error) {
+	// Make pattern part of cache key somehow.
+	replacer := strings.NewReplacer(".", "_", "/", "_")
+	cacheKey := repo + replacer.Replace(strings.Join(sparseCheckoutPatterns, "-"))
+	c.lockRepo(cacheKey)
+	defer c.unlockRepo(cacheKey)
+
+	base := c.base
+	user, pass := c.getCredentials()
+	if user != "" && pass != "" {
+		host, scheme := gitHostAndScheme(c.base)
+		base = fmt.Sprintf("%s://%s:%s@%s", scheme, user, pass, host)
+	}
+	cache := filepath.Join(c.dir, cacheKey)
+	if _, err := os.Stat(cache); os.IsNotExist(err) {
+		// Cache miss, clone it now.
+		c.logger.Infof("Cloning %s for the first time.", repo)
+		if err := os.Mkdir(filepath.Dir(cache), os.ModePerm); err != nil && !os.IsExist(err) {
+			return nil, err
+		}
+		remote := getRemote(repo, c, base)
+
+		if b, err := retryCmd(c.logger, "", c.git, "clone", "--no-checkout", "--filter=blob:none", "--sparse", "--depth=10", "--no-single-branch", remote, cache); err != nil {
+			return nil, fmt.Errorf("failed to clone repository: %v. output: %s", err, string(b))
+		}
+		if b, err := retryCmd(c.logger, cache, c.git, append([]string{"sparse-checkout", "set"}, sparseCheckoutPatterns...)...); err != nil {
+			return nil, fmt.Errorf("failed to set sparse checkout patterns to %v: %v. output: %s", sparseCheckoutPatterns, err, string(b))
+		}
+	} else if err != nil {
+		return nil, err
+	} else {
+		// Cache hit. Do a git fetch to keep updated.
+		c.logger.Infof("Fetching %s.", repo)
+		if b, err := retryCmd(c.logger, cache, c.git, "fetch", "--depth=10"); err != nil {
+			return nil, fmt.Errorf("git fetch error: %v. output: %s", err, string(b))
+		}
+	}
+	return &Repo{
+		Dir:    cache,
+		logger: c.logger,
+		git:    c.git,
+		base:   base,
+		repo:   repo,
+		user:   user,
+		pass:   pass,
+	}, nil
+}
+
+func getRemote(repo string, c *client, base string) string {
+	prefix := ""
+	repoText := repo
+	if c.gitKind == kindBitbucketServer {
+		prefix = "scm/"
+		idx := strings.Index(repo, "/")
+
+		// to clone on bitbucket we need to lower case the projectKey owner
+		if idx > 0 {
+			repoText = fmt.Sprintf("%s/%s", strings.ToLower(repo[0:idx]), repo[idx+1:])
+		}
+	}
+	return fmt.Sprintf("%s/%s%s", base, prefix, repoText)
+}
+
 func gitHostAndScheme(s string) (string, string) {
 	u, err := url.Parse(s)
 	if err == nil {
@@ -359,6 +413,7 @@ func retryCmd(l *logrus.Entry, dir, cmd string, arg ...string) ([]byte, error) {
 			sleepyTime *= 2
 			continue
 		}
+		l.Debugf("Running %s %v succeeded with output %s.", cmd, arg, string(b))
 		break
 	}
 	return b, err

pkg/git/v2/client_factory.go

@@ -21,6 +21,7 @@ import (
 	"os"
 	"path"
 	"runtime"
+	"strconv"
 	"sync"
 	"time"
 
@@ -34,7 +35,7 @@ type ClientFactory interface {
 	// been cloned to the given directory.
 	ClientFromDir(org, repo, dir string) (RepoClient, error)
 	// ClientFor creates a client that operates on a new clone of the repo.
-	ClientFor(org, repo string) (RepoClient, error)
+	ClientFor(org, repo string, sparseCheckoutPatterns []string) (RepoClient, error)
 
 	// Clean removes the caches used to generate clients
 	Clean() error
@@ -245,7 +246,7 @@ func (c *clientFactory) ClientFromDir(org, repo, dir string) (RepoClient, error)
 // In that case, it must do a full git mirror clone. For large repos, this can
 // take a while. Once that is done, it will do a git fetch instead of a clone,
 // which will usually take at most a few seconds.
-func (c *clientFactory) ClientFor(org, repo string) (RepoClient, error) {
+func (c *clientFactory) ClientFor(org, repo string, sparseCheckoutPatterns []string) (RepoClient, error) {
 	start := time.Now()
 	cacheDir := path.Join(c.cacheDir, org, repo)
 	l := c.logger.WithFields(logrus.Fields{"org": org, "repo": repo, "dir": cacheDir})
@@ -292,6 +293,12 @@ func (c *clientFactory) ClientFor(org, repo string) (RepoClient, error) {
 	if err := repoClientCloner.Clone(cacheDir); err != nil {
 		return nil, err
 	}
+	sparseCheckout, _ := strconv.ParseBool(os.Getenv("SPARSE_CHECKOUT"))
+	if sparseCheckout && len(sparseCheckoutPatterns) > 0 {
+		if err := repoClient.SetSparseCheckoutPatterns(sparseCheckoutPatterns); err != nil {
+			return nil, err
+		}
+	}
 
 	duration := time.Now().Sub(start)
 	l.WithField("Duration", duration.String()).Debug("cloned repository")

pkg/git/v2/interactor.go

@@ -22,6 +22,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"strconv"
 	"strings"
 
 	"github.com/sirupsen/logrus"
@@ -70,6 +71,8 @@ type Interactor interface {
 	HasSHA(ref string) (string, error)
 	// Pull pulls any changes into a branch from the remote
 	Pull() error
+	// SetSparseCheckoutPatterns initialises att set patterns for sparse checkout
+	SetSparseCheckoutPatterns(sparseCheckoutPatterns []string) error
 }
 
 // cacher knows how to cache and update repositories in a central cache
@@ -109,23 +112,38 @@ func (i *interactor) Clean() error {
 	return os.RemoveAll(i.dir)
 }
 
-// Clone clones the repository from a local path.
-func (i *interactor) Clone(from string) error {
-	i.logger.Debugf("Creating a clone of the repo at %s from %s", i.dir, from)
-	if out, err := i.executor.Run("clone", from, i.dir); err != nil {
+// Clone clones the repository from a repository.
+func (i *interactor) Clone(repo string) error {
+	sparseCheckout, _ := strconv.ParseBool(os.Getenv("SPARSE_CHECKOUT"))
+	i.logger.Debugf("Creating a clone of the repo at %s from %s", i.dir, repo)
+	args := []string{"clone", "--no-checkout", "--depth=10", "--no-single-branch"}
+	if sparseCheckout {
+		args = append(args, "--filter=blob:none")
+	}
+	if out, err := i.executor.Run(append(args, repo, i.dir)...); err != nil {
 		return fmt.Errorf("error creating a clone: %v %v", err, string(out))
 	}
 	return nil
 }
 
+func (i *interactor) SetSparseCheckoutPatterns(sparseCheckoutPatterns []string) error {
+	if out, err := i.executor.Run("sparse-checkout", "init"); err != nil {
+		return fmt.Errorf("failed to init sparse checkout: %v. output: %s", err, string(out))
+	}
+	if out, err := i.executor.Run(append([]string{"sparse-checkout", "set"}, sparseCheckoutPatterns...)...); err != nil {
+		return fmt.Errorf("failed to set sparse checkout patterns to %v: %v. output: %s", sparseCheckoutPatterns, err, string(out))
+	}
+	return nil
+}
+
 // MirrorClone sets up a mirror of the source repository.
 func (i *interactor) MirrorClone() error {
 	i.logger.Debugf("Creating a mirror of the repo at %s", i.dir)
 	remote, err := i.remote()
 	if err != nil {
 		return fmt.Errorf("could not resolve remote for cloning: %v", err)
 	}
-	out, err := i.executor.Run("clone", "--mirror", remote, i.dir)
+	out, err := i.executor.Run("clone", "--mirror", "--depth=10", "--no-single-branch", remote, i.dir)
 	if err != nil {
 		// the returned error is not being reported
 		i.logger.Errorf("error creating a mirror clone: %v %v", err, string(out))