diff --git a/internal/plugin/autodetect/auto_detect_util.go b/internal/plugin/autodetect/auto_detect_util.go new file mode 100644 index 00000000..2b901132 --- /dev/null +++ b/internal/plugin/autodetect/auto_detect_util.go @@ -0,0 +1,102 @@ +package autodetect + +import ( + "crypto/md5" // #nosec + "encoding/hex" + "io" + "os" + "path/filepath" +) + +type buildToolInfo struct { + globToDetect string + tool string + preparer RepoPreparer +} + +func DetectDirectoriesToCache() ([]string, []string, string, error) { + var buildToolInfoMapping = []buildToolInfo{ + { + globToDetect: "*pom.xml", + tool: "maven", + preparer: newMavenPreparer(), + }, + { + globToDetect: "*build.gradle", + tool: "gradle", + preparer: newGradlePreparer(), + }, + } + + var directoriesToCache []string + + var buildToolsDetected []string + + var hashes string + + for _, supportedTool := range buildToolInfoMapping { + hash, err := hashIfFileExist(supportedTool.globToDetect) + if err != nil { + return nil, nil, "", err + } + + if hash != "" { + dirToCache, err := supportedTool.preparer.PrepareRepo() + if err != nil { + return nil, nil, "", err + } + + directoriesToCache = append(directoriesToCache, dirToCache) + buildToolsDetected = append(buildToolsDetected, supportedTool.tool) + hashes += hash + } + } + + return directoriesToCache, buildToolsDetected, hashes, nil +} + +func hashIfFileExist(glob string) (string, error) { + matches, _ := filepath.Glob(glob) + + if len(matches) == 0 { + return "", nil + } + + return calculateMd5FromFiles(matches) +} + +func calculateMd5FromFiles(fileList []string) (string, error) { + rootMostFile := shortestPath(fileList) + file, err := os.Open(rootMostFile) + + if err != nil { + return "", err + } + + defer file.Close() + + if err != nil { + return "", err + } + + hash := md5.New() // #nosec + _, err = io.Copy(hash, file) + + if err != nil { + return "", err + } + + return hex.EncodeToString(hash.Sum(nil)), nil +} + +func shortestPath(input []string) (shortest string) { + size := len(input[0]) + for _, v := range input { + if len(v) <= size { + shortest = v + size = len(v) + } + } + + return +} diff --git a/internal/plugin/autodetect/auto_detect_util_test.go b/internal/plugin/autodetect/auto_detect_util_test.go new file mode 100644 index 00000000..366c1702 --- /dev/null +++ b/internal/plugin/autodetect/auto_detect_util_test.go @@ -0,0 +1,120 @@ +package autodetect + +import ( + "os" + "path/filepath" + "testing" + + "github.com/meltwater/drone-cache/test" +) + +const ( + pomFile = "pom.xml" + nestedDirectory = "dir" + bazelBuildFile = "build.gradle" + testFileContent = "some_content" + testFileContent2 = "some_other_content" + toolMaven = "maven" + toolMavenDir = ".m2/repository" + toolGradle = "gradle" + toolGradleDir = ".gradle" +) + +func TestDetectDirectoriesToCacheMaven(t *testing.T) { + f, err := os.Create(pomFile) + test.Ok(t, err) + defer f.Close() + _, err = f.WriteString(testFileContent) + test.Ok(t, err) + directoriesToCache, buildToolsDetected, hashes, err := DetectDirectoriesToCache() + test.Ok(t, err) + test.Ok(t, os.RemoveAll(pomFile)) + expectedCacheDir := []string{toolMavenDir} + expectedDetectedTool := []string{toolMaven} + test.Equals(t, directoriesToCache, expectedCacheDir) + test.Equals(t, buildToolsDetected, expectedDetectedTool) + test.Equals(t, hashes, "baab6c16d9143523b7865d46896e4596") +} + +func TestDetectDirectoriesToCacheMavenMultiMaven(t *testing.T) { + f, err := os.Create(pomFile) + test.Ok(t, err) + defer f.Close() + + _, err = f.WriteString(testFileContent) + + test.Ok(t, err) + test.Ok(t, os.MkdirAll(nestedDirectory, 0755)) + + f2, err := os.Create(filepath.Join(nestedDirectory, pomFile)) + + test.Ok(t, err) + defer f2.Close() + + _, err = f2.WriteString(testFileContent2) + + test.Ok(t, err) + directoriesToCache, buildToolsDetected, hashes, err := DetectDirectoriesToCache() + + test.Ok(t, err) + test.Ok(t, os.RemoveAll(pomFile)) + test.Ok(t, os.RemoveAll(filepath.Join(nestedDirectory, pomFile))) + + expectedCacheDir := []string{toolMavenDir} + expectedDetectedTool := []string{toolMaven} + + test.Equals(t, directoriesToCache, expectedCacheDir) + test.Equals(t, buildToolsDetected, expectedDetectedTool) + test.Equals(t, hashes, "baab6c16d9143523b7865d46896e4596") +} + +func TestDetectDirectoriesToCacheBazel(t *testing.T) { + f, err := os.Create(bazelBuildFile) + test.Ok(t, err) + defer f.Close() + + _, err = f.WriteString(testFileContent) + test.Ok(t, err) + + directoriesToCache, buildToolsDetected, hashes, err := DetectDirectoriesToCache() + + test.Ok(t, os.RemoveAll(bazelBuildFile)) + test.Ok(t, err) + + expectedCacheDir := []string{toolGradleDir} + expectedDetectedTool := []string{toolGradle} + + test.Equals(t, directoriesToCache, expectedCacheDir) + test.Equals(t, buildToolsDetected, expectedDetectedTool) + test.Equals(t, hashes, "baab6c16d9143523b7865d46896e4596") +} + +func TestDetectDirectoriesToCacheCombined(t *testing.T) { + f, err := os.Create(bazelBuildFile) + test.Ok(t, err) + defer f.Close() + + _, err = f.WriteString(testFileContent) + + test.Ok(t, err) + f2, err := os.Create(pomFile) + + test.Ok(t, err) + defer f2.Close() + + _, err = f2.WriteString(testFileContent2) + + test.Ok(t, err) + directoriesToCache, buildToolsDetected, hashes, err := DetectDirectoriesToCache() + + test.Ok(t, os.RemoveAll(bazelBuildFile)) + test.Ok(t, os.RemoveAll(pomFile)) + test.Ok(t, err) + + expectedCacheDir := []string{toolMavenDir, toolGradleDir} + expectedDetectedTool := []string{toolMaven, toolGradle} + + test.Equals(t, directoriesToCache, expectedCacheDir) + test.Equals(t, buildToolsDetected, expectedDetectedTool) + test.Equals(t, hashes, "1eb00e74bffac0c4fa2d6dbfd8c26cb7baab6c16d9143523b7865d46896e4596") +} diff --git a/internal/plugin/autodetect/prepare_gradle.go b/internal/plugin/autodetect/prepare_gradle.go new file mode 100644 index 00000000..27bc87e3 --- /dev/null +++ b/internal/plugin/autodetect/prepare_gradle.go @@ -0,0 +1,48 @@ +package autodetect + +import ( + "errors" + "fmt" + "os" +) + +type gradlePreparer struct{} + +func newGradlePreparer() *gradlePreparer { + return &gradlePreparer{} +} + +func (*gradlePreparer) PrepareRepo() (string, error) { + fileName := "gradle.properties" + pathToCache := ".gradle" + cmdToOverrideRepo := fmt.Sprintf("systemProp.gradle.user.home=/%s/\norg.gradle.caching=true\n", pathToCache) + + if _, err := os.Stat(fileName); errors.Is(err, os.ErrNotExist) { + f, err := os.Create(fileName) + if err != nil { + return "", err + } + defer f.Close() + _, err = f.WriteString(cmdToOverrideRepo) + + if err != nil { + return "", err + } + + return pathToCache, nil + } + + f, err := os.OpenFile(fileName, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + + if err != nil { + return "", err + } + defer f.Close() + _, err = f.WriteString(cmdToOverrideRepo) + + if err != nil { + return "", err + } + + return pathToCache, nil +} diff --git a/internal/plugin/autodetect/prepare_maven.go b/internal/plugin/autodetect/prepare_maven.go new file mode 100644 index 00000000..d67d6e43 --- /dev/null +++ b/internal/plugin/autodetect/prepare_maven.go @@ -0,0 +1,56 @@ +package autodetect + +import ( + "errors" + "fmt" + "os" + "path/filepath" +) + +type mavenPreparer struct{} + +func newMavenPreparer() *mavenPreparer { + return &mavenPreparer{} +} +func (*mavenPreparer) PrepareRepo() (string, error) { + configPath := ".mvn" + fileName := "maven.config" + pathToCache := filepath.Join(".m2", "repository") + cmdToOverrideRepo := fmt.Sprintf(" -Dmaven.repo.local=%s ", pathToCache) + + if _, err := os.Stat(filepath.Join(configPath, fileName)); errors.Is(err, os.ErrNotExist) { + err := os.MkdirAll(configPath, os.ModePerm) + + if err != nil { + return "", err + } + + f, err := os.Create(filepath.Join(configPath, fileName)) + + if err != nil { + return "", err + } + defer f.Close() + _, err = f.WriteString(cmdToOverrideRepo) + + if err != nil { + return "", err + } + + return pathToCache, err + } + + f, err := os.OpenFile(filepath.Join(configPath, fileName), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) + + if err != nil { + return "", err + } + defer f.Close() + _, err = f.WriteString(cmdToOverrideRepo) + + if err != nil { + return "", err + } + + return pathToCache, nil +} diff --git a/internal/plugin/autodetect/prepare_repo.go b/internal/plugin/autodetect/prepare_repo.go new file mode 100644 index 00000000..160120ae --- /dev/null +++ b/internal/plugin/autodetect/prepare_repo.go @@ -0,0 +1,6 @@ +package autodetect + +type RepoPreparer interface { + // PrepareRepo change local files to a state where cache intelligence options can be performed + PrepareRepo() (string, error) +} diff --git a/internal/plugin/config.go b/internal/plugin/config.go index b7061a06..5f12faeb 100644 --- a/internal/plugin/config.go +++ b/internal/plugin/config.go @@ -17,11 +17,13 @@ type Config struct { CacheKeyTemplate string RemoteRoot string LocalRoot string + AccountID string // Modes - Debug bool - Rebuild bool - Restore bool + Debug bool + Rebuild bool + Restore bool + AutoDetect bool // Optional SkipSymlinks bool diff --git a/internal/plugin/plugin.go b/internal/plugin/plugin.go index a09d9255..8e7efa0e 100644 --- a/internal/plugin/plugin.go +++ b/internal/plugin/plugin.go @@ -6,6 +6,9 @@ import ( "fmt" "os" "path/filepath" + "strings" + + "github.com/meltwater/drone-cache/internal/plugin/autodetect" "github.com/meltwater/drone-cache/archive" "github.com/meltwater/drone-cache/cache" @@ -84,16 +87,39 @@ func (p *Plugin) Exec() error { // nolint:funlen // } var generator key.Generator - if cfg.CacheKeyTemplate != "" { - generator = keygen.NewMetadata(p.logger, cfg.CacheKeyTemplate, p.Metadata) - if err := generator.Check(); err != nil { - return fmt.Errorf("parse failed, falling back to default, %w", err) - } - options = append(options, cache.WithFallbackGenerator(keygen.NewHash(p.Metadata.Commit.Branch))) - } else { - generator = keygen.NewHash(p.Metadata.Commit.Branch) - options = append(options, cache.WithFallbackGenerator(keygen.NewStatic(p.Metadata.Commit.Branch))) + switch { + case cfg.CacheKeyTemplate != "": + { + generator = keygen.NewMetadata(p.logger, cfg.CacheKeyTemplate, p.Metadata) + if err := generator.Check(); err != nil { + return fmt.Errorf("parse failed, falling back to default, %w", err) + } + + options = append(options, cache.WithFallbackGenerator(keygen.NewHash(p.Metadata.Commit.Branch))) + } + case cfg.AutoDetect: + { + dirs, buildTools, hash, err := autodetect.DetectDirectoriesToCache() + if err != nil { + return fmt.Errorf("autodetect enabled but failed to detect, falling back to default, %w", err) + } + p.logger.Log("msg", "build tools detected: "+strings.Join(buildTools, ", ")) + if len(p.Config.Mount) == 0 { + p.Config.Mount = dirs + } + generator = keygen.NewMetadata(p.logger, cfg.AccountID+"/"+hash, p.Metadata) + if err := generator.Check(); err != nil { + return fmt.Errorf("parse failed, falling back to default, %w", err) + } + + options = append(options, cache.WithFallbackGenerator(keygen.NewHash(cfg.AccountID+p.Metadata.Commit.Branch))) + } + default: + { + generator = keygen.NewHash(p.Metadata.Commit.Branch) + options = append(options, cache.WithFallbackGenerator(keygen.NewStatic(p.Metadata.Commit.Branch))) + } } options = append(options, cache.WithOverride(p.Config.Override), diff --git a/main.go b/main.go index b9ac1af7..5972f69b 100644 --- a/main.go +++ b/main.go @@ -267,6 +267,17 @@ func main() { Value: true, EnvVars: []string{"PLUGIN_OVERRIDE"}, }, + &cli.BoolFlag{ + Name: "auto-detect", + Usage: "automatically detect the cache directory and generate cache key", + Value: false, + EnvVars: []string{"PLUGIN_AUTO_CACHE"}, + }, + &cli.StringFlag{ + Name: "account-id", + Usage: "account-id used for automatic key generation", + EnvVars: []string{"PLUGIN_ACCOUNT_ID"}, + }, // CACHE-KEYS // REBUILD-KEYS // RESTORE-KEYS @@ -549,6 +560,8 @@ func run(c *cli.Context) error { Mount: c.StringSlice("mount"), Rebuild: c.Bool("rebuild"), Restore: c.Bool("restore"), + AutoDetect: c.Bool("auto-detect"), + AccountID: c.String("account-id"), RemoteRoot: c.String("remote-root"), LocalRoot: c.String("local-root"), Override: c.Bool("override"),