From a45eaffb51e970318943821856f10b2ace7300d5 Mon Sep 17 00:00:00 2001 From: John Soo Date: Thu, 10 Mar 2022 22:54:31 -0800 Subject: [PATCH 1/2] Test directory artifact paths. --- agent/artifact_uploader_test.go | 23 ++++++++++++++++--- .../artifacts/nested/nested0/logfile.out | 1 + test/fixtures/artifacts/nested/output.log | 1 + 3 files changed, 22 insertions(+), 3 deletions(-) create mode 100644 test/fixtures/artifacts/nested/nested0/logfile.out create mode 100644 test/fixtures/artifacts/nested/output.log diff --git a/agent/artifact_uploader_test.go b/agent/artifact_uploader_test.go index d88c4b3705..da13ab89f6 100644 --- a/agent/artifact_uploader_test.go +++ b/agent/artifact_uploader_test.go @@ -80,12 +80,29 @@ func TestCollect(t *testing.T) { Sha1Sum: "bd4caf2e01e59777744ac1d52deafa01c2cb9bfd", Sha256Sum: "fc5e8608c7772e4ae834fbc47eec3d902099eb3599f5191e40d9e3d9b3764b0e", }, + { + Name: "output.log", + Path: []string{"test", "fixtures", "artifacts", "nested", "output.log"}, + AbsolutePath: filepath.Join(root, "test", "fixtures", "artifacts", "nested", "output.log"), + GlobPath: filepath.Join("test", "fixtures", "artifacts", "nested"), + FileSize: 14, + Sha1Sum: "1ec2e13e3871a980eaee0f11c6f47ad1d1f97b3e", + }, + { + Name: "logfile.out", + Path: []string{"test", "fixtures", "artifacts", "nested", "nested0", "logfile.out"}, + AbsolutePath: filepath.Join(root, "test", "fixtures", "artifacts", "nested", "nested0", "logfile.out"), + GlobPath: filepath.Join("test", "fixtures", "artifacts", "nested"), + FileSize: 19, + Sha1Sum: "e30f8100faed6430e6fd617da8ea17100ac97188", + }, } uploader := NewArtifactUploader(logger.Discard, nil, ArtifactUploaderConfig{ - Paths: fmt.Sprintf("%s;%s", + Paths: fmt.Sprintf("%s;%s;%s", filepath.Join("test", "fixtures", "artifacts", "**/*.jpg"), filepath.Join(root, "test", "fixtures", "artifacts", "**/*.gif"), + filepath.Join("test", "fixtures", "artifacts", "nested"), ), }) @@ -108,13 +125,13 @@ func TestCollect(t *testing.T) { if err != nil { t.Fatalf("[normalised-upload-paths disabled] uploader.Collect() error = %v", err) } - assert.Equal(t, 5, len(artifactsWithoutExperimentEnabled)) + assert.Equal(t, 7, len(artifactsWithoutExperimentEnabled)) artifactsWithExperimentEnabled, err := uploader.Collect(ctxExpEnabled) if err != nil { t.Fatalf("[normalised-upload-paths enabled] uploader.Collect() error = %v", err) } - assert.Equal(t, 5, len(artifactsWithExperimentEnabled)) + assert.Equal(t, 7, len(artifactsWithExperimentEnabled)) // These test cases use filepath.Join, which uses per-OS path separators; // this is the behaviour without normalised-upload-paths. diff --git a/test/fixtures/artifacts/nested/nested0/logfile.out b/test/fixtures/artifacts/nested/nested0/logfile.out new file mode 100644 index 0000000000..0364518667 --- /dev/null +++ b/test/fixtures/artifacts/nested/nested0/logfile.out @@ -0,0 +1 @@ +logging out to file \ No newline at end of file diff --git a/test/fixtures/artifacts/nested/output.log b/test/fixtures/artifacts/nested/output.log new file mode 100644 index 0000000000..9d0445bbd1 --- /dev/null +++ b/test/fixtures/artifacts/nested/output.log @@ -0,0 +1 @@ +logging output \ No newline at end of file From 5af6007188d61debf7e38f4cb30bb1523fbfc0f8 Mon Sep 17 00:00:00 2001 From: John Soo Date: Fri, 11 Mar 2022 11:10:12 -0800 Subject: [PATCH 2/2] Collect artifacts in directories. --- agent/artifact_uploader.go | 82 +++++++++++++++++++++++++++++++++++++- 1 file changed, 81 insertions(+), 1 deletion(-) diff --git a/agent/artifact_uploader.go b/agent/artifact_uploader.go index 2166773e25..ae9e484f45 100644 --- a/agent/artifact_uploader.go +++ b/agent/artifact_uploader.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "io" + "io/fs" "os" "path/filepath" "runtime" @@ -117,7 +118,86 @@ func (a *ArtifactUploader) Collect(ctx context.Context) (artifacts []*api.Artifa // file paths are deduplicated after resolving globs etc seenPaths := make(map[string]bool) - for _, globPath := range strings.Split(a.conf.Paths, ArtifactPathDelimiter) { + prospects := strings.Split(a.conf.Paths, ArtifactPathDelimiter) + + // Handle directories first + for _, literalPath := range prospects { + // I think this is a bug; ' name.ext ' is a valid filepath. + // Keeping it because it was handled in the globbing case. + literalPath = strings.TrimSpace(literalPath) + if literalPath == "" { + continue + } + + a.logger.Debug("Searching for %s", literalPath) + + absolutePath, err := filepath.Abs(literalPath) + if err != nil { + return nil, err + } + + if _, ok := seenPaths[absolutePath]; ok { + a.logger.Debug("Skipping duplicate path %s", literalPath) + continue + } + + if isDir(absolutePath) { + a.logger.Debug("Searching for %s", literalPath) + + err := filepath.WalkDir(absolutePath, func(p string, d fs.DirEntry, error error) error { + if error != nil { + return error + } + + if _, ok := seenPaths[p]; ok { + a.logger.Debug("Skipping duplicate path %s", p); + return nil + } + + if d.IsDir() { + return nil + } + + path := p + + if a.conf.GlobResolveFollowSymlinks { + resolvedLink, err := filepath.EvalSymlinks(path) + if err != nil { + return err + } + path = resolvedLink + } + + seenPaths[p] = true + + path, err := filepath.Rel(wd, path) + if err != nil { + return err + } + + if experiments.IsEnabled(`normalised-upload-paths`) { + // Convert any Windows paths to Unix/URI form + path = filepath.ToSlash(path) + } + + // Build an artifact object using the paths we have. + artifact, err := a.build(path, p, literalPath) + if err != nil { + return err + } + + artifacts = append(artifacts, artifact) + + + return nil + }) + if err != nil { + return nil, err + } + } + } + + for _, globPath := range prospects { globPath = strings.TrimSpace(globPath) if globPath == "" { continue