diff --git a/agent/artifact_downloader.go b/agent/artifact_downloader.go index 0b8bc12106..7537546e82 100644 --- a/agent/artifact_downloader.go +++ b/agent/artifact_downloader.go @@ -12,6 +12,7 @@ import ( "github.com/aws/aws-sdk-go/service/s3" "github.com/buildkite/agent/v3/api" + iartifact "github.com/buildkite/agent/v3/internal/artifact" "github.com/buildkite/agent/v3/logger" "github.com/buildkite/agent/v3/pool" ) @@ -57,14 +58,14 @@ func NewArtifactDownloader(l logger.Logger, ac APIClient, c ArtifactDownloaderCo func (a *ArtifactDownloader) Download(ctx context.Context) error { // Turn the download destination into an absolute path and confirm it exists - downloadDestination, _ := filepath.Abs(a.conf.Destination) - fileInfo, err := os.Stat(downloadDestination) + destination, _ := filepath.Abs(a.conf.Destination) + fileInfo, err := os.Stat(destination) if err != nil { return fmt.Errorf("Could not find information about destination: %s %v", - downloadDestination, err) + destination, err) } if !fileInfo.IsDir() { - return fmt.Errorf("%s is not a directory", downloadDestination) + return fmt.Errorf("%s is not a directory", destination) } artifacts, err := NewArtifactSearcher(a.logger, a.apiClient, a.conf.BuildID). @@ -79,7 +80,7 @@ func (a *ArtifactDownloader) Download(ctx context.Context) error { return errors.New("No artifacts found for downloading") } - a.logger.Info("Found %d artifacts. Starting to download to: %s", artifactCount, downloadDestination) + a.logger.Info("Found %d artifacts. Starting to download to: %s", artifactCount, destination) p := pool.New(pool.MaxConcurrencyLimit) errors := []error{} @@ -101,46 +102,7 @@ func (a *ArtifactDownloader) Download(ctx context.Context) error { path = strings.Replace(path, `\`, `/`, -1) } - // Handle downloading from S3, GS, or RT - var dler interface { - Start(context.Context) error - } - switch { - case strings.HasPrefix(artifact.UploadDestination, "s3://"): - bucketName, _ := ParseS3Destination(artifact.UploadDestination) - dler = NewS3Downloader(a.logger, S3DownloaderConfig{ - S3Client: s3Clients[bucketName], - Path: path, - S3Path: artifact.UploadDestination, - Destination: downloadDestination, - Retries: 5, - DebugHTTP: a.conf.DebugHTTP, - }) - case strings.HasPrefix(artifact.UploadDestination, "gs://"): - dler = NewGSDownloader(a.logger, GSDownloaderConfig{ - Path: path, - Bucket: artifact.UploadDestination, - Destination: downloadDestination, - Retries: 5, - DebugHTTP: a.conf.DebugHTTP, - }) - case strings.HasPrefix(artifact.UploadDestination, "rt://"): - dler = NewArtifactoryDownloader(a.logger, ArtifactoryDownloaderConfig{ - Path: path, - Repository: artifact.UploadDestination, - Destination: downloadDestination, - Retries: 5, - DebugHTTP: a.conf.DebugHTTP, - }) - default: - dler = NewDownload(a.logger, http.DefaultClient, DownloadConfig{ - URL: artifact.URL, - Path: path, - Destination: downloadDestination, - Retries: 5, - DebugHTTP: a.conf.DebugHTTP, - }) - } + dler := a.createDownloader(artifact, path, destination, s3Clients) // If the downloaded encountered an error, lock // the pool, collect it, then unlock the pool @@ -188,3 +150,59 @@ func (a *ArtifactDownloader) generateS3Clients(artifacts []*api.Artifact) (map[s return s3Clients, nil } + +type downloader interface { + Start(context.Context) error +} + +func (a *ArtifactDownloader) createDownloader(artifact *api.Artifact, path, destination string, s3Clients map[string]*s3.S3) downloader { + // Handle downloading from S3, GS, RT, or Azure + switch { + case strings.HasPrefix(artifact.UploadDestination, "s3://"): + bucketName, _ := ParseS3Destination(artifact.UploadDestination) + return NewS3Downloader(a.logger, S3DownloaderConfig{ + S3Client: s3Clients[bucketName], + Path: path, + S3Path: artifact.UploadDestination, + Destination: destination, + Retries: 5, + DebugHTTP: a.conf.DebugHTTP, + }) + + case strings.HasPrefix(artifact.UploadDestination, "gs://"): + return NewGSDownloader(a.logger, GSDownloaderConfig{ + Path: path, + Bucket: artifact.UploadDestination, + Destination: destination, + Retries: 5, + DebugHTTP: a.conf.DebugHTTP, + }) + + case strings.HasPrefix(artifact.UploadDestination, "rt://"): + return NewArtifactoryDownloader(a.logger, ArtifactoryDownloaderConfig{ + Path: path, + Repository: artifact.UploadDestination, + Destination: destination, + Retries: 5, + DebugHTTP: a.conf.DebugHTTP, + }) + + case iartifact.IsAzureBlobPath(artifact.UploadDestination): + return iartifact.NewAzureBlobDownloader(a.logger, iartifact.AzureBlobDownloaderConfig{ + Path: path, + Repository: artifact.UploadDestination, + Destination: destination, + Retries: 5, + DebugHTTP: a.conf.DebugHTTP, + }) + + default: + return NewDownload(a.logger, http.DefaultClient, DownloadConfig{ + URL: artifact.URL, + Path: path, + Destination: destination, + Retries: 5, + DebugHTTP: a.conf.DebugHTTP, + }) + } +} diff --git a/agent/artifact_uploader.go b/agent/artifact_uploader.go index ee5fa515dd..2166773e25 100644 --- a/agent/artifact_uploader.go +++ b/agent/artifact_uploader.go @@ -15,6 +15,7 @@ import ( "time" "github.com/buildkite/agent/v3/api" + "github.com/buildkite/agent/v3/internal/artifact" "github.com/buildkite/agent/v3/internal/experiments" "github.com/buildkite/agent/v3/internal/mime" "github.com/buildkite/agent/v3/logger" @@ -246,41 +247,59 @@ func (a *ArtifactUploader) build(path string, absolutePath string, globPath stri return artifact, nil } -func (a *ArtifactUploader) upload(ctx context.Context, artifacts []*api.Artifact) error { - var uploader Uploader - var err error - - // Determine what uploader to use - if a.conf.Destination != "" { - if strings.HasPrefix(a.conf.Destination, "s3://") { - uploader, err = NewS3Uploader(a.logger, S3UploaderConfig{ - Destination: a.conf.Destination, - DebugHTTP: a.conf.DebugHTTP, - }) - } else if strings.HasPrefix(a.conf.Destination, "gs://") { - uploader, err = NewGSUploader(a.logger, GSUploaderConfig{ - Destination: a.conf.Destination, - DebugHTTP: a.conf.DebugHTTP, - }) - } else if strings.HasPrefix(a.conf.Destination, "rt://") { - uploader, err = NewArtifactoryUploader(a.logger, ArtifactoryUploaderConfig{ - Destination: a.conf.Destination, - DebugHTTP: a.conf.DebugHTTP, - }) - } else { - return fmt.Errorf("invalid upload destination: '%v'. Only s3://, gs:// or rt:// upload schemes are allowed. Did you forget to surround your artifact upload pattern in double quotes?", a.conf.Destination) +// createUploader applies some heuristics to the destination to infer which +// uploader to use. +func (a *ArtifactUploader) createUploader() (uploader Uploader, err error) { + var dest string + defer func() { + if err != nil || dest == "" { + return } + a.logger.Info("Uploading to %s (%q), using your agent configuration", dest, a.conf.Destination) + }() - a.logger.Info("Uploading to %q, using your agent configuration", a.conf.Destination) - } else { - uploader = NewFormUploader(a.logger, FormUploaderConfig{ + switch { + case a.conf.Destination == "": + a.logger.Info("Uploading to default Buildkite artifact storage") + return NewFormUploader(a.logger, FormUploaderConfig{ DebugHTTP: a.conf.DebugHTTP, + }), nil + + case strings.HasPrefix(a.conf.Destination, "s3://"): + dest = "Amazon S3" + return NewS3Uploader(a.logger, S3UploaderConfig{ + Destination: a.conf.Destination, + DebugHTTP: a.conf.DebugHTTP, }) - a.logger.Info("Uploading to default Buildkite artifact storage") + case strings.HasPrefix(a.conf.Destination, "gs://"): + dest = "Google Cloud Storage" + return NewGSUploader(a.logger, GSUploaderConfig{ + Destination: a.conf.Destination, + DebugHTTP: a.conf.DebugHTTP, + }) + + case strings.HasPrefix(a.conf.Destination, "rt://"): + dest = "Artifactory" + return NewArtifactoryUploader(a.logger, ArtifactoryUploaderConfig{ + Destination: a.conf.Destination, + DebugHTTP: a.conf.DebugHTTP, + }) + + case artifact.IsAzureBlobPath(a.conf.Destination): + dest = "Azure Blob storage" + return artifact.NewAzureBlobUploader(a.logger, artifact.AzureBlobUploaderConfig{ + Destination: a.conf.Destination, + }) + + default: + return nil, fmt.Errorf("invalid upload destination: '%v'. Only s3://*, gs://*, rt://*, or https://*.blob.core.windows.net destinations are allowed. Did you forget to surround your artifact upload pattern in double quotes?", a.conf.Destination) } +} - // Check if creation caused an error +func (a *ArtifactUploader) upload(ctx context.Context, artifacts []*api.Artifact) error { + // Determine what uploader to use + uploader, err := a.createUploader() if err != nil { return fmt.Errorf("creating uploader: %v", err) } @@ -413,7 +432,7 @@ func (a *ArtifactUploader) upload(ctx context.Context, artifacts []*api.Artifact roko.WithMaxAttempts(10), roko.WithStrategy(roko.Constant(5*time.Second)), ).DoWithContext(ctx, func(r *roko.Retrier) error { - if err := uploader.Upload(artifact); err != nil { + if err := uploader.Upload(ctx, artifact); err != nil { a.logger.Warn("%s (%s)", err, r) return err } diff --git a/agent/artifactory_uploader.go b/agent/artifactory_uploader.go index 1e583957b0..2db2690094 100644 --- a/agent/artifactory_uploader.go +++ b/agent/artifactory_uploader.go @@ -1,6 +1,7 @@ package agent import ( + "context" "crypto/md5" "crypto/sha1" "crypto/sha256" @@ -98,7 +99,7 @@ func (u *ArtifactoryUploader) URL(artifact *api.Artifact) string { return url.String() } -func (u *ArtifactoryUploader) Upload(artifact *api.Artifact) error { +func (u *ArtifactoryUploader) Upload(_ context.Context, artifact *api.Artifact) error { // Open file from filesystem u.logger.Debug("Reading file \"%s\"", artifact.AbsolutePath) f, err := os.Open(artifact.AbsolutePath) diff --git a/agent/form_uploader.go b/agent/form_uploader.go index 38395eae53..67be20e751 100644 --- a/agent/form_uploader.go +++ b/agent/form_uploader.go @@ -2,6 +2,7 @@ package agent import ( "bytes" + "context" _ "crypto/sha512" // import sha512 to make sha512 ssl certs work "fmt" "io" @@ -52,7 +53,7 @@ func (u *FormUploader) URL(artifact *api.Artifact) string { return "" } -func (u *FormUploader) Upload(artifact *api.Artifact) error { +func (u *FormUploader) Upload(_ context.Context, artifact *api.Artifact) error { if artifact.FileSize > maxFormUploadedArtifactSize { return errArtifactTooLarge{Size: artifact.FileSize} } diff --git a/agent/form_uploader_test.go b/agent/form_uploader_test.go index 03e9c4a83c..7f0550c311 100644 --- a/agent/form_uploader_test.go +++ b/agent/form_uploader_test.go @@ -2,6 +2,7 @@ package agent import ( "bytes" + "context" "errors" "fmt" "io" @@ -16,6 +17,8 @@ import ( ) func TestFormUploading(t *testing.T) { + ctx := context.Background() + server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { switch req.URL.Path { case "/buildkiteartifacts.com": @@ -106,7 +109,7 @@ func TestFormUploading(t *testing.T) { }}, } - if err := uploader.Upload(artifact); err != nil { + if err := uploader.Upload(ctx, artifact); err != nil { t.Errorf("uploader.Upload(artifact) = %v", err) } } @@ -117,6 +120,7 @@ func TestFormUploading(t *testing.T) { } func TestFormUploadFileMissing(t *testing.T) { + ctx := context.Background() server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { http.Error(rw, "Not found", http.StatusNotFound) })) @@ -154,12 +158,13 @@ func TestFormUploadFileMissing(t *testing.T) { }}, } - if err := uploader.Upload(artifact); !os.IsNotExist(err) { + if err := uploader.Upload(ctx, artifact); !os.IsNotExist(err) { t.Errorf("uploader.Upload(artifact) = %v, want os.ErrNotExist", err) } } func TestFormUploadTooBig(t *testing.T) { + ctx := context.Background() uploader := NewFormUploader(logger.Discard, FormUploaderConfig{}) const size = int64(6442450944) // 6Gb artifact := &api.Artifact{ @@ -172,7 +177,7 @@ func TestFormUploadTooBig(t *testing.T) { UploadInstructions: &api.ArtifactUploadInstructions{}, } - if err := uploader.Upload(artifact); !errors.Is(err, errArtifactTooLarge{Size: size}) { + if err := uploader.Upload(ctx, artifact); !errors.Is(err, errArtifactTooLarge{Size: size}) { t.Errorf("uploader.Upload(artifact) = %v, want errArtifactTooLarge", err) } } diff --git a/agent/gs_uploader.go b/agent/gs_uploader.go index 49e2b40d94..4da96fbe6d 100644 --- a/agent/gs_uploader.go +++ b/agent/gs_uploader.go @@ -119,7 +119,7 @@ func (u *GSUploader) URL(artifact *api.Artifact) string { return artifactURL.String() } -func (u *GSUploader) Upload(artifact *api.Artifact) error { +func (u *GSUploader) Upload(_ context.Context, artifact *api.Artifact) error { permission := os.Getenv("BUILDKITE_GS_ACL") // The dirtiest validation method ever... diff --git a/agent/s3_uploader.go b/agent/s3_uploader.go index 5aa47d4125..9e2e21895f 100644 --- a/agent/s3_uploader.go +++ b/agent/s3_uploader.go @@ -1,6 +1,7 @@ package agent import ( + "context" "fmt" "net/url" "os" @@ -80,7 +81,7 @@ func (u *S3Uploader) URL(artifact *api.Artifact) string { return url.String() } -func (u *S3Uploader) Upload(artifact *api.Artifact) error { +func (u *S3Uploader) Upload(_ context.Context, artifact *api.Artifact) error { permission, err := u.resolvePermission() if err != nil { diff --git a/agent/uploader.go b/agent/uploader.go index bddcd1c1ab..827f55f229 100644 --- a/agent/uploader.go +++ b/agent/uploader.go @@ -1,6 +1,8 @@ package agent import ( + "context" + "github.com/buildkite/agent/v3/api" ) @@ -10,5 +12,5 @@ type Uploader interface { URL(*api.Artifact) string // The actual uploading of the file - Upload(*api.Artifact) error + Upload(context.Context, *api.Artifact) error } diff --git a/clicommand/global.go b/clicommand/global.go index 9b6bd1f836..7e3c2a66a8 100644 --- a/clicommand/global.go +++ b/clicommand/global.go @@ -103,6 +103,9 @@ var RedactedVars = cli.StringSliceFlag{ "*_PRIVATE_KEY", "*_ACCESS_KEY", "*_SECRET_KEY", + // Connection strings frequently contain passwords, e.g. + // https://user:pass@host/ or Server=foo;Database=my-db;User Id=user;Password=pass; + "*_CONNECTION_STRING", }, } diff --git a/go.mod b/go.mod index 81e3467b49..3ba71fb434 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,8 @@ go 1.18 require ( cloud.google.com/go/compute/metadata v0.2.3 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.1 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0 github.com/DataDog/datadog-go/v5 v5.3.0 github.com/aws/aws-sdk-go v1.44.317 github.com/brunoscheufler/aws-ecs-metadata-go v0.0.0-20220812150832-b6b31c6eeeaf @@ -50,6 +52,9 @@ require ( require ( cloud.google.com/go/compute v1.23.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 // indirect github.com/DataDog/appsec-internal-go v1.0.0 // indirect github.com/DataDog/datadog-agent/pkg/obfuscate v0.45.0-rc.1 // indirect github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.46.0-rc.4 // indirect @@ -67,6 +72,7 @@ require ( github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/goccy/go-json v0.10.2 // indirect + github.com/golang-jwt/jwt/v5 v5.0.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/s2a-go v0.1.4 // indirect @@ -75,6 +81,7 @@ require ( github.com/googleapis/gax-go/v2 v2.12.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/lestrrat-go/blackmagic v1.0.1 // indirect github.com/lestrrat-go/httpcc v1.0.1 // indirect github.com/lestrrat-go/httprc v1.0.4 // indirect @@ -83,6 +90,7 @@ require ( github.com/outcaste-io/ristretto v0.2.1 // indirect github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect github.com/philhofer/fwd v1.1.2 // indirect + github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/qri-io/jsonpointer v0.1.1 // indirect diff --git a/go.sum b/go.sum index c080a32394..27bdfabf47 100644 --- a/go.sum +++ b/go.sum @@ -35,6 +35,17 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.1 h1:/iHxaJhsFr0+xVFfbMr5vxz848jyiWuIEDhYq3y5odY= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.1/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.1 h1:LNHhpdK7hzUcx/k1LIcuh5k7k1LGIWLQfCjaneSj7Fc= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.1/go.mod h1:uE9zaUfEQT/nbQjVi2IblCG9iaLtZsuYZ8ne+PuQ02M= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.2.0 h1:Ma67P/GGprNwsslzEH6+Kb8nybI8jpDTm4Wmzu2ReK8= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0 h1:nVocQV40OQne5613EeLayJiRAJuKlBGy+m22qWG+WRg= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.1.0/go.mod h1:7QJP7dr2wznCMeqIrhMgWGf7XpAQnVrJqDm9nvV3Cu4= +github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 h1:WpB/QDNLpMw72xHJc34BNNykqSOeEJDAWkhf0u12/Jk= +github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= @@ -107,6 +118,7 @@ github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMS github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= @@ -143,6 +155,8 @@ github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= +github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE= +github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= @@ -243,6 +257,8 @@ github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/lestrrat-go/blackmagic v1.0.1 h1:lS5Zts+5HIC/8og6cGHb0uCcNCa3OUt1ygh3Qz2Fe80= github.com/lestrrat-go/blackmagic v1.0.1/go.mod h1:UrEqBzIR2U6CnzVyUtfM6oZNMt/7O7Vohk2J0OGSAtU= github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE= @@ -283,6 +299,8 @@ github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 h1:q2e307iGHPdTGp github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o= github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw= github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0= +github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= +github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -535,6 +553,7 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/internal/artifact/azure_blob.go b/internal/artifact/azure_blob.go new file mode 100644 index 0000000000..f84dd85c13 --- /dev/null +++ b/internal/artifact/azure_blob.go @@ -0,0 +1,110 @@ +package artifact + +import ( + "fmt" + "net/url" + "os" + "path" + "strings" + + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service" + "github.com/buildkite/agent/v3/logger" +) + +// The domain suffix for Azure Blob storage. +const azureBlobHostSuffix = ".blob.core.windows.net" + +// NewAzureBlobClient creates a new Azure Blob Storage client. +func NewAzureBlobClient(l logger.Logger, storageAccountName string) (*service.Client, error) { + + // TODO: Other credential types? + // https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/sdk/azidentity#readme-credential-types + + if connStr := os.Getenv("BUILDKITE_AZURE_BLOB_CONNECTION_STRING"); connStr != "" { + l.Debug("Connecting to Azure Blob Storage using Connection String") + client, err := service.NewClientFromConnectionString(connStr, nil) + if err != nil { + return nil, fmt.Errorf("creating Azure Blob storage client with connection string: %w", err) + } + return client, nil + } + + url := fmt.Sprintf("https://%s%s/", storageAccountName, azureBlobHostSuffix) + + if accKey := os.Getenv("BUILDKITE_AZURE_BLOB_ACCESS_KEY"); accKey != "" { + l.Debug("Connecting to Azure Blob Storage using Shared Key Credential") + cred, err := service.NewSharedKeyCredential(storageAccountName, accKey) + if err != nil { + return nil, fmt.Errorf("creating Azure shared key credential: %w", err) + } + client, err := service.NewClientWithSharedKeyCredential(url, cred, nil) + if err != nil { + return nil, fmt.Errorf("creating Azure Blob storage client with a shared key credential: %w", err) + } + return client, nil + } + + l.Debug("Connecting to Azure Blob Storage using Default Azure Credential") + cred, err := azidentity.NewDefaultAzureCredential(nil) + if err != nil { + return nil, fmt.Errorf("creating default Azure credential: %w", err) + } + + client, err := service.NewClient(url, cred, nil) + if err != nil { + return nil, fmt.Errorf("creating Azure Blob storage client with default Azure credential: %w", err) + } + return client, nil +} + +// AzureBlobLocation specifies the location of a blob in Azure Blob Storage. +type AzureBlobLocation struct { + StorageAccountName string + ContainerName string + BlobPath string +} + +// URL returns an Azure Blob Storage URL for the blob. +func (l *AzureBlobLocation) URL(blob string) string { + return (&url.URL{ + Scheme: "https", + Host: l.StorageAccountName + azureBlobHostSuffix, + Path: path.Join(l.ContainerName, l.BlobPath, blob), + }).String() +} + +// String returns the location as a URL string. +func (l *AzureBlobLocation) String() string { + return l.URL("") +} + +// ParseAzureBlobLocation parses a URL into an Azure Blob Storage location. +func ParseAzureBlobLocation(loc string) (*AzureBlobLocation, error) { + u, err := url.Parse(loc) + if err != nil { + return nil, fmt.Errorf("parsing location: %w", err) + } + if u.Scheme != "https" { + return nil, fmt.Errorf("parsing location: want https:// scheme, got %q", u.Scheme) + } + san, ok := strings.CutSuffix(u.Host, azureBlobHostSuffix) + if !ok { + return nil, fmt.Errorf("parsing location: want subdomain of %s, got %q", azureBlobHostSuffix, u.Host) + } + ctr, blob, ok := strings.Cut(strings.TrimPrefix(u.Path, "/"), "/") + if !ok { + return nil, fmt.Errorf("parsing location: want container name as first segment of path, got %q", u.Path) + } + return &AzureBlobLocation{ + StorageAccountName: san, + ContainerName: ctr, + BlobPath: blob, + }, nil +} + +// IsAzureBlobPath reports if the location is an Azure Blob Storage path. +func IsAzureBlobPath(loc string) bool { + _, err := ParseAzureBlobLocation(loc) + return err == nil +} diff --git a/internal/artifact/azure_blob_downloader.go b/internal/artifact/azure_blob_downloader.go new file mode 100644 index 0000000000..39b8430448 --- /dev/null +++ b/internal/artifact/azure_blob_downloader.go @@ -0,0 +1,71 @@ +package artifact + +import ( + "context" + "os" + "path" + + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/buildkite/agent/v3/logger" +) + +// AzureBlobUploaderConfig configures AzureBlobDownloader. +type AzureBlobDownloaderConfig struct { + Path string + Repository string + Destination string + Retries int + DebugHTTP bool +} + +// AzureBlobDownloader downloads files from Azure Blob storage. +type AzureBlobDownloader struct { + logger logger.Logger + conf AzureBlobDownloaderConfig +} + +// NewAzureBlobDownloader creates a new AzureBlobDownloader. +func NewAzureBlobDownloader(l logger.Logger, c AzureBlobDownloaderConfig) *AzureBlobDownloader { + return &AzureBlobDownloader{ + logger: l, + conf: c, + } +} + +// Start starts the download. +func (d *AzureBlobDownloader) Start(ctx context.Context) error { + loc, err := ParseAzureBlobLocation(d.conf.Repository) + if err != nil { + return err + } + + d.logger.Debug("Azure Blob Storage path: %v", loc) + + client, err := NewAzureBlobClient(d.logger, loc.StorageAccountName) + if err != nil { + return err + } + + f, err := os.Create(d.conf.Path) + if err != nil { + return err + } + defer f.Close() + + fullPath := path.Join(loc.BlobPath, d.conf.Path) + + // Show a nice message that we're starting to download the file + d.logger.Debug("Downloading %s to %s", loc.URL(d.conf.Path), d.conf.Path) + + opts := &azblob.DownloadFileOptions{ + RetryReaderOptionsPerBlock: azblob.RetryReaderOptions{ + MaxRetries: int32(d.conf.Retries), + }, + } + bc := client.NewContainerClient(loc.ContainerName).NewBlobClient(fullPath) + if _, err := bc.DownloadFile(ctx, f, opts); err != nil { + return err + } + + return f.Close() +} diff --git a/internal/artifact/azure_blob_test.go b/internal/artifact/azure_blob_test.go new file mode 100644 index 0000000000..b9f80c12cc --- /dev/null +++ b/internal/artifact/azure_blob_test.go @@ -0,0 +1,134 @@ +package artifact + +import ( + "testing" + + "github.com/google/go-cmp/cmp" +) + +func TestParseAzureBlobLocation(t *testing.T) { + t.Parallel() + tests := []struct { + desc string + input string + want *AzureBlobLocation + }{ + { + desc: "example path", + input: "https://asdf.blob.core.windows.net/my-container/blob.txt", + want: &AzureBlobLocation{ + StorageAccountName: "asdf", + ContainerName: "my-container", + BlobPath: "blob.txt", + }, + }, + { + desc: "more complex path", + input: "https://asdf.blob.core.windows.net/my-container/some-directory/blob.txt?the-ignored-bit", + want: &AzureBlobLocation{ + StorageAccountName: "asdf", + ContainerName: "my-container", + BlobPath: "some-directory/blob.txt", + }, + }, + } + + for _, test := range tests { + test := test + t.Run(test.desc, func(t *testing.T) { + t.Parallel() + + got, err := ParseAzureBlobLocation(test.input) + if err != nil { + t.Errorf("ParseAzureBlobLocation(%q) error = %v", test.input, err) + } + + if diff := cmp.Diff(got, test.want); diff != "" { + t.Errorf("parsed AzureBlobLocation diff (-got +want):\n%s", diff) + } + + if !IsAzureBlobPath(test.input) { + t.Errorf("IsAzureBlobPath(%q) = false, want true", test.input) + } + }) + } +} + +func TestParseAzureBlobLocationErrors(t *testing.T) { + t.Parallel() + tests := []struct { + desc string + input string + }{ + { + desc: "not https", + input: "s3://azzyazzyazzyoioioi.blob.core.windows.net/container/file.txt", + }, + { + desc: "not .blob.core.windows.net", + input: "https://azzyazzyazzyoioioi.blorb.clorb.windorbs.net/container/file.txt", + }, + { + desc: "no container", + input: "https://azzyazzyazzyoioioi.blob.core.windows.net/file.txt", + }, + } + + for _, test := range tests { + test := test + t.Run(test.desc, func(t *testing.T) { + t.Parallel() + + if _, err := ParseAzureBlobLocation(test.input); err == nil { + t.Errorf("ParseAzureBlobLocation(%q) error = %v, want non-nil error", test.input, err) + } + + if IsAzureBlobPath(test.input) { + t.Errorf("IsAzureBlobPath(%q) = true, want false", test.input) + } + }) + } +} + +func TestAzureBlobLocationURL(t *testing.T) { + t.Parallel() + tests := []struct { + desc string + base *AzureBlobLocation + blob string + want string + }{ + { + desc: "example path", + base: &AzureBlobLocation{ + StorageAccountName: "asdf", + ContainerName: "my-container", + BlobPath: "", + }, + blob: "blob.txt", + want: "https://asdf.blob.core.windows.net/my-container/blob.txt", + }, + { + desc: "more complex path", + base: &AzureBlobLocation{ + StorageAccountName: "asdf", + ContainerName: "my-container", + BlobPath: "some-directory", + }, + blob: "blob.txt", + want: "https://asdf.blob.core.windows.net/my-container/some-directory/blob.txt", + }, + } + + for _, test := range tests { + test := test + t.Run(test.desc, func(t *testing.T) { + t.Parallel() + + got := test.base.URL(test.blob) + if diff := cmp.Diff(got, test.want); diff != "" { + t.Errorf("AzureBlobLocation.URL(%q) diff (-got +want):\n%s", test.blob, diff) + } + }) + } +} diff --git a/internal/artifact/azure_blob_uploader.go b/internal/artifact/azure_blob_uploader.go new file mode 100644 index 0000000000..106a1d0b27 --- /dev/null +++ b/internal/artifact/azure_blob_uploader.go @@ -0,0 +1,107 @@ +package artifact + +import ( + "context" + "fmt" + "os" + "path" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service" + "github.com/buildkite/agent/v3/api" + "github.com/buildkite/agent/v3/logger" +) + +// AzureBlobUploaderConfig configures AzureBlobUploader. +type AzureBlobUploaderConfig struct { + // The destination which includes the storage account name and the path. + // For example, "https://my-storage-account.blob.core.windows.net/my-container/my-virtual-directory/artifacts-go-here/" + Destination string +} + +// AzureBlobUploader uploads artifacts to Azure Blob Storage. +type AzureBlobUploader struct { + // Upload location in Azure Blob Storage. + loc *AzureBlobLocation + + // Azure Blob storage client. + client *service.Client + + // The original configuration + conf AzureBlobUploaderConfig + + // The logger instance to use + logger logger.Logger +} + +// NewAzureBlobUploader creates a new AzureBlobUploader. +func NewAzureBlobUploader(l logger.Logger, c AzureBlobUploaderConfig) (*AzureBlobUploader, error) { + loc, err := ParseAzureBlobLocation(c.Destination) + if err != nil { + return nil, err + } + + // Initialize the Azure client, and authenticate it + client, err := NewAzureBlobClient(l, loc.StorageAccountName) + if err != nil { + return nil, err + } + + return &AzureBlobUploader{ + logger: l, + conf: c, + client: client, + loc: loc, + }, nil +} + +// URL returns the full destination URL of an artifact. +func (u *AzureBlobUploader) URL(artifact *api.Artifact) string { + outURL := u.loc.URL(artifact.Path) + + // Generate a shared access signature token for the URL? + sasDur := os.Getenv("BUILDKITE_AZURE_BLOB_SAS_TOKEN_DURATION") + if sasDur == "" { + // no. plain URL. + return outURL + } + + dur, err := time.ParseDuration(sasDur) + if err != nil { + u.logger.Error("BUILDKITE_AZURE_BLOB_SAS_TOKEN_DURATION is not a valid duration: %v", err) + return outURL + } + + fullPath := path.Join(u.loc.BlobPath, artifact.Path) + blobClient := u.client.NewContainerClient(u.loc.ContainerName).NewBlobClient(fullPath) + perms := sas.BlobPermissions{Read: true} + expiry := time.Now().Add(dur) + + sasURL, err := blobClient.GetSASURL(perms, expiry, nil) + if err != nil { + u.logger.Error("Couldn't generate SAS URL for container: %v", err) + return outURL + } + + u.logger.Debug("Generated Azure Blob SAS URL %q", sasURL) + return sasURL +} + +// Upload uploads an artifact file. +func (u *AzureBlobUploader) Upload(ctx context.Context, artifact *api.Artifact) error { + u.logger.Debug("Reading file %q", artifact.AbsolutePath) + f, err := os.Open(artifact.AbsolutePath) + if err != nil { + return fmt.Errorf("failed to open file %q (%v)", artifact.AbsolutePath, err) + } + defer f.Close() + + blobName := path.Join(u.loc.BlobPath, artifact.Path) + + u.logger.Debug("Uploading %s to %s", artifact.Path, u.loc.URL(blobName)) + + bbc := u.client.NewContainerClient(u.loc.ContainerName).NewBlockBlobClient(blobName) + _, err = bbc.UploadFile(ctx, f, nil) + return err +}