From 90d2f7593dbe96a69c8f17e1f6a34ad465f236f5 Mon Sep 17 00:00:00 2001 From: Taylor Silva Date: Sat, 8 Jun 2024 10:59:42 -0400 Subject: [PATCH 1/2] Add assume_aws_role_arn that uses EC2 instance profile Setting this field will cause the S3 resource to assume the role specified using the Concourse workers IAM role to authenticate to the STS API Signed-off-by: Taylor Silva --- README.md | 5 +++++ cmd/check/main.go | 8 ++++++-- cmd/in/main.go | 8 ++++++-- cmd/out/main.go | 8 ++++++-- models.go | 1 + s3client.go | 35 +++++++++++++++++++++++------------ 6 files changed, 47 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index ad6fc932..c2aaa04f 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,11 @@ version numbers. * `aws_role_arn`: *Optional.* The AWS role ARN to be assumed by the user identified by `access_key_id` and `secret_access_key`. +* `assume_aws_role_arn`: *Optional.* The AWS role ARN to be assumed using the + Concourse workers EC2 instance credentials. The workers instance role must + have permissions to assume the role. **This is different from the + `aws_role_arn` and takes precedence over it** + * `region_name`: *Optional.* The region the bucket is in. Defaults to `us-east-1`. diff --git a/cmd/check/main.go b/cmd/check/main.go index e79b6a32..c5a80a84 100644 --- a/cmd/check/main.go +++ b/cmd/check/main.go @@ -4,7 +4,7 @@ import ( "encoding/json" "os" - "github.com/concourse/s3-resource" + s3resource "github.com/concourse/s3-resource" "github.com/concourse/s3-resource/check" ) @@ -16,18 +16,22 @@ func main() { request.Source.AccessKeyID, request.Source.SecretAccessKey, request.Source.SessionToken, + request.Source.AssumeAwsRoleARN, request.Source.RegionName, request.Source.Endpoint, request.Source.DisableSSL, request.Source.SkipSSLVerification, ) - client := s3resource.NewS3Client( + client, err := s3resource.NewS3Client( os.Stderr, awsConfig, request.Source.UseV2Signing, request.Source.AwsRoleARN, ) + if err != nil { + s3resource.Fatal("failed to create new S3 client", err) + } command := check.NewCommand(client) response, err := command.Run(request) diff --git a/cmd/in/main.go b/cmd/in/main.go index c46ea802..fba0a10f 100644 --- a/cmd/in/main.go +++ b/cmd/in/main.go @@ -9,7 +9,7 @@ import ( "strings" "github.com/aws/aws-sdk-go/aws" - "github.com/concourse/s3-resource" + s3resource "github.com/concourse/s3-resource" "github.com/concourse/s3-resource/in" ) @@ -28,6 +28,7 @@ func main() { request.Source.AccessKeyID, request.Source.SecretAccessKey, request.Source.SessionToken, + request.Source.AssumeAwsRoleARN, request.Source.RegionName, request.Source.Endpoint, request.Source.DisableSSL, @@ -50,12 +51,15 @@ func main() { awsConfig.Endpoint = aws.String(fmt.Sprintf("%s://%s", cloudfrontUrl.Scheme, fqdn)) } - client := s3resource.NewS3Client( + client, err := s3resource.NewS3Client( os.Stderr, awsConfig, request.Source.UseV2Signing, request.Source.AwsRoleARN, ) + if err != nil { + s3resource.Fatal("failed to create new S3 client", err) + } command := in.NewCommand(client) diff --git a/cmd/out/main.go b/cmd/out/main.go index b0138d3b..46ba308a 100644 --- a/cmd/out/main.go +++ b/cmd/out/main.go @@ -4,7 +4,7 @@ import ( "encoding/json" "os" - "github.com/concourse/s3-resource" + s3resource "github.com/concourse/s3-resource" "github.com/concourse/s3-resource/out" ) @@ -23,18 +23,22 @@ func main() { request.Source.AccessKeyID, request.Source.SecretAccessKey, request.Source.SessionToken, + request.Source.AssumeAwsRoleARN, request.Source.RegionName, request.Source.Endpoint, request.Source.DisableSSL, request.Source.SkipSSLVerification, ) - client := s3resource.NewS3Client( + client, err := s3resource.NewS3Client( os.Stderr, awsConfig, request.Source.UseV2Signing, request.Source.AwsRoleARN, ) + if err != nil { + s3resource.Fatal("failed to create new S3 client", err) + } command := out.NewCommand(os.Stderr, client) response, err := command.Run(sourceDir, request) diff --git a/models.go b/models.go index ccf23a61..d161501b 100644 --- a/models.go +++ b/models.go @@ -5,6 +5,7 @@ type Source struct { SecretAccessKey string `json:"secret_access_key"` SessionToken string `json:"session_token"` AwsRoleARN string `json:"aws_role_arn"` + AssumeAwsRoleARN string `json:"assume_aws_role_arn"` Bucket string `json:"bucket"` Regexp string `json:"regexp"` VersionedFile string `json:"versioned_file"` diff --git a/s3client.go b/s3client.go index 4a3e9d76..1c0622fe 100644 --- a/s3client.go +++ b/s3client.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "os" "strings" "time" @@ -73,8 +72,11 @@ func NewS3Client( awsConfig *aws.Config, useV2Signing bool, roleToAssume string, -) S3Client { - sess := session.New(awsConfig) +) (S3Client, error) { + sess, err := session.NewSession(awsConfig) + if err != nil { + return nil, err + } assumedRoleAwsConfig := fetchCredentialsForRoleIfDefined(roleToAssume, awsConfig) @@ -89,12 +91,12 @@ func NewS3Client( session: sess, progressOutput: progressOutput, - } + }, nil } func fetchCredentialsForRoleIfDefined(roleToAssume string, awsConfig *aws.Config) aws.Config { assumedRoleAwsConfig := aws.Config{} - if len(roleToAssume) != 0 { + if roleToAssume != "" { stsConfig := awsConfig.Copy() stsConfig.Endpoint = nil stsSession := session.Must(session.NewSession(stsConfig)) @@ -109,6 +111,7 @@ func NewAwsConfig( accessKey string, secretKey string, sessionToken string, + assumeRoleArn string, regionName string, endpoint string, disableSSL bool, @@ -116,14 +119,22 @@ func NewAwsConfig( ) *aws.Config { var creds *credentials.Credentials - if accessKey == "" && secretKey == "" { - creds = credentials.AnonymousCredentials - } else { - creds = credentials.NewStaticCredentials(accessKey, secretKey, sessionToken) + if regionName == "" { + regionName = "us-east-1" } - if len(regionName) == 0 { - regionName = "us-east-1" + switch { + case assumeRoleArn != "": + sess := session.Must(session.NewSession(&aws.Config{ + Region: aws.String(regionName), + })) + creds = stscreds.NewCredentials(sess, assumeRoleArn) + + case accessKey == "" && secretKey == "": + creds = credentials.AnonymousCredentials + + default: + creds = credentials.NewStaticCredentials(accessKey, secretKey, sessionToken) } var httpClient *http.Client @@ -403,7 +414,7 @@ func (client *s3client) DownloadTags(bucketName string, remotePath string, versi return err } - return ioutil.WriteFile(localPath, tagsJSON, 0644) + return os.WriteFile(localPath, tagsJSON, 0644) } func (client *s3client) URL(bucketName string, remotePath string, private bool, versionID string) string { From cff7028511affa5ddbff1ae7914b5df6cdabd4d1 Mon Sep 17 00:00:00 2001 From: Taylor Silva Date: Sat, 8 Jun 2024 10:57:33 -0400 Subject: [PATCH 2/2] fix integration test suite and remove usage of deprecated ioutil package Signed-off-by: Taylor Silva --- check/check_suite_test.go | 3 +- check/command_test.go | 3 +- in/command.go | 11 +++---- in/command_test.go | 45 +++++++++++++------------ integration/check_test.go | 25 +++++++------- integration/in_test.go | 47 +++++++++++++-------------- integration/integration_suite_test.go | 29 ++++++++++++----- integration/out_test.go | 17 +++++----- integration/s3client_test.go | 11 +++---- out/command_test.go | 5 ++- out/out_suite_test.go | 3 +- 11 files changed, 101 insertions(+), 98 deletions(-) diff --git a/check/check_suite_test.go b/check/check_suite_test.go index 1bb48a45..bf283df5 100644 --- a/check/check_suite_test.go +++ b/check/check_suite_test.go @@ -1,7 +1,6 @@ package check_test import ( - "io/ioutil" "os" "path/filepath" "testing" @@ -37,7 +36,7 @@ func TestCheck(t *testing.T) { func Fixture(filename string) string { path := filepath.Join("fixtures", filename) - contents, err := ioutil.ReadFile(path) + contents, err := os.ReadFile(path) if err != nil { panic(err) } diff --git a/check/command_test.go b/check/command_test.go index 2f90d8a7..1b45f8fc 100644 --- a/check/command_test.go +++ b/check/command_test.go @@ -1,7 +1,6 @@ package check_test import ( - "io/ioutil" "os" . "github.com/onsi/ginkgo" @@ -25,7 +24,7 @@ var _ = Describe("Check Command", func() { BeforeEach(func() { var err error - tmpPath, err = ioutil.TempDir("", "check_command") + tmpPath, err = os.MkdirTemp("", "check_command") Ω(err).ShouldNot(HaveOccurred()) request = Request{ diff --git a/in/command.go b/in/command.go index ba6d21c9..59e1319d 100644 --- a/in/command.go +++ b/in/command.go @@ -4,13 +4,12 @@ import ( "encoding/base64" "errors" "fmt" - "io/ioutil" "os" "path" "path/filepath" "strconv" - "github.com/concourse/s3-resource" + s3resource "github.com/concourse/s3-resource" "github.com/concourse/s3-resource/versions" ) @@ -179,11 +178,11 @@ func (command *Command) Run(destinationDir string, request Request) (Response, e } func (command *Command) writeURLFile(destDir string, url string) error { - return ioutil.WriteFile(filepath.Join(destDir, "url"), []byte(url), 0644) + return os.WriteFile(filepath.Join(destDir, "url"), []byte(url), 0644) } func (command *Command) writeVersionFile(versionNumber string, destDir string) error { - return ioutil.WriteFile(filepath.Join(destDir, "version"), []byte(versionNumber), 0644) + return os.WriteFile(filepath.Join(destDir, "version"), []byte(versionNumber), 0644) } func (command *Command) downloadFile(bucketName string, remotePath string, versionID string, destinationDir string, destinationFile string) error { @@ -209,7 +208,7 @@ func (command *Command) downloadTags(bucketName string, remotePath string, versi } func (command *Command) createInitialFile(destDir string, destFile string, data []byte) error { - return ioutil.WriteFile(filepath.Join(destDir, destFile), []byte(data), 0644) + return os.WriteFile(filepath.Join(destDir, destFile), []byte(data), 0644) } func (command *Command) metadata(remotePath string, private bool, url string) []s3resource.MetadataPair { @@ -241,7 +240,7 @@ func extractArchive(mime, filename string) error { } if mime == "application/gzip" || mime == "application/x-gzip" { - fileInfos, err := ioutil.ReadDir(destDir) + fileInfos, err := os.ReadDir(destDir) if err != nil { return fmt.Errorf("failed to read dir: %s", err) } diff --git a/in/command_test.go b/in/command_test.go index 13b66cc7..16f4891c 100644 --- a/in/command_test.go +++ b/in/command_test.go @@ -5,7 +5,6 @@ import ( "archive/zip" "compress/gzip" "io" - "io/ioutil" "log" "os" "path" @@ -15,7 +14,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/concourse/s3-resource" + s3resource "github.com/concourse/s3-resource" . "github.com/concourse/s3-resource/in" "github.com/concourse/s3-resource/fakes" @@ -34,7 +33,7 @@ var _ = Describe("In Command", func() { BeforeEach(func() { var err error - tmpPath, err = ioutil.TempDir("", "in_command") + tmpPath, err = os.MkdirTemp("", "in_command") Ω(err).ShouldNot(HaveOccurred()) destDir = filepath.Join(tmpPath, "destination") @@ -154,7 +153,7 @@ var _ = Describe("In Command", func() { Ω(err).ShouldNot(HaveOccurred()) Ω(urlPath).Should(ExistOnFilesystem()) - contents, err := ioutil.ReadFile(urlPath) + contents, err := os.ReadFile(urlPath) Ω(err).ShouldNot(HaveOccurred()) Ω(string(contents)).Should(Equal("http://google.com")) @@ -178,7 +177,7 @@ var _ = Describe("In Command", func() { Ω(err).ShouldNot(HaveOccurred()) Ω(urlPath).Should(ExistOnFilesystem()) - contents, err := ioutil.ReadFile(urlPath) + contents, err := os.ReadFile(urlPath) Ω(err).ShouldNot(HaveOccurred()) Ω(string(contents)).Should(Equal("http://google.com")) @@ -199,7 +198,7 @@ var _ = Describe("In Command", func() { Ω(err).ShouldNot(HaveOccurred()) Ω(versionFile).Should(ExistOnFilesystem()) - contents, err := ioutil.ReadFile(versionFile) + contents, err := os.ReadFile(versionFile) Ω(err).ShouldNot(HaveOccurred()) Ω(string(contents)).Should(Equal("1.3")) }) @@ -262,7 +261,7 @@ var _ = Describe("In Command", func() { s3client.DownloadFileStub = func(bucketName string, remotePath string, versionID string, localPath string) error { src := filepath.Join(tmpPath, "some-file") - err := ioutil.WriteFile(src, []byte("some-contents"), os.ModePerm) + err := os.WriteFile(src, []byte("some-contents"), os.ModePerm) Expect(err).NotTo(HaveOccurred()) err = createTarball([]string{src}, tmpPath, localPath) @@ -279,7 +278,7 @@ var _ = Describe("In Command", func() { _, err := command.Run(destDir, request) Expect(err).NotTo(HaveOccurred()) - bs, err := ioutil.ReadFile(filepath.Join(destDir, "some-file")) + bs, err := os.ReadFile(filepath.Join(destDir, "some-file")) Expect(err).NotTo(HaveOccurred()) Expect(bs).To(Equal([]byte("some-contents"))) @@ -289,10 +288,10 @@ var _ = Describe("In Command", func() { Context("when the file is a zip", func() { BeforeEach(func() { s3client.DownloadFileStub = func(bucketName string, remotePath string, versionID string, localPath string) error { - inDir, err := ioutil.TempDir(tmpPath, "zip-dir") + inDir, err := os.MkdirTemp(tmpPath, "zip-dir") Expect(err).NotTo(HaveOccurred()) - err = ioutil.WriteFile(path.Join(inDir, "some-file"), []byte("some-contents"), os.ModePerm) + err = os.WriteFile(path.Join(inDir, "some-file"), []byte("some-contents"), os.ModePerm) Expect(err).NotTo(HaveOccurred()) err = zipit(path.Join(inDir, "/"), localPath, "") @@ -306,7 +305,7 @@ var _ = Describe("In Command", func() { _, err := command.Run(destDir, request) Expect(err).NotTo(HaveOccurred()) - bs, err := ioutil.ReadFile(filepath.Join(destDir, "some-file")) + bs, err := os.ReadFile(filepath.Join(destDir, "some-file")) Expect(err).NotTo(HaveOccurred()) Expect(bs).To(Equal([]byte("some-contents"))) @@ -338,7 +337,7 @@ var _ = Describe("In Command", func() { _, err := command.Run(destDir, request) Expect(err).NotTo(HaveOccurred()) - bs, err := ioutil.ReadFile(filepath.Join(destDir, "a-file-1.3")) + bs, err := os.ReadFile(filepath.Join(destDir, "a-file-1.3")) Expect(err).NotTo(HaveOccurred()) Expect(string(bs)).To(Equal("some-contents")) @@ -356,12 +355,12 @@ var _ = Describe("In Command", func() { someFile1 := filepath.Join(tmpPath, "some-dir", "some-file") - err = ioutil.WriteFile(someFile1, []byte("some-contents"), os.ModePerm) + err = os.WriteFile(someFile1, []byte("some-contents"), os.ModePerm) Expect(err).NotTo(HaveOccurred()) someFile2 := filepath.Join(tmpPath, "some-file") - err = ioutil.WriteFile(someFile2, []byte("some-other-contents"), os.ModePerm) + err = os.WriteFile(someFile2, []byte("some-other-contents"), os.ModePerm) Expect(err).NotTo(HaveOccurred()) tarPath := filepath.Join(tmpPath, "some-tar") @@ -395,11 +394,11 @@ var _ = Describe("In Command", func() { Expect(filepath.Join(destDir, "some-dir", "some-file")).To(BeARegularFile()) - bs, err := ioutil.ReadFile(filepath.Join(destDir, "some-dir", "some-file")) + bs, err := os.ReadFile(filepath.Join(destDir, "some-dir", "some-file")) Expect(err).NotTo(HaveOccurred()) Expect(bs).To(Equal([]byte("some-contents"))) - bs, err = ioutil.ReadFile(filepath.Join(destDir, "some-file")) + bs, err = os.ReadFile(filepath.Join(destDir, "some-file")) Expect(err).NotTo(HaveOccurred()) Expect(bs).To(Equal([]byte("some-other-contents"))) }) @@ -408,7 +407,7 @@ var _ = Describe("In Command", func() { Context("when the file is not an archive", func() { BeforeEach(func() { s3client.DownloadFileStub = func(bucketName string, remotePath string, versionID string, localPath string) error { - err := ioutil.WriteFile(localPath, []byte("some-contents"), os.ModePerm) + err := os.WriteFile(localPath, []byte("some-contents"), os.ModePerm) Expect(err).NotTo(HaveOccurred()) return nil @@ -438,7 +437,7 @@ var _ = Describe("In Command", func() { contentFile := filepath.Join(destDir, initialFilename) Ω(contentFile).Should(BeARegularFile()) - contents, err := ioutil.ReadFile(contentFile) + contents, err := os.ReadFile(contentFile) Ω(err).ShouldNot(HaveOccurred()) Ω(string(contents)).Should(Equal(request.Source.InitialContentText)) }) @@ -454,7 +453,7 @@ var _ = Describe("In Command", func() { contentFile := filepath.Join(destDir, initialFilename) Ω(contentFile).Should(BeARegularFile()) - contents, err := ioutil.ReadFile(contentFile) + contents, err := os.ReadFile(contentFile) Ω(err).ShouldNot(HaveOccurred()) Ω(string(contents)).Should(Equal("the hard questions are hard 🙈")) }) @@ -496,7 +495,7 @@ var _ = Describe("In Command", func() { contentFile := filepath.Join(destDir, initialFilename) Ω(contentFile).Should(BeARegularFile()) - contents, err := ioutil.ReadFile(contentFile) + contents, err := os.ReadFile(contentFile) Ω(err).ShouldNot(HaveOccurred()) Ω(string(contents)).Should(Equal(request.Source.InitialContentText)) }) @@ -519,7 +518,7 @@ var _ = Describe("In Command", func() { contentFile := filepath.Join(destDir, filename) Ω(contentFile).Should(BeARegularFile()) - contents, err := ioutil.ReadFile(contentFile) + contents, err := os.ReadFile(contentFile) Ω(err).ShouldNot(HaveOccurred()) Ω(string(contents)).Should(Equal(request.Source.InitialContentText)) }) @@ -535,7 +534,7 @@ var _ = Describe("In Command", func() { contentFile := filepath.Join(destDir, filename) Ω(contentFile).Should(BeARegularFile()) - contents, err := ioutil.ReadFile(contentFile) + contents, err := os.ReadFile(contentFile) Ω(err).ShouldNot(HaveOccurred()) Ω(string(contents)).Should(Equal("the hard questions are hard 🙈")) }) @@ -577,7 +576,7 @@ var _ = Describe("In Command", func() { contentFile := filepath.Join(destDir, filename) Ω(contentFile).Should(BeARegularFile()) - contents, err := ioutil.ReadFile(contentFile) + contents, err := os.ReadFile(contentFile) Ω(err).ShouldNot(HaveOccurred()) Ω(string(contents)).Should(Equal(request.Source.InitialContentText)) }) diff --git a/integration/check_test.go b/integration/check_test.go index 621f32fb..931cc4fe 100644 --- a/integration/check_test.go +++ b/integration/check_test.go @@ -3,7 +3,6 @@ package integration_test import ( "bytes" "encoding/json" - "io/ioutil" "os" "os/exec" "path/filepath" @@ -11,7 +10,7 @@ import ( . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/concourse/s3-resource" + s3resource "github.com/concourse/s3-resource" "github.com/concourse/s3-resource/check" "github.com/onsi/gomega/gbytes" "github.com/onsi/gomega/gexec" @@ -101,7 +100,7 @@ var _ = Describe("check", func() { err := json.NewEncoder(stdin).Encode(checkRequest) Ω(err).ShouldNot(HaveOccurred()) - tempFile, err := ioutil.TempFile("", "file-to-upload") + tempFile, err := os.CreateTemp("", "file-to-upload") Ω(err).ShouldNot(HaveOccurred()) tempFile.Close() @@ -135,7 +134,7 @@ var _ = Describe("check", func() { err := json.NewEncoder(stdin).Encode(checkRequest) Ω(err).ShouldNot(HaveOccurred()) - tempFile, err := ioutil.TempFile("", "file-to-upload") + tempFile, err := os.CreateTemp("", "file-to-upload") Ω(err).ShouldNot(HaveOccurred()) tempFile.Close() @@ -199,7 +198,7 @@ var _ = Describe("check", func() { err := json.NewEncoder(stdin).Encode(checkRequest) Ω(err).ShouldNot(HaveOccurred()) - tempFile, err := ioutil.TempFile("", "file-to-upload") + tempFile, err := os.CreateTemp("", "file-to-upload") Ω(err).ShouldNot(HaveOccurred()) tempFile.Close() @@ -228,7 +227,7 @@ var _ = Describe("check", func() { err := json.NewEncoder(stdin).Encode(checkRequest) Ω(err).ShouldNot(HaveOccurred()) - tempFile, err := ioutil.TempFile("", "file-to-upload") + tempFile, err := os.CreateTemp("", "file-to-upload") Ω(err).ShouldNot(HaveOccurred()) tempFile.Close() @@ -268,7 +267,7 @@ var _ = Describe("check", func() { err := json.NewEncoder(stdin).Encode(checkRequest) Ω(err).ShouldNot(HaveOccurred()) - tempFile, err := ioutil.TempFile("", "file-to-upload") + tempFile, err := os.CreateTemp("", "file-to-upload") Ω(err).ShouldNot(HaveOccurred()) tempFile.Close() @@ -341,7 +340,7 @@ var _ = Describe("check", func() { err := json.NewEncoder(stdin).Encode(checkRequest) Ω(err).ShouldNot(HaveOccurred()) - tempFile, err := ioutil.TempFile("", "file-to-upload") + tempFile, err := os.CreateTemp("", "file-to-upload") Ω(err).ShouldNot(HaveOccurred()) tempFile.Close() @@ -376,7 +375,7 @@ var _ = Describe("check", func() { err := json.NewEncoder(stdin).Encode(checkRequest) Ω(err).ShouldNot(HaveOccurred()) - tempFile, err := ioutil.TempFile("", "file-to-upload") + tempFile, err := os.CreateTemp("", "file-to-upload") Ω(err).ShouldNot(HaveOccurred()) tempFile.Close() @@ -430,7 +429,7 @@ var _ = Describe("check", func() { err := json.NewEncoder(stdin).Encode(checkRequest) Ω(err).ShouldNot(HaveOccurred()) - tempFile, err := ioutil.TempFile("", "file-to-upload") + tempFile, err := os.CreateTemp("", "file-to-upload") Ω(err).ShouldNot(HaveOccurred()) tempFile.Close() @@ -500,7 +499,7 @@ var _ = Describe("check", func() { BeforeEach(func() { directoryPrefix = "files-in-bucket-that-do-not-match-with-version" - tempFile, err := ioutil.TempFile("", "file-to-upload") + tempFile, err := os.CreateTemp("", "file-to-upload") Ω(err).ShouldNot(HaveOccurred()) tempFile.Close() @@ -546,7 +545,7 @@ var _ = Describe("check", func() { BeforeEach(func() { directoryPrefix = "files-in-bucket-that-do-match-with-version" - tempFile, err := ioutil.TempFile("", "file-to-upload") + tempFile, err := os.CreateTemp("", "file-to-upload") Ω(err).ShouldNot(HaveOccurred()) tempFile.Close() @@ -609,7 +608,7 @@ var _ = Describe("check", func() { BeforeEach(func() { directoryPrefix = "files-in-bucket-with-latest-version-deleted" - tempFile, err := ioutil.TempFile("", "file-to-upload") + tempFile, err := os.CreateTemp("", "file-to-upload") Ω(err).ShouldNot(HaveOccurred()) tempFile.Close() diff --git a/integration/in_test.go b/integration/in_test.go index 3018a2d1..77a03908 100644 --- a/integration/in_test.go +++ b/integration/in_test.go @@ -4,12 +4,11 @@ import ( "bytes" "encoding/json" "fmt" - "io/ioutil" "os" "os/exec" "path/filepath" - "github.com/concourse/s3-resource" + s3resource "github.com/concourse/s3-resource" "github.com/concourse/s3-resource/in" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -30,7 +29,7 @@ var _ = Describe("in", func() { BeforeEach(func() { var err error - destDir, err = ioutil.TempDir("", "s3_in_integration_test") + destDir, err = os.MkdirTemp("", "s3_in_integration_test") Ω(err).ShouldNot(HaveOccurred()) stdin = &bytes.Buffer{} @@ -104,12 +103,12 @@ var _ = Describe("in", func() { }, } - tempFile, err := ioutil.TempFile("", "file-to-upload") + tempFile, err := os.CreateTemp("", "file-to-upload") Ω(err).ShouldNot(HaveOccurred()) tempFile.Close() for i := 1; i <= 3; i++ { - err = ioutil.WriteFile(tempFile.Name(), []byte(fmt.Sprintf("some-file-%d", i)), 0755) + err = os.WriteFile(tempFile.Name(), []byte(fmt.Sprintf("some-file-%d", i)), 0755) Ω(err).ShouldNot(HaveOccurred()) _, err = s3client.UploadFile(bucketName, filepath.Join(directoryPrefix, fmt.Sprintf("some-file-%d", i)), tempFile.Name(), s3resource.NewUploadFileOptions()) @@ -151,17 +150,17 @@ var _ = Describe("in", func() { })) Ω(filepath.Join(destDir, "some-file-2")).Should(BeARegularFile()) - contents, err := ioutil.ReadFile(filepath.Join(destDir, "some-file-2")) + contents, err := os.ReadFile(filepath.Join(destDir, "some-file-2")) Ω(err).ShouldNot(HaveOccurred()) Ω(contents).Should(Equal([]byte("some-file-2"))) Ω(filepath.Join(destDir, "version")).Should(BeARegularFile()) - versionContents, err := ioutil.ReadFile(filepath.Join(destDir, "version")) + versionContents, err := os.ReadFile(filepath.Join(destDir, "version")) Ω(err).ShouldNot(HaveOccurred()) Ω(versionContents).Should(Equal([]byte("2"))) Ω(filepath.Join(destDir, "url")).Should(BeARegularFile()) - urlContents, err := ioutil.ReadFile(filepath.Join(destDir, "url")) + urlContents, err := os.ReadFile(filepath.Join(destDir, "url")) Ω(err).ShouldNot(HaveOccurred()) Ω(urlContents).Should(Equal([]byte(buildEndpoint(bucketName, endpoint) + "/in-request-files/some-file-2"))) }) @@ -192,12 +191,12 @@ var _ = Describe("in", func() { })) Ω(filepath.Join(destDir, "some-file-0.0.0")).Should(BeARegularFile()) - contents, err := ioutil.ReadFile(filepath.Join(destDir, "some-file-0.0.0")) + contents, err := os.ReadFile(filepath.Join(destDir, "some-file-0.0.0")) Ω(err).ShouldNot(HaveOccurred()) Ω(contents).Should(Equal([]byte(inRequest.Source.InitialContentText))) Ω(filepath.Join(destDir, "version")).Should(BeARegularFile()) - versionContents, err := ioutil.ReadFile(filepath.Join(destDir, "version")) + versionContents, err := os.ReadFile(filepath.Join(destDir, "version")) Ω(err).ShouldNot(HaveOccurred()) Ω(versionContents).Should(Equal([]byte("0.0.0"))) @@ -226,12 +225,12 @@ var _ = Describe("in", func() { Version: s3resource.Version{}, } - tempFile, err := ioutil.TempFile("", "file-to-upload") + tempFile, err := os.CreateTemp("", "file-to-upload") Ω(err).ShouldNot(HaveOccurred()) tempFile.Close() for i := 1; i <= 3; i++ { - err = ioutil.WriteFile(tempFile.Name(), []byte(fmt.Sprintf("some-file-%d", i)), 0755) + err = os.WriteFile(tempFile.Name(), []byte(fmt.Sprintf("some-file-%d", i)), 0755) Ω(err).ShouldNot(HaveOccurred()) _, err = s3client.UploadFile(versionedBucketName, filepath.Join(directoryPrefix, "some-file"), tempFile.Name(), s3resource.NewUploadFileOptions()) @@ -279,17 +278,17 @@ var _ = Describe("in", func() { })) Ω(filepath.Join(destDir, "some-file")).Should(BeARegularFile()) - contents, err := ioutil.ReadFile(filepath.Join(destDir, "some-file")) + contents, err := os.ReadFile(filepath.Join(destDir, "some-file")) Ω(err).ShouldNot(HaveOccurred()) Ω(contents).Should(Equal([]byte("some-file-2"))) Ω(filepath.Join(destDir, "version")).Should(BeARegularFile()) - versionContents, err := ioutil.ReadFile(filepath.Join(destDir, "version")) + versionContents, err := os.ReadFile(filepath.Join(destDir, "version")) Ω(err).ShouldNot(HaveOccurred()) Ω(versionContents).Should(Equal([]byte(expectedVersion))) Ω(filepath.Join(destDir, "url")).Should(BeARegularFile()) - urlContents, err := ioutil.ReadFile(filepath.Join(destDir, "url")) + urlContents, err := os.ReadFile(filepath.Join(destDir, "url")) Ω(err).ShouldNot(HaveOccurred()) Ω(urlContents).Should(Equal([]byte(buildEndpoint(versionedBucketName, endpoint) + "/in-request-files-versioned/some-file?versionId=" + expectedVersion))) }) @@ -321,12 +320,12 @@ var _ = Describe("in", func() { })) Ω(filepath.Join(destDir, "some-file")).Should(BeARegularFile()) - contents, err := ioutil.ReadFile(filepath.Join(destDir, "some-file")) + contents, err := os.ReadFile(filepath.Join(destDir, "some-file")) Ω(err).ShouldNot(HaveOccurred()) Ω(contents).Should(Equal([]byte(inRequest.Source.InitialContentText))) Ω(filepath.Join(destDir, "version")).Should(BeARegularFile()) - versionContents, err := ioutil.ReadFile(filepath.Join(destDir, "version")) + versionContents, err := os.ReadFile(filepath.Join(destDir, "version")) Ω(err).ShouldNot(HaveOccurred()) Ω(versionContents).Should(Equal([]byte(expectedVersion))) @@ -363,12 +362,12 @@ var _ = Describe("in", func() { err := json.NewEncoder(stdin).Encode(inRequest) Ω(err).ShouldNot(HaveOccurred()) - tempFile, err := ioutil.TempFile("", "file-to-upload") + tempFile, err := os.CreateTemp("", "file-to-upload") Ω(err).ShouldNot(HaveOccurred()) tempFile.Close() for i := 1; i <= 3; i++ { - err = ioutil.WriteFile(tempFile.Name(), []byte(fmt.Sprintf("some-file-%d", i)), 0755) + err = os.WriteFile(tempFile.Name(), []byte(fmt.Sprintf("some-file-%d", i)), 0755) Ω(err).ShouldNot(HaveOccurred()) _, err = s3client.UploadFile(bucketName, filepath.Join(directoryPrefix, fmt.Sprintf("some-file-%d", i)), tempFile.Name(), s3resource.NewUploadFileOptions()) @@ -407,12 +406,12 @@ var _ = Describe("in", func() { })) Ω(filepath.Join(destDir, "some-file-2")).Should(BeARegularFile()) - contents, err := ioutil.ReadFile(filepath.Join(destDir, "some-file-2")) + contents, err := os.ReadFile(filepath.Join(destDir, "some-file-2")) Ω(err).ShouldNot(HaveOccurred()) Ω(contents).Should(Equal([]byte("some-file-2"))) Ω(filepath.Join(destDir, "url")).Should(BeARegularFile()) - urlContents, err := ioutil.ReadFile(filepath.Join(destDir, "url")) + urlContents, err := os.ReadFile(filepath.Join(destDir, "url")) Ω(err).ShouldNot(HaveOccurred()) Ω(urlContents).Should(Equal([]byte(inRequest.Source.CloudfrontURL + "/in-request-cloudfront-files/some-file-2"))) }) @@ -474,11 +473,11 @@ var _ = Describe("in", func() { err := json.NewEncoder(stdin).Encode(inRequest) Ω(err).ShouldNot(HaveOccurred()) - tempFile, err := ioutil.TempFile("", "file-to-upload") + tempFile, err := os.CreateTemp("", "file-to-upload") Ω(err).ShouldNot(HaveOccurred()) tempFile.Close() - err = ioutil.WriteFile(tempFile.Name(), []byte("some-file-1"), 0755) + err = os.WriteFile(tempFile.Name(), []byte("some-file-1"), 0755) Ω(err).ShouldNot(HaveOccurred()) _, err = s3client.UploadFile(bucketName, filepath.Join(directoryPrefix, "some-file-1"), tempFile.Name(), s3resource.NewUploadFileOptions()) @@ -499,7 +498,7 @@ var _ = Describe("in", func() { It("writes the tags to tags.json", func() { Ω(filepath.Join(destDir, "tags.json")).Should(BeARegularFile()) - actualTagsJSON, err := ioutil.ReadFile(filepath.Join(destDir, "tags.json")) + actualTagsJSON, err := os.ReadFile(filepath.Join(destDir, "tags.json")) Ω(err).ShouldNot(HaveOccurred()) expectedTagsJSON, err := json.Marshal(tags) diff --git a/integration/integration_suite_test.go b/integration/integration_suite_test.go index 125be024..dfc589b3 100644 --- a/integration/integration_suite_test.go +++ b/integration/integration_suite_test.go @@ -2,15 +2,16 @@ package integration_test import ( "encoding/json" - "github.com/aws/aws-sdk-go/aws/credentials/stscreds" - "io/ioutil" + "io" "os" + "github.com/aws/aws-sdk-go/aws/credentials/stscreds" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/sts" - "github.com/concourse/s3-resource" + s3resource "github.com/concourse/s3-resource" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/onsi/gomega/gexec" @@ -31,6 +32,7 @@ var bucketName = os.Getenv("S3_TESTING_BUCKET") var regionName = os.Getenv("S3_TESTING_REGION") var endpoint = os.Getenv("S3_ENDPOINT") var v2signing = os.Getenv("S3_V2_SIGNING") +var assumeRoleArn = "" var awsConfig *aws.Config var s3client s3resource.S3Client var s3Service *s3.S3 @@ -64,7 +66,9 @@ func getSessionTokenS3Client(awsConfig *aws.Config) (*s3.S3, s3resource.S3Client HTTPClient: awsConfig.HTTPClient, } - svc := sts.New(session.New(stsAwsConfig), stsAwsConfig) + stsSess, err := session.NewSession(stsAwsConfig) + Ω(err).ShouldNot(HaveOccurred()) + svc := sts.New(stsSess, stsAwsConfig) duration := int64(900) params := &sts.GetSessionTokenInput{ @@ -78,13 +82,18 @@ func getSessionTokenS3Client(awsConfig *aws.Config) (*s3.S3, s3resource.S3Client *resp.Credentials.AccessKeyId, *resp.Credentials.SecretAccessKey, *resp.Credentials.SessionToken, + assumeRoleArn, regionName, endpoint, false, false, ) - s3Service := s3.New(session.New(newAwsConfig), newAwsConfig) - s3client := s3resource.NewS3Client(ioutil.Discard, newAwsConfig, v2signing == "true", awsRoleARN) + + sess, err := session.NewSession(awsConfig) + Ω(err).ShouldNot(HaveOccurred()) + s3Service = s3.New(sess, awsConfig, newAwsConfig) + s3client, err = s3resource.NewS3Client(io.Discard, awsConfig, v2signing == "true", awsRoleARN) + Ω(err).ShouldNot(HaveOccurred()) return s3Service, s3client } @@ -124,6 +133,7 @@ var _ = SynchronizedBeforeSuite(func() []byte { accessKeyID, secretAccessKey, sessionToken, + assumeRoleArn, regionName, endpoint, false, @@ -140,8 +150,11 @@ var _ = SynchronizedBeforeSuite(func() []byte { additionalAwsConfig.Credentials = roleCredentials } - s3Service = s3.New(session.New(awsConfig), awsConfig, &additionalAwsConfig) - s3client = s3resource.NewS3Client(ioutil.Discard, awsConfig, v2signing == "true", awsRoleARN) + sess, err := session.NewSession(awsConfig) + Ω(err).ShouldNot(HaveOccurred()) + s3Service = s3.New(sess, awsConfig, &additionalAwsConfig) + s3client, err = s3resource.NewS3Client(io.Discard, awsConfig, v2signing == "true", awsRoleARN) + Ω(err).ShouldNot(HaveOccurred()) } }) diff --git a/integration/out_test.go b/integration/out_test.go index 36379d95..a83deb1c 100644 --- a/integration/out_test.go +++ b/integration/out_test.go @@ -3,7 +3,6 @@ package integration_test import ( "bytes" "encoding/json" - "io/ioutil" "os" "os/exec" "path/filepath" @@ -33,7 +32,7 @@ var _ = Describe("out", func() { BeforeEach(func() { var err error - sourceDir, err = ioutil.TempDir("", "s3_out_integration_test") + sourceDir, err = os.MkdirTemp("", "s3_out_integration_test") Ω(err).ShouldNot(HaveOccurred()) stdin = &bytes.Buffer{} @@ -88,7 +87,7 @@ var _ = Describe("out", func() { Context("with a content-type", func() { BeforeEach(func() { - ioutil.WriteFile(filepath.Join(sourceDir, "content-typed-file"), []byte("text only"), 0755) + os.WriteFile(filepath.Join(sourceDir, "content-typed-file"), []byte("text only"), 0755) outRequest := out.Request{ Source: s3resource.Source{ @@ -132,7 +131,7 @@ var _ = Describe("out", func() { Context("without a content-type", func() { BeforeEach(func() { - ioutil.WriteFile(filepath.Join(sourceDir, "uncontent-typed-file"), []byte("text only"), 0755) + os.WriteFile(filepath.Join(sourceDir, "uncontent-typed-file"), []byte("text only"), 0755) outRequest := out.Request{ Source: s3resource.Source{ @@ -220,7 +219,7 @@ var _ = Describe("out", func() { Context("with a file glob and public read ACL specified", func() { BeforeEach(func() { - err := ioutil.WriteFile(filepath.Join(sourceDir, "glob-file-to-upload"), []byte("contents"), 0755) + err := os.WriteFile(filepath.Join(sourceDir, "glob-file-to-upload"), []byte("contents"), 0755) Ω(err).ShouldNot(HaveOccurred()) outRequest := out.Request{ @@ -348,7 +347,7 @@ var _ = Describe("out", func() { Context("with regexp", func() { BeforeEach(func() { - err := ioutil.WriteFile(filepath.Join(sourceDir, "file-to-upload"), []byte("contents"), 0755) + err := os.WriteFile(filepath.Join(sourceDir, "file-to-upload"), []byte("contents"), 0755) Ω(err).ShouldNot(HaveOccurred()) outRequest := out.Request{ @@ -403,7 +402,7 @@ var _ = Describe("out", func() { Context("with versioned_file", func() { BeforeEach(func() { - err := ioutil.WriteFile(filepath.Join(sourceDir, "file-to-upload-local"), []byte("contents"), 0755) + err := os.WriteFile(filepath.Join(sourceDir, "file-to-upload-local"), []byte("contents"), 0755) Ω(err).ShouldNot(HaveOccurred()) outRequest := out.Request{ @@ -454,7 +453,7 @@ var _ = Describe("out", func() { Context("with versioned_file", func() { BeforeEach(func() { - err := ioutil.WriteFile(filepath.Join(sourceDir, "file-to-upload-local"), []byte("contents"), 0755) + err := os.WriteFile(filepath.Join(sourceDir, "file-to-upload-local"), []byte("contents"), 0755) Ω(err).ShouldNot(HaveOccurred()) outRequest := out.Request{ @@ -513,7 +512,7 @@ var _ = Describe("out", func() { Context("with regexp", func() { BeforeEach(func() { - err := ioutil.WriteFile(filepath.Join(sourceDir, "file-to-upload"), []byte("contents"), 0755) + err := os.WriteFile(filepath.Join(sourceDir, "file-to-upload"), []byte("contents"), 0755) Ω(err).ShouldNot(HaveOccurred()) outRequest := out.Request{ diff --git a/integration/s3client_test.go b/integration/s3client_test.go index bfd9c69d..9d156e9b 100644 --- a/integration/s3client_test.go +++ b/integration/s3client_test.go @@ -3,14 +3,13 @@ package integration_test import ( "encoding/json" "fmt" - "io/ioutil" "os" "path/filepath" "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/s3" - "github.com/concourse/s3-resource" + s3resource "github.com/concourse/s3-resource" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) @@ -28,10 +27,10 @@ var _ = Describe("S3client", func() { directoryPrefix = "s3client-tests" runtime = fmt.Sprintf("%d", time.Now().Unix()) - tempDir, err = ioutil.TempDir("", "s3-upload-dir") + tempDir, err = os.MkdirTemp("", "s3-upload-dir") Ω(err).ShouldNot(HaveOccurred()) - tempFile, err = ioutil.TempFile(tempDir, "file-to-upload") + tempFile, err = os.CreateTemp(tempDir, "file-to-upload") Ω(err).ShouldNot(HaveOccurred()) tempFile.Write([]byte("hello-" + runtime)) @@ -100,7 +99,7 @@ var _ = Describe("S3client", func() { err = s3client.DownloadFile(versionedBucketName, filepath.Join(directoryPrefix, "file-to-upload-1"), "", filepath.Join(tempDir, "downloaded-file")) Ω(err).ShouldNot(HaveOccurred()) - read, err := ioutil.ReadFile(filepath.Join(tempDir, "downloaded-file")) + read, err := os.ReadFile(filepath.Join(tempDir, "downloaded-file")) Ω(err).ShouldNot(HaveOccurred()) Ω(read).Should(Equal([]byte("hello-" + runtime))) @@ -110,7 +109,7 @@ var _ = Describe("S3client", func() { expectedTagsJSON, err := json.Marshal(tags) Ω(err).ShouldNot(HaveOccurred()) - actualTagsJSON, err := ioutil.ReadFile(filepath.Join(tempDir, "tags.json")) + actualTagsJSON, err := os.ReadFile(filepath.Join(tempDir, "tags.json")) Ω(err).ShouldNot(HaveOccurred()) Ω(actualTagsJSON).Should(MatchJSON(expectedTagsJSON)) diff --git a/out/command_test.go b/out/command_test.go index 2652d5e6..d46e1131 100644 --- a/out/command_test.go +++ b/out/command_test.go @@ -1,11 +1,10 @@ package out_test import ( - "io/ioutil" "os" "path/filepath" - "github.com/concourse/s3-resource" + s3resource "github.com/concourse/s3-resource" "github.com/concourse/s3-resource/fakes" "github.com/concourse/s3-resource/out" "github.com/onsi/gomega/gbytes" @@ -28,7 +27,7 @@ var _ = Describe("Out Command", func() { BeforeEach(func() { var err error - tmpPath, err = ioutil.TempDir("", "out_command") + tmpPath, err = os.MkdirTemp("", "out_command") Ω(err).ShouldNot(HaveOccurred()) sourceDir = filepath.Join(tmpPath, "source") diff --git a/out/out_suite_test.go b/out/out_suite_test.go index 2c6c65f6..bea54abe 100644 --- a/out/out_suite_test.go +++ b/out/out_suite_test.go @@ -1,7 +1,6 @@ package out_test import ( - "io/ioutil" "os" "path/filepath" "testing" @@ -37,7 +36,7 @@ func TestOut(t *testing.T) { func Fixture(filename string) string { path := filepath.Join("fixtures", filename) - contents, err := ioutil.ReadFile(path) + contents, err := os.ReadFile(path) if err != nil { panic(err) }