From ba74b80d33c47055c5dff6f741ef7846092ceb1b Mon Sep 17 00:00:00 2001 From: chenxin0723 <870720136@qq.com> Date: Thu, 21 Nov 2024 15:07:37 +0800 Subject: [PATCH 1/5] chore: upgrade aws-sdk-go to aws-sdk-go-v2 --- go.mod | 22 ++++++- go.sum | 71 ++++++++++---------- s3/s3.go | 177 +++++++++++++++++++++++++++----------------------- s3/s3_test.go | 8 +-- 4 files changed, 156 insertions(+), 122 deletions(-) diff --git a/go.mod b/go.mod index 59f495a..1f52d0c 100644 --- a/go.mod +++ b/go.mod @@ -1,18 +1,34 @@ module github.com/qor/oss -go 1.20 +go 1.21 require ( github.com/aliyun/aliyun-oss-go-sdk v2.2.7+incompatible - github.com/aws/aws-sdk-go v1.44.300 + github.com/aws/aws-sdk-go-v2 v1.32.4 + github.com/aws/aws-sdk-go-v2/config v1.28.3 + github.com/aws/aws-sdk-go-v2/credentials v1.17.44 + github.com/aws/aws-sdk-go-v2/service/s3 v1.67.0 + github.com/aws/aws-sdk-go-v2/service/sts v1.32.4 github.com/jinzhu/configor v1.2.1 github.com/qiniu/api.v7/v7 v7.8.2 ) require ( github.com/BurntSushi/toml v0.3.1 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.19 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.23 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.24.5 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.4 // indirect + github.com/aws/smithy-go v1.22.0 // indirect github.com/gookit/color v1.3.6 // indirect - github.com/jmespath/go-jmespath v0.4.0 // indirect golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect golang.org/x/time v0.3.0 // indirect gopkg.in/yaml.v2 v2.2.8 // indirect diff --git a/go.sum b/go.sum index 08c3a8c..bafb48c 100644 --- a/go.sum +++ b/go.sum @@ -2,19 +2,48 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/aliyun/aliyun-oss-go-sdk v2.2.7+incompatible h1:KpbJFXwhVeuxNtBJ74MCGbIoaBok2uZvkD7QXp2+Wis= github.com/aliyun/aliyun-oss-go-sdk v2.2.7+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= -github.com/aws/aws-sdk-go v1.44.300 h1:Zn+3lqgYahIf9yfrwZ+g+hq/c3KzUBaQ8wqY/ZXiAbY= -github.com/aws/aws-sdk-go v1.44.300/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go-v2 v1.32.4 h1:S13INUiTxgrPueTmrm5DZ+MiAo99zYzHEFh1UNkOxNE= +github.com/aws/aws-sdk-go-v2 v1.32.4/go.mod h1:2SK5n0a2karNTv5tbP1SjsX0uhttou00v/HpXKM1ZUo= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6 h1:pT3hpW0cOHRJx8Y0DfJUEQuqPild8jRGmSFmBgvydr0= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.6/go.mod h1:j/I2++U0xX+cr44QjHay4Cvxj6FUbnxrgmqN3H1jTZA= +github.com/aws/aws-sdk-go-v2/config v1.28.3 h1:kL5uAptPcPKaJ4q0sDUjUIdueO18Q7JDzl64GpVwdOM= +github.com/aws/aws-sdk-go-v2/config v1.28.3/go.mod h1:SPEn1KA8YbgQnwiJ/OISU4fz7+F6Fe309Jf0QTsRCl4= +github.com/aws/aws-sdk-go-v2/credentials v1.17.44 h1:qqfs5kulLUHUEXlHEZXLJkgGoF3kkUeFUTVA585cFpU= +github.com/aws/aws-sdk-go-v2/credentials v1.17.44/go.mod h1:0Lm2YJ8etJdEdw23s+q/9wTpOeo2HhNE97XcRa7T8MA= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.19 h1:woXadbf0c7enQ2UGCi8gW/WuKmE0xIzxBF/eD94jMKQ= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.19/go.mod h1:zminj5ucw7w0r65bP6nhyOd3xL6veAUMc3ElGMoLVb4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.23 h1:A2w6m6Tmr+BNXjDsr7M90zkWjsu4JXHwrzPg235STs4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.23/go.mod h1:35EVp9wyeANdujZruvHiQUAo9E3vbhnIO1mTCAxMlY0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.23 h1:pgYW9FCabt2M25MoHYCfMrVY2ghiiBKYWUVXfwZs+sU= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.23/go.mod h1:c48kLgzO19wAu3CPkDWC28JbaJ+hfQlsdl7I2+oqIbk= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.23 h1:1SZBDiRzzs3sNhOMVApyWPduWYGAX0imGy06XiBnCAM= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.23/go.mod h1:i9TkxgbZmHVh2S0La6CAXtnyFhlCX/pJ0JsOvBAS6Mk= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0 h1:TToQNkvGguu209puTojY/ozlqy2d/SFNcoLIqTFi42g= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.0/go.mod h1:0jp+ltwkf+SwG2fm/PKo8t4y8pJSgOCO4D8Lz3k0aHQ= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.4 h1:aaPpoG15S2qHkWm4KlEyF01zovK1nW4BBbyXuHNSE90= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.4/go.mod h1:eD9gS2EARTKgGr/W5xwgY/ik9z/zqpW+m/xOQbVxrMk= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.4 h1:tHxQi/XHPK0ctd/wdOw0t7Xrc2OxcRCnVzv8lwWPu0c= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.4/go.mod h1:4GQbF1vJzG60poZqWatZlhP31y8PGCCVTvIGPdaaYJ0= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.4 h1:E5ZAVOmI2apR8ADb72Q63KqwwwdW1XcMeXIlrZ1Psjg= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.4/go.mod h1:wezzqVUOVVdk+2Z/JzQT4NxAU0NbhRe5W8pIE72jsWI= +github.com/aws/aws-sdk-go-v2/service/s3 v1.67.0 h1:SwaJ0w0MOp0pBTIKTamLVeTKD+iOWyNJRdJ2KCQRg6Q= +github.com/aws/aws-sdk-go-v2/service/s3 v1.67.0/go.mod h1:TMhLIyRIyoGVlaEMAt+ITMbwskSTpcGsCPDq91/ihY0= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.5 h1:HJwZwRt2Z2Tdec+m+fPjvdmkq2s9Ra+VR0hjF7V2o40= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.5/go.mod h1:wrMCEwjFPms+V86TCQQeOxQF/If4vT44FGIOFiMC2ck= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.4 h1:zcx9LiGWZ6i6pjdcoE9oXAB6mUdeyC36Ia/QEiIvYdg= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.4/go.mod h1:Tp/ly1cTjRLGBBmNccFumbZ8oqpZlpdhFf80SrRh4is= +github.com/aws/aws-sdk-go-v2/service/sts v1.32.4 h1:yDxvkz3/uOKfxnv8YhzOi9m+2OGIxF+on3KOISbK5IU= +github.com/aws/aws-sdk-go-v2/service/sts v1.32.4/go.mod h1:9XEUty5v5UAsMiFOBJrNibZgwCeOma73jgGwwhgffa8= +github.com/aws/smithy-go v1.22.0 h1:uunKnWlcoL3zO7q+gG2Pk53joueEOsnNB28QdMsmiMM= +github.com/aws/smithy-go v1.22.0/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/gookit/color v1.3.6 h1:Rgbazd4JO5AgSTVGS3o0nvaSdwdrS8bzvIXwtK6OiMk= github.com/gookit/color v1.3.6/go.mod h1:R3ogXq2B9rTbXoSHJ1HyUVAZ3poOJHpd9nQmyGZsfvQ= github.com/jinzhu/configor v1.2.1 h1:OKk9dsR8i6HPOCZR8BcMtcEImAFjIhbJFZNyn5GCZko= github.com/jinzhu/configor v1.2.1/go.mod h1:nX89/MOmDba7ZX7GCyU/VIaQ2Ar2aizBl2d3JLF/rDc= -github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= -github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= -github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/qiniu/api.v7/v7 v7.8.2 h1:f08kI0MmsJNzK4sUS8bG3HDH67ktwd/ji23Gkiy2ra4= @@ -22,39 +51,11 @@ github.com/qiniu/api.v7/v7 v7.8.2/go.mod h1:FPsIqxh1Ym3X01sANE5ZwXfLZSWoCUp5+jNI github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0= -golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/s3/s3.go b/s3/s3.go index 136f716..e4d0edb 100644 --- a/s3/s3.go +++ b/s3/s3.go @@ -2,9 +2,9 @@ package s3 import ( "bytes" + "context" "fmt" "io" - "io/ioutil" "mime" "net/http" "net/url" @@ -15,19 +15,20 @@ import ( "strings" "time" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds" - "github.com/aws/aws-sdk-go/aws/credentials/stscreds" - "github.com/aws/aws-sdk-go/aws/ec2metadata" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/credentials/ec2rolecreds" + "github.com/aws/aws-sdk-go-v2/credentials/stscreds" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/aws-sdk-go-v2/service/sts" "github.com/qor/oss" ) // Client S3 storage type Client struct { - *s3.S3 + S3 *s3.Client Config *Config } @@ -38,80 +39,89 @@ type Config struct { Region string Bucket string SessionToken string - ACL string + ACL types.ObjectCannedACL Endpoint string S3Endpoint string S3ForcePathStyle bool CacheControl string - Session *session.Session - - RoleARN string + AwsConfig *aws.Config + RoleARN string + EnableEC2IAMRole bool } -func ec2RoleAwsCreds(config *Config) *credentials.Credentials { - ec2m := ec2metadata.New(session.New(), &aws.Config{ - HTTPClient: &http.Client{Timeout: 10 * time.Second}, - Endpoint: aws.String("http://169.254.169.254/latest"), - }) +// New initialize S3 storage +func New(cfg *Config) *Client { + if cfg.ACL == "" { + cfg.ACL = types.ObjectCannedACLPublicRead // default ACL + } - return credentials.NewCredentials(&ec2rolecreds.EC2RoleProvider{ - Client: ec2m, - }) -} + client := &Client{Config: cfg} -// Does this need to be publicly exported? -func EC2RoleAwsConfig(config *Config) *aws.Config { - return &aws.Config{ - Region: aws.String(config.Region), - Credentials: ec2RoleAwsCreds(config), - } -} + // use role ARN to fetch credentials + if cfg.RoleARN != "" { + awsCfg, err := config.LoadDefaultConfig(context.TODO()) + if err != nil { + panic(err) + } -// New initialize S3 storage -func New(config *Config) *Client { - if config.ACL == "" { - config.ACL = s3.BucketCannedACLPublicRead - } + provider := stscreds.NewAssumeRoleProvider(sts.NewFromConfig(awsCfg), cfg.RoleARN) + creds := aws.NewCredentialsCache(provider) - client := &Client{Config: config} + s3Client := s3.NewFromConfig(awsCfg, func(o *s3.Options) { + o.Region = cfg.Region + o.BaseEndpoint = aws.String(cfg.S3Endpoint) + o.UsePathStyle = cfg.S3ForcePathStyle - if config.RoleARN != "" { - sess := session.Must(session.NewSession()) - creds := stscreds.NewCredentials(sess, config.RoleARN) + o.Credentials = creds + }) - s3Config := &aws.Config{ - Region: &config.Region, - Endpoint: &config.S3Endpoint, - S3ForcePathStyle: &config.S3ForcePathStyle, - Credentials: creds, - } + client.S3 = s3Client + return client + } + + // use alreay configured aws config + if cfg.AwsConfig != nil { + s3Client := s3.NewFromConfig(*cfg.AwsConfig, func(o *s3.Options) { + o.Region = cfg.Region + o.BaseEndpoint = aws.String(cfg.S3Endpoint) + o.UsePathStyle = cfg.S3ForcePathStyle + }) - client.S3 = s3.New(sess, s3Config) + client.S3 = s3Client return client } - s3Config := &aws.Config{ - Region: &config.Region, - Endpoint: &config.S3Endpoint, - S3ForcePathStyle: &config.S3ForcePathStyle, + cfgOptions := []func(*config.LoadOptions) error{ + config.WithRegion(cfg.Region), } - if config.Session != nil { - client.S3 = s3.New(config.Session, s3Config) - } else if config.AccessID == "" && config.AccessKey == "" { - // use aws default Credentials - // s3Config.Credentials = ec2RoleAwsCreds(config) - sess := session.Must(session.NewSession()) - client.S3 = s3.New(sess, s3Config) - } else { - creds := credentials.NewStaticCredentials(config.AccessID, config.AccessKey, config.SessionToken) - if _, err := creds.Get(); err == nil { - s3Config.Credentials = creds - client.S3 = s3.New(session.New(), s3Config) - } + // use EC2 IAM role + if cfg.EnableEC2IAMRole { + cfgOptions = append(cfgOptions, config.WithCredentialsProvider( + ec2rolecreds.New(), + )) } + // use static credentials + if cfg.AccessID != "" && cfg.AccessKey != "" { + cfgOptions = append(cfgOptions, config.WithCredentialsProvider( + credentials.NewStaticCredentialsProvider(cfg.AccessID, cfg.AccessKey, cfg.SessionToken), + )) + } + + awsConfig, err := config.LoadDefaultConfig(context.TODO(), cfgOptions...) + if err != nil { + panic(err) + } + + s3Client := s3.NewFromConfig(awsConfig, func(o *s3.Options) { + o.Region = cfg.Region + o.BaseEndpoint = aws.String(cfg.S3Endpoint) + o.UsePathStyle = cfg.S3ForcePathStyle + }) + + client.S3 = s3Client return client } @@ -123,7 +133,7 @@ func (client Client) Get(path string) (file *os.File, err error) { pattern := fmt.Sprintf("s3*%s", ext) if err == nil { - if file, err = ioutil.TempFile("/tmp", pattern); err == nil { + if file, err = os.CreateTemp("/tmp", pattern); err == nil { defer readCloser.Close() _, err = io.Copy(file, readCloser) file.Seek(0, 0) @@ -135,7 +145,7 @@ func (client Client) Get(path string) (file *os.File, err error) { // GetStream get file as stream func (client Client) GetStream(path string) (io.ReadCloser, error) { - getResponse, err := client.S3.GetObject(&s3.GetObjectInput{ + getResponse, err := client.S3.GetObject(context.TODO(), &s3.GetObjectInput{ Bucket: aws.String(client.Config.Bucket), Key: aws.String(client.ToRelativePath(path)), }) @@ -150,7 +160,7 @@ func (client Client) Put(urlPath string, reader io.Reader) (*oss.Object, error) } urlPath = client.ToRelativePath(urlPath) - buffer, err := ioutil.ReadAll(reader) + buffer, err := io.ReadAll(reader) fileType := mime.TypeByExtension(path.Ext(urlPath)) if fileType == "" { @@ -160,7 +170,7 @@ func (client Client) Put(urlPath string, reader io.Reader) (*oss.Object, error) params := &s3.PutObjectInput{ Bucket: aws.String(client.Config.Bucket), // required Key: aws.String(urlPath), // required - ACL: aws.String(client.Config.ACL), + ACL: client.Config.ACL, Body: bytes.NewReader(buffer), ContentLength: aws.Int64(int64(len(buffer))), ContentType: aws.String(fileType), @@ -169,7 +179,7 @@ func (client Client) Put(urlPath string, reader io.Reader) (*oss.Object, error) params.CacheControl = aws.String(client.Config.CacheControl) } - _, err = client.S3.PutObject(params) + _, err = client.S3.PutObject(context.Background(), params) now := time.Now() return &oss.Object{ @@ -182,7 +192,7 @@ func (client Client) Put(urlPath string, reader io.Reader) (*oss.Object, error) // Delete delete file func (client Client) Delete(path string) error { - _, err := client.S3.DeleteObject(&s3.DeleteObjectInput{ + _, err := client.S3.DeleteObject(context.Background(), &s3.DeleteObjectInput{ Bucket: aws.String(client.Config.Bucket), Key: aws.String(client.ToRelativePath(path)), }) @@ -191,20 +201,20 @@ func (client Client) Delete(path string) error { // DeleteObjects delete files in bulk func (client Client) DeleteObjects(paths []string) (err error) { - var objs []*s3.ObjectIdentifier + var objs []types.ObjectIdentifier for _, v := range paths { - var obj s3.ObjectIdentifier + var obj types.ObjectIdentifier obj.Key = aws.String(strings.TrimPrefix(client.ToRelativePath(v), "/")) - objs = append(objs, &obj) + objs = append(objs, obj) } input := &s3.DeleteObjectsInput{ Bucket: aws.String(client.Config.Bucket), - Delete: &s3.Delete{ + Delete: &types.Delete{ Objects: objs, }, } - _, err = client.S3.DeleteObjects(input) + _, err = client.S3.DeleteObjects(context.Background(), input) if err != nil { return } @@ -220,7 +230,7 @@ func (client Client) List(path string) ([]*oss.Object, error) { prefix = strings.Trim(path, "/") + "/" } - listObjectsResponse, err := client.S3.ListObjectsV2(&s3.ListObjectsV2Input{ + listObjectsResponse, err := client.S3.ListObjectsV2(context.Background(), &s3.ListObjectsV2Input{ Bucket: aws.String(client.Config.Bucket), Prefix: aws.String(prefix), }) @@ -245,7 +255,7 @@ func (client Client) GetEndpoint() string { return client.Config.Endpoint } - endpoint := client.S3.Endpoint + endpoint := *client.S3.Options().BaseEndpoint for _, prefix := range []string{"https://", "http://"} { endpoint = strings.TrimPrefix(endpoint, prefix) } @@ -274,14 +284,21 @@ func (client Client) ToRelativePath(urlPath string) string { // GetURL get public accessible URL func (client Client) GetURL(path string) (url string, err error) { - if client.Endpoint == "" { - if client.Config.ACL == s3.BucketCannedACLPrivate || client.Config.ACL == s3.BucketCannedACLAuthenticatedRead { - getResponse, _ := client.S3.GetObjectRequest(&s3.GetObjectInput{ + if client.Config.Endpoint == "" { + + if client.Config.ACL == types.ObjectCannedACLPrivate || client.Config.ACL == types.ObjectCannedACLAuthenticatedRead { + + presignClient := s3.NewPresignClient(client.S3) + presignedGetURL, err := presignClient.PresignGetObject(context.TODO(), &s3.GetObjectInput{ Bucket: aws.String(client.Config.Bucket), Key: aws.String(client.ToRelativePath(path)), + }, func(opts *s3.PresignOptions) { + opts.Expires = 1 * time.Hour }) - return getResponse.Presign(1 * time.Hour) + if err == nil && presignedGetURL != nil { + return presignedGetURL.URL, nil + } } } @@ -290,7 +307,7 @@ func (client Client) GetURL(path string) (url string, err error) { // Copy copy s3 file from "from" to "to" func (client Client) Copy(from, to string) (err error) { - _, err = client.S3.CopyObject(&s3.CopyObjectInput{ + _, err = client.S3.CopyObject(context.Background(), &s3.CopyObjectInput{ Bucket: aws.String(client.Config.Bucket), CopySource: aws.String(from), Key: aws.String(to), diff --git a/s3/s3_test.go b/s3/s3_test.go index 4518452..6827ed2 100644 --- a/s3/s3_test.go +++ b/s3/s3_test.go @@ -4,7 +4,7 @@ import ( "fmt" "testing" - awss3 "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/jinzhu/configor" "github.com/qor/oss/s3" "github.com/qor/oss/tests" @@ -34,11 +34,11 @@ func TestAll(t *testing.T) { tests.TestAll(client, t) fmt.Println("testing S3 with private ACL") - privateClient := s3.New(&s3.Config{AccessID: config.AccessID, AccessKey: config.AccessKey, Region: config.Region, Bucket: config.Bucket, ACL: awss3.BucketCannedACLPrivate, Endpoint: config.Endpoint}) + privateClient := s3.New(&s3.Config{AccessID: config.AccessID, AccessKey: config.AccessKey, Region: config.Region, Bucket: config.Bucket, ACL: types.ObjectCannedACLPrivate, Endpoint: config.Endpoint}) tests.TestAll(privateClient, t) fmt.Println("testing S3 with AuthenticatedRead ACL") - authenticatedReadClient := s3.New(&s3.Config{AccessID: config.AccessID, AccessKey: config.AccessKey, Region: config.Region, Bucket: config.Bucket, ACL: awss3.BucketCannedACLAuthenticatedRead, Endpoint: config.Endpoint}) + authenticatedReadClient := s3.New(&s3.Config{AccessID: config.AccessID, AccessKey: config.AccessKey, Region: config.Region, Bucket: config.Bucket, ACL: types.ObjectCannedACLAuthenticatedRead, Endpoint: config.Endpoint}) tests.TestAll(authenticatedReadClient, t) } @@ -48,7 +48,7 @@ func TestToRelativePath(t *testing.T) { "https://qor-example.com/myobject.ext": "/myobject.ext", "//mybucket.s3.amazonaws.com/myobject.ext": "/myobject.ext", "http://mybucket.s3.amazonaws.com/myobject.ext": "/myobject.ext", - "myobject.ext": "/myobject.ext", + "myobject.ext": "/myobject.ext", } for url, path := range urlMap { From 883ebc124da55a82c2397e745edced4f73b2dea7 Mon Sep 17 00:00:00 2001 From: chenxin0723 <870720136@qq.com> Date: Fri, 22 Nov 2024 14:56:42 +0800 Subject: [PATCH 2/5] chore: ensure the s3 object key don't have a prefix / --- s3/s3.go | 105 ++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 73 insertions(+), 32 deletions(-) diff --git a/s3/s3.go b/s3/s3.go index e4d0edb..b3665f0 100644 --- a/s3/s3.go +++ b/s3/s3.go @@ -23,6 +23,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/aws/aws-sdk-go-v2/service/sts" + smithyendpoints "github.com/aws/smithy-go/endpoints" "github.com/qor/oss" ) @@ -58,6 +59,20 @@ func New(cfg *Config) *Client { client := &Client{Config: cfg} + s3CfgOptions := []func(o *s3.Options){ + func(o *s3.Options) { + o.Region = cfg.Region + o.UsePathStyle = cfg.S3ForcePathStyle + }, + } + + if cfg.S3Endpoint != "" { + s3CfgOptions = append(s3CfgOptions, s3.WithEndpointResolverV2(&endpointResolverV2{ + Url: cfg.S3Endpoint, + })) + + } + // use role ARN to fetch credentials if cfg.RoleARN != "" { awsCfg, err := config.LoadDefaultConfig(context.TODO()) @@ -68,59 +83,46 @@ func New(cfg *Config) *Client { provider := stscreds.NewAssumeRoleProvider(sts.NewFromConfig(awsCfg), cfg.RoleARN) creds := aws.NewCredentialsCache(provider) - s3Client := s3.NewFromConfig(awsCfg, func(o *s3.Options) { - o.Region = cfg.Region - o.BaseEndpoint = aws.String(cfg.S3Endpoint) - o.UsePathStyle = cfg.S3ForcePathStyle - + s3CfgOptions = append(s3CfgOptions, func(o *s3.Options) { o.Credentials = creds }) + s3Client := s3.NewFromConfig(awsCfg, s3CfgOptions...) client.S3 = s3Client return client } // use alreay configured aws config if cfg.AwsConfig != nil { - s3Client := s3.NewFromConfig(*cfg.AwsConfig, func(o *s3.Options) { - o.Region = cfg.Region - o.BaseEndpoint = aws.String(cfg.S3Endpoint) - o.UsePathStyle = cfg.S3ForcePathStyle - }) - + s3Client := s3.NewFromConfig(*cfg.AwsConfig, s3CfgOptions...) client.S3 = s3Client return client } - cfgOptions := []func(*config.LoadOptions) error{ + aswCfgOptions := []func(*config.LoadOptions) error{ config.WithRegion(cfg.Region), } // use EC2 IAM role if cfg.EnableEC2IAMRole { - cfgOptions = append(cfgOptions, config.WithCredentialsProvider( + aswCfgOptions = append(aswCfgOptions, config.WithCredentialsProvider( ec2rolecreds.New(), )) } // use static credentials if cfg.AccessID != "" && cfg.AccessKey != "" { - cfgOptions = append(cfgOptions, config.WithCredentialsProvider( + aswCfgOptions = append(aswCfgOptions, config.WithCredentialsProvider( credentials.NewStaticCredentialsProvider(cfg.AccessID, cfg.AccessKey, cfg.SessionToken), )) } - awsConfig, err := config.LoadDefaultConfig(context.TODO(), cfgOptions...) + awsConfig, err := config.LoadDefaultConfig(context.TODO(), aswCfgOptions...) if err != nil { panic(err) } - s3Client := s3.NewFromConfig(awsConfig, func(o *s3.Options) { - o.Region = cfg.Region - o.BaseEndpoint = aws.String(cfg.S3Endpoint) - o.UsePathStyle = cfg.S3ForcePathStyle - }) - + s3Client := s3.NewFromConfig(awsConfig, s3CfgOptions...) client.S3 = s3Client return client } @@ -147,7 +149,7 @@ func (client Client) Get(path string) (file *os.File, err error) { func (client Client) GetStream(path string) (io.ReadCloser, error) { getResponse, err := client.S3.GetObject(context.TODO(), &s3.GetObjectInput{ Bucket: aws.String(client.Config.Bucket), - Key: aws.String(client.ToRelativePath(path)), + Key: aws.String(client.ToS3Key(path)), }) return getResponse.Body, err @@ -159,7 +161,7 @@ func (client Client) Put(urlPath string, reader io.Reader) (*oss.Object, error) seeker.Seek(0, 0) } - urlPath = client.ToRelativePath(urlPath) + key := client.ToS3Key(urlPath) buffer, err := io.ReadAll(reader) fileType := mime.TypeByExtension(path.Ext(urlPath)) @@ -169,7 +171,7 @@ func (client Client) Put(urlPath string, reader io.Reader) (*oss.Object, error) params := &s3.PutObjectInput{ Bucket: aws.String(client.Config.Bucket), // required - Key: aws.String(urlPath), // required + Key: aws.String(key), // required ACL: client.Config.ACL, Body: bytes.NewReader(buffer), ContentLength: aws.Int64(int64(len(buffer))), @@ -194,7 +196,7 @@ func (client Client) Put(urlPath string, reader io.Reader) (*oss.Object, error) func (client Client) Delete(path string) error { _, err := client.S3.DeleteObject(context.Background(), &s3.DeleteObjectInput{ Bucket: aws.String(client.Config.Bucket), - Key: aws.String(client.ToRelativePath(path)), + Key: aws.String(client.ToS3Key(path)), }) return err } @@ -204,7 +206,7 @@ func (client Client) DeleteObjects(paths []string) (err error) { var objs []types.ObjectIdentifier for _, v := range paths { var obj types.ObjectIdentifier - obj.Key = aws.String(strings.TrimPrefix(client.ToRelativePath(v), "/")) + obj.Key = aws.String(strings.TrimPrefix(client.ToS3Key(v), "/")) objs = append(objs, obj) } input := &s3.DeleteObjectsInput{ @@ -238,7 +240,7 @@ func (client Client) List(path string) ([]*oss.Object, error) { if err == nil { for _, content := range listObjectsResponse.Contents { objects = append(objects, &oss.Object{ - Path: client.ToRelativePath(*content.Key), + Path: client.ToS3Key(*content.Key), Name: filepath.Base(*content.Key), LastModified: content.LastModified, StorageInterface: client, @@ -255,16 +257,36 @@ func (client Client) GetEndpoint() string { return client.Config.Endpoint } - endpoint := *client.S3.Options().BaseEndpoint - for _, prefix := range []string{"https://", "http://"} { - endpoint = strings.TrimPrefix(endpoint, prefix) + if client.Config.S3Endpoint != "" { + return client.Config.S3Endpoint + } + + if client.Config.S3ForcePathStyle { + return fmt.Sprintf("s3.%s.amazonaws.com/%s", client.Config.Region, client.Config.Bucket) } - return client.Config.Bucket + "." + endpoint + return fmt.Sprintf("%s.s3.%s.amazonaws.com", client.Config.Bucket, client.Config.Region) } var urlRegexp = regexp.MustCompile(`(https?:)?//((\w+).)+(\w+)/`) +// ToS3Key convert URL path to S3 key +func (client Client) ToS3Key(urlPath string) string { + if urlRegexp.MatchString(urlPath) { + if u, err := url.Parse(urlPath); err == nil { + if client.Config.S3ForcePathStyle { // First part of path will be bucket name + return strings.TrimPrefix(u.Path, "/"+client.Config.Bucket) + } + return strings.TrimPrefix(u.Path, "/") + } + } + + if client.Config.S3ForcePathStyle { // First part of path will be bucket name + return strings.TrimPrefix(urlPath, "/"+client.Config.Bucket+"/") + } + return strings.TrimPrefix(urlPath, "/") +} + // ToRelativePath process path to relative path func (client Client) ToRelativePath(urlPath string) string { if urlRegexp.MatchString(urlPath) { @@ -291,7 +313,7 @@ func (client Client) GetURL(path string) (url string, err error) { presignClient := s3.NewPresignClient(client.S3) presignedGetURL, err := presignClient.PresignGetObject(context.TODO(), &s3.GetObjectInput{ Bucket: aws.String(client.Config.Bucket), - Key: aws.String(client.ToRelativePath(path)), + Key: aws.String(client.ToS3Key(path)), }, func(opts *s3.PresignOptions) { opts.Expires = 1 * time.Hour }) @@ -314,3 +336,22 @@ func (client Client) Copy(from, to string) (err error) { }) return } + +type endpointResolverV2 struct { + Url string +} + +func (r *endpointResolverV2) ResolveEndpoint( + ctx context.Context, params s3.EndpointParameters, +) ( + endpoint smithyendpoints.Endpoint, err error, +) { + + u, err := url.Parse(r.Url) + if err != nil { + return smithyendpoints.Endpoint{}, err + } + return smithyendpoints.Endpoint{ + URI: *u, + }, nil +} From 7507fea2888697934db209602f94292e4234a193 Mon Sep 17 00:00:00 2001 From: chenxin0723 <870720136@qq.com> Date: Fri, 22 Nov 2024 16:59:20 +0800 Subject: [PATCH 3/5] chore: add more test cases --- s3/s3.go | 8 +++++--- s3/s3_test.go | 16 ++++++++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/s3/s3.go b/s3/s3.go index b3665f0..a324ca3 100644 --- a/s3/s3.go +++ b/s3/s3.go @@ -152,6 +152,10 @@ func (client Client) GetStream(path string) (io.ReadCloser, error) { Key: aws.String(client.ToS3Key(path)), }) + if err != nil { + return nil, err + } + return getResponse.Body, err } @@ -240,7 +244,7 @@ func (client Client) List(path string) ([]*oss.Object, error) { if err == nil { for _, content := range listObjectsResponse.Contents { objects = append(objects, &oss.Object{ - Path: client.ToS3Key(*content.Key), + Path: "/" + client.ToS3Key(*content.Key), Name: filepath.Base(*content.Key), LastModified: content.LastModified, StorageInterface: client, @@ -307,9 +311,7 @@ func (client Client) ToRelativePath(urlPath string) string { // GetURL get public accessible URL func (client Client) GetURL(path string) (url string, err error) { if client.Config.Endpoint == "" { - if client.Config.ACL == types.ObjectCannedACLPrivate || client.Config.ACL == types.ObjectCannedACLAuthenticatedRead { - presignClient := s3.NewPresignClient(client.S3) presignedGetURL, err := presignClient.PresignGetObject(context.TODO(), &s3.GetObjectInput{ Bucket: aws.String(client.Config.Bucket), diff --git a/s3/s3_test.go b/s3/s3_test.go index 6827ed2..62c95da 100644 --- a/s3/s3_test.go +++ b/s3/s3_test.go @@ -58,6 +58,22 @@ func TestToRelativePath(t *testing.T) { } } +func TestToS3Key(t *testing.T) { + urlMap := map[string]string{ + "https://mybucket.s3.amazonaws.com/test/myobject.ext": "test/myobject.ext", + "https://qor-example.com/myobject.ext": "myobject.ext", + "//mybucket.s3.amazonaws.com/myobject.ext": "myobject.ext", + "http://mybucket.s3.amazonaws.com/myobject.ext": "myobject.ext", + "/test/myobject.ext": "test/myobject.ext", + } + + for url, path := range urlMap { + if client.ToS3Key(url) != path { + t.Errorf("%v's relative path should be %v, but got %v", url, path, client.ToRelativePath(url)) + } + } +} + func TestToRelativePathWithS3ForcePathStyle(t *testing.T) { urlMap := map[string]string{ "https://s3.amazonaws.com/mybucket/myobject.ext": "/myobject.ext", From ecff5c5746930d2a5d81e2972fc3352d8abc3116 Mon Sep 17 00:00:00 2001 From: zealllot <502372679@qq.com> Date: Mon, 25 Nov 2024 17:01:33 +0800 Subject: [PATCH 4/5] chore: add listObjectsResponse IsTruncated check --- go.mod | 2 +- s3/s3.go | 23 +++++++++++++++++------ 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 1f52d0c..54cb546 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/aws/aws-sdk-go-v2/credentials v1.17.44 github.com/aws/aws-sdk-go-v2/service/s3 v1.67.0 github.com/aws/aws-sdk-go-v2/service/sts v1.32.4 + github.com/aws/smithy-go v1.22.0 github.com/jinzhu/configor v1.2.1 github.com/qiniu/api.v7/v7 v7.8.2 ) @@ -27,7 +28,6 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.4 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.24.5 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.4 // indirect - github.com/aws/smithy-go v1.22.0 // indirect github.com/gookit/color v1.3.6 // indirect golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect golang.org/x/time v0.3.0 // indirect diff --git a/s3/s3.go b/s3/s3.go index a324ca3..edb7bb5 100644 --- a/s3/s3.go +++ b/s3/s3.go @@ -231,17 +231,22 @@ func (client Client) DeleteObjects(paths []string) (err error) { func (client Client) List(path string) ([]*oss.Object, error) { var objects []*oss.Object var prefix string + var continuationToken *string if path != "" { prefix = strings.Trim(path, "/") + "/" } - listObjectsResponse, err := client.S3.ListObjectsV2(context.Background(), &s3.ListObjectsV2Input{ - Bucket: aws.String(client.Config.Bucket), - Prefix: aws.String(prefix), - }) + for { + listObjectsResponse, err := client.S3.ListObjectsV2(context.TODO(), &s3.ListObjectsV2Input{ + Bucket: aws.String(client.Config.Bucket), + Prefix: aws.String(prefix), + ContinuationToken: continuationToken, + }) + if err != nil { + return nil, err + } - if err == nil { for _, content := range listObjectsResponse.Contents { objects = append(objects, &oss.Object{ Path: "/" + client.ToS3Key(*content.Key), @@ -250,9 +255,15 @@ func (client Client) List(path string) ([]*oss.Object, error) { StorageInterface: client, }) } + + if listObjectsResponse.IsTruncated != nil && *listObjectsResponse.IsTruncated { + continuationToken = listObjectsResponse.NextContinuationToken + } else { + break + } } - return objects, err + return objects, nil } // GetEndpoint get endpoint, FileSystem's endpoint is / From 139e1297d34882a1a474009765485b5a4525fc59 Mon Sep 17 00:00:00 2001 From: chenxin0723 <870720136@qq.com> Date: Mon, 25 Nov 2024 17:37:59 +0800 Subject: [PATCH 5/5] chore: configure the CustomEndpointResolver as an option --- go.mod | 2 +- s3/s3.go | 66 +++++++++++++++++++++++++------------------------------- 2 files changed, 30 insertions(+), 38 deletions(-) diff --git a/go.mod b/go.mod index 54cb546..1f52d0c 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,6 @@ require ( github.com/aws/aws-sdk-go-v2/credentials v1.17.44 github.com/aws/aws-sdk-go-v2/service/s3 v1.67.0 github.com/aws/aws-sdk-go-v2/service/sts v1.32.4 - github.com/aws/smithy-go v1.22.0 github.com/jinzhu/configor v1.2.1 github.com/qiniu/api.v7/v7 v7.8.2 ) @@ -28,6 +27,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.4 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.24.5 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.4 // indirect + github.com/aws/smithy-go v1.22.0 // indirect github.com/gookit/color v1.3.6 // indirect golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect golang.org/x/time v0.3.0 // indirect diff --git a/s3/s3.go b/s3/s3.go index edb7bb5..105cb4b 100644 --- a/s3/s3.go +++ b/s3/s3.go @@ -23,7 +23,6 @@ import ( "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/aws/aws-sdk-go-v2/service/sts" - smithyendpoints "github.com/aws/smithy-go/endpoints" "github.com/qor/oss" ) @@ -35,14 +34,17 @@ type Client struct { // Config S3 client config type Config struct { - AccessID string - AccessKey string - Region string - Bucket string - SessionToken string - ACL types.ObjectCannedACL - Endpoint string - S3Endpoint string + AccessID string + AccessKey string + Region string + Bucket string + SessionToken string + ACL types.ObjectCannedACL + Endpoint string + + S3Endpoint string + CustomEndpointResolver s3.EndpointResolverV2 + S3ForcePathStyle bool CacheControl string @@ -67,10 +69,17 @@ func New(cfg *Config) *Client { } if cfg.S3Endpoint != "" { - s3CfgOptions = append(s3CfgOptions, s3.WithEndpointResolverV2(&endpointResolverV2{ - Url: cfg.S3Endpoint, - })) + s3CfgOptions = append(s3CfgOptions, + func(o *s3.Options) { + o.BaseEndpoint = aws.String(cfg.S3Endpoint) + }, + ) + } + if cfg.CustomEndpointResolver != nil { + s3CfgOptions = append(s3CfgOptions, + s3.WithEndpointResolverV2(cfg.CustomEndpointResolver), + ) } // use role ARN to fetch credentials @@ -272,12 +281,14 @@ func (client Client) GetEndpoint() string { return client.Config.Endpoint } - if client.Config.S3Endpoint != "" { - return client.Config.S3Endpoint - } - - if client.Config.S3ForcePathStyle { - return fmt.Sprintf("s3.%s.amazonaws.com/%s", client.Config.Region, client.Config.Bucket) + if re, err := client.S3.Options().EndpointResolverV2.ResolveEndpoint(context.Background(), s3.EndpointParameters{ + Region: aws.String(client.Config.Region), + }); err == nil { + endpoint := re.URI.String() + for _, prefix := range []string{"https://", "http://"} { + endpoint = strings.TrimPrefix(endpoint, prefix) + } + return client.Config.Bucket + "." + endpoint } return fmt.Sprintf("%s.s3.%s.amazonaws.com", client.Config.Bucket, client.Config.Region) @@ -349,22 +360,3 @@ func (client Client) Copy(from, to string) (err error) { }) return } - -type endpointResolverV2 struct { - Url string -} - -func (r *endpointResolverV2) ResolveEndpoint( - ctx context.Context, params s3.EndpointParameters, -) ( - endpoint smithyendpoints.Endpoint, err error, -) { - - u, err := url.Parse(r.Url) - if err != nil { - return smithyendpoints.Endpoint{}, err - } - return smithyendpoints.Endpoint{ - URI: *u, - }, nil -}