Skip to content

Commit

Permalink
FIX: several bugs relating types and pointers
Browse files Browse the repository at this point in the history
  • Loading branch information
@rgglez
rgglez committed Jul 11, 2024
1 parent 49b8389 commit 248c241
Show file tree
Hide file tree
Showing 3 changed files with 38 additions and 26 deletions.
3 changes: 3 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,3 +23,6 @@ coverage.*
# Jetbrain IDE
.idea
*.iml

# Patch files
*.patch
42 changes: 24 additions & 18 deletions services/s3/storage.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,10 +71,11 @@ func (s *Storage) createDir(ctx context.Context, path string, opt pairStorageCre
//ref: https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-folders.html
rp += "/"

contentLength := int64(0)
input := &s3.PutObjectInput{
Bucket: aws.String(s.name),
Key: aws.String(rp),
ContentLength: int64(0),
ContentLength: &contentLength,
}
if opt.HasStorageClass {
input.StorageClass = s3types.StorageClass(opt.StorageClass)
Expand Down Expand Up @@ -106,7 +107,7 @@ func (s *Storage) createDir(ctx context.Context, path string, opt pairStorageCre
if v := aws.ToString(output.SSECustomerKeyMD5); v != "" {
sm.ServerSideEncryptionCustomerKeyMd5 = v
}
sm.ServerSideEncryptionBucketKeyEnabled = output.BucketKeyEnabled
sm.ServerSideEncryptionBucketKeyEnabled = *output.BucketKeyEnabled
o.SetSystemMetadata(sm)

return o, nil
Expand Down Expand Up @@ -162,7 +163,7 @@ func (s *Storage) createLink(ctx context.Context, path string, target string, op
if v := aws.ToString(output.SSECustomerKeyMD5); v != "" {
sm.ServerSideEncryptionCustomerKeyMd5 = v
}
sm.ServerSideEncryptionBucketKeyEnabled = output.BucketKeyEnabled
sm.ServerSideEncryptionBucketKeyEnabled = *output.BucketKeyEnabled

o.SetSystemMetadata(sm)

Expand Down Expand Up @@ -201,7 +202,7 @@ func (s *Storage) createMultipart(ctx context.Context, path string, opt pairStor
if v := aws.ToString(output.SSECustomerKeyMD5); v != "" {
sm.ServerSideEncryptionCustomerKeyMd5 = v
}
sm.ServerSideEncryptionBucketKeyEnabled = output.BucketKeyEnabled
sm.ServerSideEncryptionBucketKeyEnabled = *output.BucketKeyEnabled

o.SetSystemMetadata(sm)
return o, nil
Expand Down Expand Up @@ -316,10 +317,11 @@ func (s *Storage) metadata(opt pairStorageMetadata) (meta *types.StorageMeta) {
func (s *Storage) nextObjectPageByDir(ctx context.Context, page *types.ObjectPage) error {
input := page.Status.(*objectPageStatus)

maxKeys := int32(input.maxKeys)
listInput := &s3.ListObjectsV2Input{
Bucket: &s.name,
Delimiter: &input.delimiter,
MaxKeys: int32(input.maxKeys),
MaxKeys: &maxKeys,
ContinuationToken: input.getServiceContinuationToken(),
Prefix: &input.prefix,
}
Expand Down Expand Up @@ -349,7 +351,7 @@ func (s *Storage) nextObjectPageByDir(ctx context.Context, page *types.ObjectPag
page.Data = append(page.Data, o)
}

if !output.IsTruncated {
if !*output.IsTruncated {
return types.IterateDone
}

Expand All @@ -360,9 +362,10 @@ func (s *Storage) nextObjectPageByDir(ctx context.Context, page *types.ObjectPag
func (s *Storage) nextObjectPageByPrefix(ctx context.Context, page *types.ObjectPage) error {
input := page.Status.(*objectPageStatus)

maxKeys := int32(input.maxKeys)
listInput := &s3.ListObjectsV2Input{
Bucket: &s.name,
MaxKeys: int32(input.maxKeys),
MaxKeys: &maxKeys,
ContinuationToken: input.getServiceContinuationToken(),
Prefix: &input.prefix,
}
Expand All @@ -382,7 +385,7 @@ func (s *Storage) nextObjectPageByPrefix(ctx context.Context, page *types.Object

page.Data = append(page.Data, o)
}
if !output.IsTruncated {
if !*output.IsTruncated {
return types.IterateDone
}
input.continuationToken = aws.ToString(output.NextContinuationToken)
Expand All @@ -391,10 +394,11 @@ func (s *Storage) nextObjectPageByPrefix(ctx context.Context, page *types.Object

func (s *Storage) nextPartObjectPageByPrefix(ctx context.Context, page *types.ObjectPage) error {
input := page.Status.(*objectPageStatus)
maxKeys := int32(input.maxKeys)
listInput := &s3.ListMultipartUploadsInput{
Bucket: &s.name,
KeyMarker: &input.keyMarker,
MaxUploads: int32(input.maxKeys),
MaxUploads: &maxKeys,
Prefix: &input.prefix,
UploadIdMarker: &input.uploadIdMarker,
}
Expand All @@ -415,7 +419,7 @@ func (s *Storage) nextPartObjectPageByPrefix(ctx context.Context, page *types.Ob

page.Data = append(page.Data, o)
}
if !output.IsTruncated {
if !*output.IsTruncated {
return types.IterateDone
}
input.keyMarker = aws.ToString(output.KeyMarker)
Expand All @@ -425,10 +429,11 @@ func (s *Storage) nextPartObjectPageByPrefix(ctx context.Context, page *types.Ob

func (s *Storage) nextPartPage(ctx context.Context, page *types.PartPage) error {
input := page.Status.(*partPageStatus)
maxKeys := int32(input.maxParts)
listInput := &s3.ListPartsInput{
Bucket: &s.name,
Key: &input.key,
MaxParts: int32(input.maxParts),
MaxParts: &maxKeys,
PartNumberMarker: &input.partNumberMarker,
UploadId: &input.uploadId,
}
Expand All @@ -444,14 +449,14 @@ func (s *Storage) nextPartPage(ctx context.Context, page *types.PartPage) error
p := &types.Part{
// The returned `PartNumber` is [1, 10000].
// Set Index=*v.PartNumber-1 here to make the `PartNumber` zero-based for user.
Index: int(v.PartNumber) - 1,
Size: v.Size,
Index: int(*v.PartNumber) - 1,
Size: *v.Size,
ETag: aws.ToString(v.ETag),
}

page.Data = append(page.Data, p)
}
if !output.IsTruncated {
if !*output.IsTruncated {
return types.IterateDone
}
input.partNumberMarker = aws.ToString(output.NextPartNumberMarker)
Expand Down Expand Up @@ -660,7 +665,7 @@ func (s *Storage) stat(ctx context.Context, path string, opt pairStorageStat) (o
o.Mode |= types.ModeRead
}
}
o.SetContentLength(output.ContentLength)
o.SetContentLength(*output.ContentLength)
o.SetLastModified(aws.ToTime(output.LastModified))
if output.ContentType != nil {
o.SetContentType(*output.ContentType)
Expand All @@ -684,7 +689,7 @@ func (s *Storage) stat(ctx context.Context, path string, opt pairStorageStat) (o
if v := aws.ToString(output.SSECustomerKeyMD5); v != "" {
sm.ServerSideEncryptionCustomerKeyMd5 = v
}
sm.ServerSideEncryptionBucketKeyEnabled = output.BucketKeyEnabled
sm.ServerSideEncryptionBucketKeyEnabled = *output.BucketKeyEnabled

o.SetSystemMetadata(sm)

Expand Down Expand Up @@ -738,15 +743,16 @@ func (s *Storage) writeMultipart(ctx context.Context, o *types.Object, r io.Read
r = iowrap.CallbackReader(r, opt.IoCallback)
}

partNumber := (int32(index + 1))
input := &s3.UploadPartInput{
Bucket: &s.name,
// For S3, the `PartNumber` is [1, 10000]. But for users, the `PartNumber` is zero-based.
// Set PartNumber=index+1 here to ensure pass in an effective `PartNumber` for `UploadPart`.
// ref: https://docs.aws.amazon.com/AmazonS3/latest/userguide/mpuoverview.html
PartNumber: (int32(index + 1)),
PartNumber: &partNumber,
Key: aws.String(o.ID),
UploadId: aws.String(o.MustGetMultipartID()),
ContentLength: size,
ContentLength: &size,
Body: iowrap.SizedReadSeekCloser(r, size),
}
if opt.HasExpectedBucketOwner {
Expand Down
19 changes: 11 additions & 8 deletions services/s3/utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,10 @@ import (
"encoding/base64"
"errors"
"fmt"
"github.com/aws/smithy-go"
"strings"

"github.com/aws/smithy-go"

"github.com/aws/aws-sdk-go-v2/aws"
signerv4 "github.com/aws/aws-sdk-go-v2/aws/signer/v4"
"github.com/aws/aws-sdk-go-v2/config"
Expand Down Expand Up @@ -307,7 +308,7 @@ func (s *Storage) formatFileObject(v s3types.Object) (o *typ.Object, err error)
// If you want to get the exact object mode, please use `stat`
o.Mode |= typ.ModeRead

o.SetContentLength(v.Size)
o.SetContentLength(*v.Size)
o.SetLastModified(aws.ToTime(v.LastModified))

if v.ETag != nil {
Expand Down Expand Up @@ -395,7 +396,7 @@ func (s *Storage) formatPutObjectInput(path string, size int64, opt pairStorageW
input = &s3.PutObjectInput{
Bucket: aws.String(s.name),
Key: aws.String(rp),
ContentLength: size,
ContentLength: &size,
}

if opt.HasContentMd5 {
Expand All @@ -408,7 +409,7 @@ func (s *Storage) formatPutObjectInput(path string, size int64, opt pairStorageW
input.ExpectedBucketOwner = &opt.ExpectedBucketOwner
}
if opt.HasServerSideEncryptionBucketKeyEnabled {
input.BucketKeyEnabled = opt.ServerSideEncryptionBucketKeyEnabled
input.BucketKeyEnabled = &opt.ServerSideEncryptionBucketKeyEnabled
}
if opt.HasServerSideEncryptionCustomerAlgorithm {
input.SSECustomerAlgorithm, input.SSECustomerKey, input.SSECustomerKeyMD5, err = calculateEncryptionHeaders(opt.ServerSideEncryptionCustomerAlgorithm, opt.ServerSideEncryptionCustomerKey)
Expand Down Expand Up @@ -479,7 +480,7 @@ func (s *Storage) formatCreateMultipartUploadInput(path string, opt pairStorageC
}

if opt.HasServerSideEncryptionBucketKeyEnabled {
input.BucketKeyEnabled = opt.ServerSideEncryptionBucketKeyEnabled
input.BucketKeyEnabled = &opt.ServerSideEncryptionBucketKeyEnabled
}
if opt.HasExpectedBucketOwner {
input.ExpectedBucketOwner = &opt.ExpectedBucketOwner
Expand Down Expand Up @@ -507,11 +508,12 @@ func (s *Storage) formatCreateMultipartUploadInput(path string, opt pairStorageC
func (s *Storage) formatCompleteMultipartUploadInput(o *typ.Object, parts []*typ.Part, opt pairStorageCompleteMultipart) (input *s3.CompleteMultipartUploadInput) {
upload := &s3types.CompletedMultipartUpload{}
for _, v := range parts {
partNumber := int32(v.Index + 1)
upload.Parts = append(upload.Parts, s3types.CompletedPart{
ETag: aws.String(v.ETag),
// For users the `PartNumber` is zero-based. But for S3, the effective `PartNumber` is [1, 10000].
// Set PartNumber=v.Index+1 here to ensure pass in an effective `PartNumber` for `CompletedPart`.
PartNumber: int32(v.Index + 1),
PartNumber: &partNumber,
})
}

Expand All @@ -530,15 +532,16 @@ func (s *Storage) formatCompleteMultipartUploadInput(o *typ.Object, parts []*typ
}

func (s *Storage) formatUploadPartInput(o *typ.Object, size int64, index int, opt pairStorageWriteMultipart) (input *s3.UploadPartInput, err error) {
partNumber := int32(index + 1)
input = &s3.UploadPartInput{
Bucket: &s.name,
// For S3, the `PartNumber` is [1, 10000]. But for users, the `PartNumber` is zero-based.
// Set PartNumber=index+1 here to ensure pass in an effective `PartNumber` for `UploadPart`.
// ref: https://docs.aws.amazon.com/AmazonS3/latest/userguide/mpuoverview.html
PartNumber: int32(index + 1),
PartNumber: &partNumber,
Key: aws.String(o.ID),
UploadId: aws.String(o.MustGetMultipartID()),
ContentLength: size,
ContentLength: &size,
}
if opt.HasExpectedBucketOwner {
input.ExpectedBucketOwner = &opt.ExpectedBucketOwner
Expand Down

0 comments on commit 248c241

Please sign in to comment.