From f7dbcdf0ac80441898a90e1a83cd7994169469a2 Mon Sep 17 00:00:00 2001 From: Pranav Malewadkar Date: Wed, 13 Nov 2024 12:10:50 -0800 Subject: [PATCH 1/3] feat: allow setting encoding type for list calls --- .../providers/s3/apis/internal/list.test.ts | 31 ++++ .../s3/utils/client/S3/cases/listObjectsV2.ts | 171 +++++++++++++++++- .../src/providers/s3/apis/internal/list.ts | 1 + .../storage/src/providers/s3/types/options.ts | 8 + .../s3/utils/client/s3data/listObjectsV2.ts | 48 ++++- .../storage/src/providers/s3/utils/index.ts | 1 + .../src/providers/s3/utils/urlDecoder.ts | 10 + 7 files changed, 267 insertions(+), 3 deletions(-) create mode 100644 packages/storage/src/providers/s3/utils/urlDecoder.ts diff --git a/packages/storage/__tests__/providers/s3/apis/internal/list.test.ts b/packages/storage/__tests__/providers/s3/apis/internal/list.test.ts index 4165926ad6a..4961ced3e71 100644 --- a/packages/storage/__tests__/providers/s3/apis/internal/list.test.ts +++ b/packages/storage/__tests__/providers/s3/apis/internal/list.test.ts @@ -1024,4 +1024,35 @@ describe('list API', () => { }); }); }); + + describe.only.each([ + { + type: 'Prefix', + listFunction: (options?: any) => + list(Amplify, { prefix: 'test/', options }), + }, + { + type: 'Path', + listFunction: (options?: any) => + list(Amplify, { path: 'test/', options }), + }, + ])('Encoding for List with $type', ({ listFunction }) => { + afterEach(() => { + mockListObject.mockClear(); + }); + it('should include encoding type', async () => { + mockListObjectsV2ApiWithPages(1); + + await listFunction({ + encodingType: 'url', + }); + expect(listObjectsV2).toBeLastCalledWithConfigAndInput( + expect.any(Object), + expect.objectContaining({ + Bucket: bucket, + EncodingType: 'url', + }), + ); + }); + }); }); diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listObjectsV2.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listObjectsV2.ts index aa60b74f906..3b785fc0c35 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listObjectsV2.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listObjectsV2.ts @@ -217,7 +217,7 @@ const listObjectsV2ErrorCase403: ApiFunctionalTestCase = [ NoSuchKey The resource you requested does not exist - /mybucket/myfoto.jpg + /mybucket/myfoto.jpg 4442587FB7D0A2F9 `, }, @@ -420,6 +420,173 @@ const listObjectsV2HappyCaseCustomEndpoint: ApiFunctionalTestCase< }) as any, ]; +const listObjectsV2HappyCaseWithEncoding: ApiFunctionalTestCase< + typeof listObjectsV2 +> = [ + 'happy case', + 'listObjectsV2 unicode values with encoding', + listObjectsV2, + { + ...defaultConfig, + }, + { + Bucket: 'bucket', + Prefix: 'Prefix', + EncodingType: 'url', + }, + expect.any(Object), + { + status: 200, + headers: DEFAULT_RESPONSE_HEADERS, + body: ` + + bucket + some%20folder%20with%20%00%20unprintable%20unicode%2F + bad%08key + bad%01key + 6 + 101 + url + false + + public/bad%3Cdiv%3Ekey + 2024-11-05T18:13:11.000Z + "c0e066cc5238dd7937e464fe7572b71a" + 5455 + STANDARD + + + bad%00key + 2024-11-05T18:13:11.000Z + "c0e066cc5238dd7937e464fe7572b71a" + 5455 + STANDARD + + + public/bad%7Fkey + 2024-11-05T18:13:11.000Z + "c0e066cc5238dd7937e464fe7572b71a" + 5455 + STANDARD + + + public/some%20folder%20with%20spaces%2F + + + public/real%0A%0A%0A%0A%0A%0A%0A%0A%0Afunny%0A%0A%0A%0A%0A%0A%0A%0A%0Abiz%2F + + + public/some%20folder%20with%20%E3%81%8A%E3%81%AF%E3%82%88%E3%81%86%20multibyte%20unicode%2F + +`, + }, + expect.objectContaining({ + CommonPrefixes: [ + { + Prefix: 'public/some folder with spaces/', + }, + { + Prefix: 'public/real\n\n\n\n\n\n\n\n\nfunny\n\n\n\n\n\n\n\n\nbiz/', + }, + { + Prefix: 'public/some folder with おはよう multibyte unicode/', + }, + ], + Contents: [ + { + Key: 'public/bad
key', + LastModified: new Date('2024-11-05T18:13:11.000Z'), + ETag: '"c0e066cc5238dd7937e464fe7572b71a"', + Size: 5455, + StorageClass: 'STANDARD', + }, + { + Key: 'bad\x00key', + LastModified: new Date('2024-11-05T18:13:11.000Z'), + ETag: '"c0e066cc5238dd7937e464fe7572b71a"', + Size: 5455, + StorageClass: 'STANDARD', + }, + { + Key: 'public/badkey', + LastModified: new Date('2024-11-05T18:13:11.000Z'), + ETag: '"c0e066cc5238dd7937e464fe7572b71a"', + Size: 5455, + StorageClass: 'STANDARD', + }, + ], + Prefix: 'some folder with \u0000 unprintable unicode/', + Delimiter: 'bad\x08key', + StartAfter: 'bad\x01key', + EncodingType: 'url', + Name: 'bucket', + }) as any, +]; + +const listObjectsV2ErrorCaseNoEncoding: ApiFunctionalTestCase< + typeof listObjectsV2 +> = [ + 'error case', + 'listObjectsV2 unicode values without encoding', + listObjectsV2, + { + ...defaultConfig, + }, + { + Bucket: 'bucket', + Prefix: 'Prefix', + EncodingType: undefined, + }, + expect.any(Object), + { + status: 200, + headers: DEFAULT_RESPONSE_HEADERS, + body: ` + + badname + bad\x01key + 5 + 101 + bad\x08key + false + おはよう multibyte unicode + + public/bad
key + 2024-11-05T18:13:11.000Z + "c0e066cc5238dd7937e464fe7572b71a" + 5455 + STANDARD + + + bad\x00key + 2024-11-05T18:13:11.000Z + "c0e066cc5238dd7937e464fe7572b71a" + 5455 + STANDARD + + + public/bad\x7fkey + 2024-11-05T18:13:11.000Z + "c0e066cc5238dd7937e464fe7572b71a" + 5455 + STANDARD + + + public/some folder with spaces/ + + + public/some folder with \x00 unprintable unicode/ + +`, + }, + { + message: 'An unknown error has occurred.', + name: 'Unknown', + }, +]; + export default [ listObjectsV2HappyCaseTruncated, listObjectsV2HappyCaseComplete, @@ -428,4 +595,6 @@ export default [ listObjectsV2ErrorCaseMissingTruncated, listObjectsV2ErrorCaseMissingToken, listObjectsV2ErrorCase403, + listObjectsV2HappyCaseWithEncoding, + listObjectsV2ErrorCaseNoEncoding, ]; diff --git a/packages/storage/src/providers/s3/apis/internal/list.ts b/packages/storage/src/providers/s3/apis/internal/list.ts index 19fbf050653..f104dae9626 100644 --- a/packages/storage/src/providers/s3/apis/internal/list.ts +++ b/packages/storage/src/providers/s3/apis/internal/list.ts @@ -85,6 +85,7 @@ export const list = async ( ContinuationToken: options?.listAll ? undefined : options?.nextToken, Delimiter: getDelimiter(options), ExpectedBucketOwner: options?.expectedBucketOwner, + EncodingType: options?.encodingType, }; logger.debug(`listing items from "${listParams.Prefix}"`); diff --git a/packages/storage/src/providers/s3/types/options.ts b/packages/storage/src/providers/s3/types/options.ts index 39891185185..a145b1e1031 100644 --- a/packages/storage/src/providers/s3/types/options.ts +++ b/packages/storage/src/providers/s3/types/options.ts @@ -57,6 +57,14 @@ interface CommonOptions { * The expected owner of the target bucket. */ expectedBucketOwner?: string; + + /** + * Specifies the encoding used for response elements that contain + * special characters. By setting this to 'url', special characters + * in object keys are URL-encoded in the response using the + * `application/x-www-form-urlencoded` encoding. + */ + encodingType?: 'url'; } /** diff --git a/packages/storage/src/providers/s3/utils/client/s3data/listObjectsV2.ts b/packages/storage/src/providers/s3/utils/client/s3data/listObjectsV2.ts index 664a8c7e273..269eb5306e1 100644 --- a/packages/storage/src/providers/s3/utils/client/s3data/listObjectsV2.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/listObjectsV2.ts @@ -25,6 +25,7 @@ import { s3TransferHandler, } from '../utils'; import { IntegrityError } from '../../../../../errors/IntegrityError'; +import { urlDecode } from '../../urlDecoder'; import type { ListObjectsV2CommandInput, @@ -94,10 +95,11 @@ const listObjectsV2Deserializer = async ( StartAfter: 'StartAfter', }); - const output = { + const output = decodeEncodedElements({ $metadata: parseMetadata(response), ...contents, - }; + }); + validateCorroboratingElements(output); return output; @@ -155,6 +157,48 @@ const validateCorroboratingElements = (response: ListObjectsV2Output) => { } }; +/** + * Decodes URL-encoded elements in the S3 `ListObjectsV2Output` response when `EncodingType` is `'url'`. + * Applies to values for 'Delimiter', 'Prefix', 'StartAfter' and 'Key' in the response. + */ +const decodeEncodedElements = ( + listOutput: ListObjectsV2Output, +): ListObjectsV2Output => { + if (listOutput.EncodingType !== 'url') { + return listOutput; + } + + const decodedListOutput = { ...listOutput }; + + // Decode top-level properties + (['Delimiter', 'Prefix', 'StartAfter'] as const).forEach(prop => { + const value = listOutput[prop]; + if (typeof value === 'string') { + decodedListOutput[prop] = urlDecode(value); + } + }); + + // Decode 'Key' in each item of 'Contents', if it exists + if (listOutput.Contents) { + decodedListOutput.Contents = listOutput.Contents.map(content => ({ + ...content, + Key: content.Key ? urlDecode(content.Key) : content.Key, + })); + } + + // Decode 'Prefix' in each item of 'CommonPrefixes', if it exists + if (listOutput.CommonPrefixes) { + decodedListOutput.CommonPrefixes = listOutput.CommonPrefixes.map( + content => ({ + ...content, + Prefix: content.Prefix ? urlDecode(content.Prefix) : content.Prefix, + }), + ); + } + + return decodedListOutput; +}; + export const listObjectsV2 = composeServiceApi( s3TransferHandler, listObjectsV2Serializer, diff --git a/packages/storage/src/providers/s3/utils/index.ts b/packages/storage/src/providers/s3/utils/index.ts index 3a1a05c6a8d..a709e025988 100644 --- a/packages/storage/src/providers/s3/utils/index.ts +++ b/packages/storage/src/providers/s3/utils/index.ts @@ -8,3 +8,4 @@ export { validateBucketOwnerID } from './validateBucketOwnerID'; export { validateStorageOperationInput } from './validateStorageOperationInput'; export { validateStorageOperationInputWithPrefix } from './validateStorageOperationInputWithPrefix'; export { isInputWithPath } from './isInputWithPath'; +export { urlDecode } from './urlDecoder'; diff --git a/packages/storage/src/providers/s3/utils/urlDecoder.ts b/packages/storage/src/providers/s3/utils/urlDecoder.ts new file mode 100644 index 00000000000..a799cad66da --- /dev/null +++ b/packages/storage/src/providers/s3/utils/urlDecoder.ts @@ -0,0 +1,10 @@ +/** + * Decodes a URL-encoded string by replacing '+' characters with spaces and applying `decodeURIComponent`. + * Reference: + * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/decodeURIComponent#decoding_query_parameters_from_a_url + * @param {string} value - The URL-encoded string to decode. + * @returns {string} The decoded string. + */ +export const urlDecode = (value: string): string => { + return decodeURIComponent(value.replace(/\+/g, ' ')); +}; From bfe24438f692f35c74b516a9d4126f8b84ff44a5 Mon Sep 17 00:00:00 2001 From: Pranav Malewadkar Date: Wed, 13 Nov 2024 17:53:56 -0800 Subject: [PATCH 2/3] refactor: move to list internal --- .../providers/s3/apis/internal/list.test.ts | 149 ++++++++++++------ .../s3/utils/client/S3/cases/listObjectsV2.ts | 20 +-- .../src/providers/s3/apis/internal/list.ts | 59 ++++++- .../storage/src/providers/s3/types/options.ts | 8 - .../s3/utils/client/s3data/listObjectsV2.ts | 47 +----- 5 files changed, 165 insertions(+), 118 deletions(-) diff --git a/packages/storage/__tests__/providers/s3/apis/internal/list.test.ts b/packages/storage/__tests__/providers/s3/apis/internal/list.test.ts index 4961ced3e71..e861652a90e 100644 --- a/packages/storage/__tests__/providers/s3/apis/internal/list.test.ts +++ b/packages/storage/__tests__/providers/s3/apis/internal/list.test.ts @@ -184,11 +184,11 @@ describe('list API', () => { expect(listObjectsV2).toHaveBeenCalledTimes(1); await expect(listObjectsV2).toBeLastCalledWithConfigAndInput( listObjectClientConfig, - { + expect.objectContaining({ Bucket: bucket, MaxKeys: 1000, Prefix: expectedKey, - }, + }), ); }); }); @@ -226,12 +226,12 @@ describe('list API', () => { expect(listObjectsV2).toHaveBeenCalledTimes(1); await expect(listObjectsV2).toBeLastCalledWithConfigAndInput( listObjectClientConfig, - { + expect.objectContaining({ Bucket: bucket, Prefix: expectedKey, ContinuationToken: nextToken, MaxKeys: customPageSize, - }, + }), ); }); }); @@ -260,11 +260,11 @@ describe('list API', () => { expect(listObjectsV2).toHaveBeenCalledTimes(1); await expect(listObjectsV2).toBeLastCalledWithConfigAndInput( listObjectClientConfig, - { + expect.objectContaining({ Bucket: bucket, MaxKeys: 1000, Prefix: expectedKey, - }, + }), ); }); }); @@ -297,23 +297,23 @@ describe('list API', () => { await expect(listObjectsV2).toHaveBeenNthCalledWithConfigAndInput( 1, listObjectClientConfig, - { + expect.objectContaining({ Bucket: bucket, Prefix: expectedKey, MaxKeys: 1000, ContinuationToken: undefined, - }, + }), ); // last input receives TEST_TOKEN as the Continuation Token await expect(listObjectsV2).toHaveBeenNthCalledWithConfigAndInput( 3, listObjectClientConfig, - { + expect.objectContaining({ Bucket: bucket, Prefix: expectedKey, MaxKeys: 1000, ContinuationToken: nextToken, - }, + }), ); }); }, @@ -348,11 +348,11 @@ describe('list API', () => { region: mockRegion, userAgentValue: expect.any(String), }, - { + expect.objectContaining({ Bucket: mockBucketName, MaxKeys: 1000, Prefix: `public/${inputKey}`, - }, + }), ); }); @@ -382,11 +382,11 @@ describe('list API', () => { region, userAgentValue: expect.any(String), }, - { + expect.objectContaining({ Bucket: bucket, MaxKeys: 1000, Prefix: `public/${inputKey}`, - }, + }), ); }); }); @@ -444,11 +444,11 @@ describe('list API', () => { expect(listObjectsV2).toHaveBeenCalledTimes(1); await expect(listObjectsV2).toBeLastCalledWithConfigAndInput( listObjectClientConfig, - { + expect.objectContaining({ Bucket: bucket, MaxKeys: 1000, Prefix: resolvePath(inputPath), - }, + }), ); }, ); @@ -487,12 +487,12 @@ describe('list API', () => { expect(listObjectsV2).toHaveBeenCalledTimes(1); await expect(listObjectsV2).toBeLastCalledWithConfigAndInput( listObjectClientConfig, - { + expect.objectContaining({ Bucket: bucket, Prefix: resolvePath(inputPath), ContinuationToken: nextToken, MaxKeys: customPageSize, - }, + }), ); }, ); @@ -516,11 +516,11 @@ describe('list API', () => { expect(listObjectsV2).toHaveBeenCalledTimes(1); await expect(listObjectsV2).toBeLastCalledWithConfigAndInput( listObjectClientConfig, - { + expect.objectContaining({ Bucket: bucket, MaxKeys: 1000, Prefix: resolvePath(path), - }, + }), ); }, ); @@ -552,23 +552,23 @@ describe('list API', () => { await expect(listObjectsV2).toHaveBeenNthCalledWithConfigAndInput( 1, listObjectClientConfig, - { + expect.objectContaining({ Bucket: bucket, Prefix: resolvedPath, MaxKeys: 1000, ContinuationToken: undefined, - }, + }), ); // last input receives TEST_TOKEN as the Continuation Token await expect(listObjectsV2).toHaveBeenNthCalledWithConfigAndInput( 3, listObjectClientConfig, - { + expect.objectContaining({ Bucket: bucket, Prefix: resolvedPath, MaxKeys: 1000, ContinuationToken: nextToken, - }, + }), ); }, ); @@ -602,11 +602,11 @@ describe('list API', () => { region: mockRegion, userAgentValue: expect.any(String), }, - { + expect.objectContaining({ Bucket: mockBucketName, MaxKeys: 1000, Prefix: 'path/', - }, + }), ); }); @@ -636,11 +636,11 @@ describe('list API', () => { region, userAgentValue: expect.any(String), }, - { + expect.objectContaining({ Bucket: bucket, MaxKeys: 1000, Prefix: 'path/', - }, + }), ); }); }); @@ -664,11 +664,11 @@ describe('list API', () => { expect(listObjectsV2).toHaveBeenCalledTimes(1); await expect(listObjectsV2).toBeLastCalledWithConfigAndInput( listObjectClientConfig, - { + expect.objectContaining({ Bucket: bucket, MaxKeys: 1000, Prefix: 'public/', - }, + }), ); expect(error.$metadata.httpStatusCode).toBe(404); } @@ -772,12 +772,12 @@ describe('list API', () => { expect(listObjectsV2).toHaveBeenCalledTimes(1); await expect(listObjectsV2).toBeLastCalledWithConfigAndInput( listObjectClientConfig, - { + expect.objectContaining({ Bucket: bucket, MaxKeys: 1000, Prefix: mockedPath, Delimiter: '/', - }, + }), ); }); @@ -806,12 +806,12 @@ describe('list API', () => { expect(listObjectsV2).toHaveBeenCalledTimes(1); await expect(listObjectsV2).toBeLastCalledWithConfigAndInput( listObjectClientConfig, - { + expect.objectContaining({ Bucket: bucket, MaxKeys: 1000, Prefix: mockedPath, Delimiter: '/', - }, + }), ); }); @@ -828,12 +828,12 @@ describe('list API', () => { expect(listObjectsV2).toHaveBeenCalledTimes(1); await expect(listObjectsV2).toBeLastCalledWithConfigAndInput( listObjectClientConfig, - { + expect.objectContaining({ Bucket: bucket, MaxKeys: 3, Prefix: mockedPath, Delimiter: '/', - }, + }), ); }); @@ -850,12 +850,12 @@ describe('list API', () => { expect(listObjectsV2).toHaveBeenCalledTimes(1); await expect(listObjectsV2).toBeLastCalledWithConfigAndInput( listObjectClientConfig, - { + expect.objectContaining({ Bucket: bucket, MaxKeys: 1000, Prefix: mockedPath, Delimiter: '-', - }, + }), ); }); @@ -871,12 +871,12 @@ describe('list API', () => { expect(listObjectsV2).toHaveBeenCalledTimes(1); await expect(listObjectsV2).toBeLastCalledWithConfigAndInput( listObjectClientConfig, - { + expect.objectContaining({ Bucket: bucket, MaxKeys: 1000, Prefix: mockedPath, Delimiter: undefined, - }, + }), ); }); @@ -887,12 +887,12 @@ describe('list API', () => { expect(listObjectsV2).toHaveBeenCalledTimes(1); await expect(listObjectsV2).toBeLastCalledWithConfigAndInput( listObjectClientConfig, - { + expect.objectContaining({ Bucket: bucket, MaxKeys: 1000, Prefix: mockedPath, Delimiter: undefined, - }, + }), ); }); }); @@ -1025,27 +1025,56 @@ describe('list API', () => { }); }); - describe.only.each([ + describe.each([ { type: 'Prefix', listFunction: (options?: any) => - list(Amplify, { prefix: 'test/', options }), + list(Amplify, { + prefix: 'some folder with unprintable unicode/', + options, + }), + key: 'key', }, { type: 'Path', listFunction: (options?: any) => - list(Amplify, { path: 'test/', options }), + list(Amplify, { + path: 'public/some folder with unprintable unicode/', + options, + }), + key: 'path', }, - ])('Encoding for List with $type', ({ listFunction }) => { + ])('Encoding for List with $type', ({ listFunction, key }) => { afterEach(() => { mockListObject.mockClear(); }); - it('should include encoding type', async () => { - mockListObjectsV2ApiWithPages(1); + it('should decode encoded list output', async () => { + const encodedBadKeys = [ + 'some+folder+with+spaces/', + 'real%0A%0A%0A%0A%0A%0A%0A%0A%0Afunny%0A%0A%0A%0A%0A%0A%0A%0A%0Abiz', + 'some+folder+with+%E3%81%8A%E3%81%AF%E3%82%88%E3%81%86+multibyte+unicode/', + 'bad%3Cdiv%3Ekey', + 'bad%00key', + 'bad%01key', + ]; + + mockListObject.mockReturnValueOnce({ + Name: bucket, + Prefix: 'public/some+folder+with++unprintable+unicode/', + Delimiter: 'bad%08key', + MaxKeys: 1000, + StartAfter: 'bad%7Fbiz/', + EncodingType: 'url', + Contents: encodedBadKeys.map(badKey => ({ + ...listObjectClientBaseResultItem, + Key: key === 'key' ? `public/${badKey}` : badKey, + })), + }); - await listFunction({ - encodingType: 'url', + const result = await listFunction({ + subpathStrategy: { strategy: 'exclude', delimiter: 'bad\x08key' }, }); + expect(listObjectsV2).toBeLastCalledWithConfigAndInput( expect.any(Object), expect.objectContaining({ @@ -1053,6 +1082,26 @@ describe('list API', () => { EncodingType: 'url', }), ); + + const decodedKeys = [ + 'some folder with spaces/', + 'real\x0a\x0a\x0a\x0a\x0a\x0a\x0a\x0a\x0afunny\x0a\x0a\x0a\x0a\x0a\x0a\x0a\x0a\x0abiz', + 'some folder with おはよう multibyte unicode/', + 'bad
key', + 'bad\x00key', + 'bad\x01key', + ]; + + const expectedResult = { + items: decodedKeys.map(decodedKey => ({ + [key]: decodedKey, + eTag: 'eTag', + lastModified: 'lastModified', + size: 'size', + })), + nextToken: undefined, + }; + expect(result).toEqual(expectedResult); }); }); }); diff --git a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listObjectsV2.ts b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listObjectsV2.ts index 3b785fc0c35..1a7ff38e323 100644 --- a/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listObjectsV2.ts +++ b/packages/storage/__tests__/providers/s3/utils/client/S3/cases/listObjectsV2.ts @@ -484,41 +484,43 @@ const listObjectsV2HappyCaseWithEncoding: ApiFunctionalTestCase< expect.objectContaining({ CommonPrefixes: [ { - Prefix: 'public/some folder with spaces/', + Prefix: 'public/some%20folder%20with%20spaces%2F', }, { - Prefix: 'public/real\n\n\n\n\n\n\n\n\nfunny\n\n\n\n\n\n\n\n\nbiz/', + Prefix: + 'public/real%0A%0A%0A%0A%0A%0A%0A%0A%0Afunny%0A%0A%0A%0A%0A%0A%0A%0A%0Abiz%2F', }, { - Prefix: 'public/some folder with おはよう multibyte unicode/', + Prefix: + 'public/some%20folder%20with%20%E3%81%8A%E3%81%AF%E3%82%88%E3%81%86%20multibyte%20unicode%2F', }, ], Contents: [ { - Key: 'public/bad
key', + Key: 'public/bad%3Cdiv%3Ekey', LastModified: new Date('2024-11-05T18:13:11.000Z'), ETag: '"c0e066cc5238dd7937e464fe7572b71a"', Size: 5455, StorageClass: 'STANDARD', }, { - Key: 'bad\x00key', + Key: 'bad%00key', LastModified: new Date('2024-11-05T18:13:11.000Z'), ETag: '"c0e066cc5238dd7937e464fe7572b71a"', Size: 5455, StorageClass: 'STANDARD', }, { - Key: 'public/badkey', + Key: 'public/bad%7Fkey', LastModified: new Date('2024-11-05T18:13:11.000Z'), ETag: '"c0e066cc5238dd7937e464fe7572b71a"', Size: 5455, StorageClass: 'STANDARD', }, ], - Prefix: 'some folder with \u0000 unprintable unicode/', - Delimiter: 'bad\x08key', - StartAfter: 'bad\x01key', + Prefix: 'some%20folder%20with%20%00%20unprintable%20unicode%2F', + Delimiter: 'bad%08key', + StartAfter: 'bad%01key', EncodingType: 'url', Name: 'bucket', }) as any, diff --git a/packages/storage/src/providers/s3/apis/internal/list.ts b/packages/storage/src/providers/s3/apis/internal/list.ts index f104dae9626..ef8d277b48d 100644 --- a/packages/storage/src/providers/s3/apis/internal/list.ts +++ b/packages/storage/src/providers/s3/apis/internal/list.ts @@ -14,6 +14,7 @@ import { } from '../../types'; import { resolveS3ConfigAndInput, + urlDecode, validateBucketOwnerID, validateStorageOperationInputWithPrefix, } from '../../utils'; @@ -85,7 +86,7 @@ export const list = async ( ContinuationToken: options?.listAll ? undefined : options?.nextToken, Delimiter: getDelimiter(options), ExpectedBucketOwner: options?.expectedBucketOwner, - EncodingType: options?.encodingType, + EncodingType: 'url', }; logger.debug(`listing items from "${listParams.Prefix}"`); @@ -160,16 +161,18 @@ const _listWithPrefix = async ({ listParamsClone, ); - validateEchoedElements(listParamsClone, response); + const listOutput = decodeEncodedElements(response); - if (!response?.Contents) { + validateEchoedElements(listParamsClone, listOutput); + + if (!listOutput?.Contents) { return { items: [], }; } return { - items: response.Contents.map(item => ({ + items: listOutput.Contents.map(item => ({ key: generatedPrefix ? item.Key!.substring(generatedPrefix.length) : item.Key!, @@ -177,7 +180,7 @@ const _listWithPrefix = async ({ lastModified: item.LastModified, size: item.Size, })), - nextToken: response.NextContinuationToken, + nextToken: listOutput.NextContinuationToken, }; }; @@ -222,7 +225,7 @@ const _listWithPath = async ({ listParamsClone.MaxKeys = MAX_PAGE_SIZE; } - const listOutput = await listObjectsV2( + const response = await listObjectsV2( { ...s3Config, userAgentValue: getStorageUserAgentValue(StorageAction.List), @@ -230,6 +233,8 @@ const _listWithPath = async ({ listParamsClone, ); + const listOutput = decodeEncodedElements(response); + validateEchoedElements(listParamsClone, listOutput); const { @@ -296,3 +301,45 @@ const validateEchoedElements = ( throw new IntegrityError(); } }; + +/** + * Decodes URL-encoded elements in the S3 `ListObjectsV2Output` response when `EncodingType` is `'url'`. + * Applies to values for 'Delimiter', 'Prefix', 'StartAfter' and 'Key' in the response. + */ +const decodeEncodedElements = ( + listOutput: ListObjectsV2Output, +): ListObjectsV2Output => { + if (listOutput.EncodingType !== 'url') { + return listOutput; + } + + const decodedListOutput = { ...listOutput }; + + // Decode top-level properties + (['Delimiter', 'Prefix', 'StartAfter'] as const).forEach(prop => { + const value = listOutput[prop]; + if (typeof value === 'string') { + decodedListOutput[prop] = urlDecode(value); + } + }); + + // Decode 'Key' in each item of 'Contents', if it exists + if (listOutput.Contents) { + decodedListOutput.Contents = listOutput.Contents.map(content => ({ + ...content, + Key: content.Key ? urlDecode(content.Key) : content.Key, + })); + } + + // Decode 'Prefix' in each item of 'CommonPrefixes', if it exists + if (listOutput.CommonPrefixes) { + decodedListOutput.CommonPrefixes = listOutput.CommonPrefixes.map( + content => ({ + ...content, + Prefix: content.Prefix ? urlDecode(content.Prefix) : content.Prefix, + }), + ); + } + + return decodedListOutput; +}; diff --git a/packages/storage/src/providers/s3/types/options.ts b/packages/storage/src/providers/s3/types/options.ts index a145b1e1031..39891185185 100644 --- a/packages/storage/src/providers/s3/types/options.ts +++ b/packages/storage/src/providers/s3/types/options.ts @@ -57,14 +57,6 @@ interface CommonOptions { * The expected owner of the target bucket. */ expectedBucketOwner?: string; - - /** - * Specifies the encoding used for response elements that contain - * special characters. By setting this to 'url', special characters - * in object keys are URL-encoded in the response using the - * `application/x-www-form-urlencoded` encoding. - */ - encodingType?: 'url'; } /** diff --git a/packages/storage/src/providers/s3/utils/client/s3data/listObjectsV2.ts b/packages/storage/src/providers/s3/utils/client/s3data/listObjectsV2.ts index 269eb5306e1..6caa8a46a8e 100644 --- a/packages/storage/src/providers/s3/utils/client/s3data/listObjectsV2.ts +++ b/packages/storage/src/providers/s3/utils/client/s3data/listObjectsV2.ts @@ -25,7 +25,6 @@ import { s3TransferHandler, } from '../utils'; import { IntegrityError } from '../../../../../errors/IntegrityError'; -import { urlDecode } from '../../urlDecoder'; import type { ListObjectsV2CommandInput, @@ -95,10 +94,10 @@ const listObjectsV2Deserializer = async ( StartAfter: 'StartAfter', }); - const output = decodeEncodedElements({ + const output = { $metadata: parseMetadata(response), ...contents, - }); + }; validateCorroboratingElements(output); @@ -157,48 +156,6 @@ const validateCorroboratingElements = (response: ListObjectsV2Output) => { } }; -/** - * Decodes URL-encoded elements in the S3 `ListObjectsV2Output` response when `EncodingType` is `'url'`. - * Applies to values for 'Delimiter', 'Prefix', 'StartAfter' and 'Key' in the response. - */ -const decodeEncodedElements = ( - listOutput: ListObjectsV2Output, -): ListObjectsV2Output => { - if (listOutput.EncodingType !== 'url') { - return listOutput; - } - - const decodedListOutput = { ...listOutput }; - - // Decode top-level properties - (['Delimiter', 'Prefix', 'StartAfter'] as const).forEach(prop => { - const value = listOutput[prop]; - if (typeof value === 'string') { - decodedListOutput[prop] = urlDecode(value); - } - }); - - // Decode 'Key' in each item of 'Contents', if it exists - if (listOutput.Contents) { - decodedListOutput.Contents = listOutput.Contents.map(content => ({ - ...content, - Key: content.Key ? urlDecode(content.Key) : content.Key, - })); - } - - // Decode 'Prefix' in each item of 'CommonPrefixes', if it exists - if (listOutput.CommonPrefixes) { - decodedListOutput.CommonPrefixes = listOutput.CommonPrefixes.map( - content => ({ - ...content, - Prefix: content.Prefix ? urlDecode(content.Prefix) : content.Prefix, - }), - ); - } - - return decodedListOutput; -}; - export const listObjectsV2 = composeServiceApi( s3TransferHandler, listObjectsV2Serializer, From 2736726b1ece76123caacc8a18e384cc079d2e97 Mon Sep 17 00:00:00 2001 From: Pranav Malewadkar Date: Wed, 13 Nov 2024 17:55:58 -0800 Subject: [PATCH 3/3] fix: add license --- packages/storage/src/providers/s3/utils/urlDecoder.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/storage/src/providers/s3/utils/urlDecoder.ts b/packages/storage/src/providers/s3/utils/urlDecoder.ts index a799cad66da..e812c8a23f4 100644 --- a/packages/storage/src/providers/s3/utils/urlDecoder.ts +++ b/packages/storage/src/providers/s3/utils/urlDecoder.ts @@ -1,3 +1,6 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + /** * Decodes a URL-encoded string by replacing '+' characters with spaces and applying `decodeURIComponent`. * Reference: