diff --git a/lib/api/apiUtils/object/versioning.js b/lib/api/apiUtils/object/versioning.js index 6dfddc9f82..e4c008fc7b 100644 --- a/lib/api/apiUtils/object/versioning.js +++ b/lib/api/apiUtils/object/versioning.js @@ -434,6 +434,47 @@ function preprocessingVersioningDelete(bucketName, bucketMD, objectMD, reqVersio return options; } +/** + * Keep metadatas when the object is restored from cold storage + * but remove the specific ones we don't want to keep + * @param {object} objMD - obj metadata + * @param {object} metadataStoreParams - custom built object containing resource details. + * @return {undefined} + */ +function restoreMetadatas(objMD, metadataStoreParams) { + /* eslint-disable no-param-reassign */ + const userMDToSkip = ['x-amz-meta-scal-s3-restore-attempt']; + // We need to keep user metadata and tags + Object.keys(objMD).forEach(key => { + if (key.startsWith('x-amz-meta-') && !userMDToSkip.includes(key)) { + metadataStoreParams.metaHeaders[key] = objMD[key]; + } + }); + + if (objMD['x-amz-website-redirect-location']) { + if (!metadataStoreParams.headers) { + metadataStoreParams.headers = {}; + } + metadataStoreParams.headers['x-amz-website-redirect-location'] = objMD['x-amz-website-redirect-location']; + } + + if (objMD.replicationInfo) { + metadataStoreParams.replicationInfo = objMD.replicationInfo; + } + + if (objMD.legalHold) { + metadataStoreParams.legalHold = objMD.legalHold; + } + + if (objMD.acl) { + metadataStoreParams.acl = objMD.acl; + } + + metadataStoreParams.creationTime = objMD['creation-time']; + metadataStoreParams.lastModifiedDate = objMD['last-modified']; + metadataStoreParams.taggingCopy = objMD.tags; +} + /** overwritingVersioning - return versioning information for S3 to handle * storing version metadata with a specific version id. * @param {object} objMD - obj metadata @@ -445,9 +486,6 @@ function preprocessingVersioningDelete(bucketName, bucketMD, objectMD, reqVersio * version id of the null version */ function overwritingVersioning(objMD, metadataStoreParams) { - /* eslint-disable no-param-reassign */ - metadataStoreParams.creationTime = objMD['creation-time']; - metadataStoreParams.lastModifiedDate = objMD['last-modified']; metadataStoreParams.updateMicroVersionId = true; // set correct originOp @@ -477,6 +515,8 @@ function overwritingVersioning(objMD, metadataStoreParams) { }; } + restoreMetadatas(objMD, metadataStoreParams); + return options; } diff --git a/lib/services.js b/lib/services.js index eda0528cfa..75e8544395 100644 --- a/lib/services.js +++ b/lib/services.js @@ -254,6 +254,12 @@ const services = { if (multipart || md.getIsDeleteMarker()) { return callback(); } + if (params.acl) { + // In case of a restore we dont pass ACL in the headers + // but we take them from the old metadata + md.setAcl(params.acl); + return callback(); + } const parseAclParams = { headers, resourceType: 'object', diff --git a/tests/functional/aws-node-sdk/test/object/mpuVersion.js b/tests/functional/aws-node-sdk/test/object/mpuVersion.js index 6be99a98eb..fa83786d65 100644 --- a/tests/functional/aws-node-sdk/test/object/mpuVersion.js +++ b/tests/functional/aws-node-sdk/test/object/mpuVersion.js @@ -160,7 +160,7 @@ describe('MPU with x-scal-s3-version-id header', () => { checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'uploadId', 'microVersionId', 'x-amz-restore', - 'archive', 'dataStoreName', 'originOp']); + 'archive', 'dataStoreName', 'originOp', 'acl']); assert.deepStrictEqual(objMDAfter, objMDBefore); return done(); @@ -202,7 +202,7 @@ describe('MPU with x-scal-s3-version-id header', () => { checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'content-md5', 'originOp', 'uploadId', 'microVersionId', - 'x-amz-restore', 'archive', 'dataStoreName']); + 'x-amz-restore', 'archive', 'dataStoreName', 'acl']); assert.deepStrictEqual(objMDAfter, objMDBefore); return done(); @@ -255,7 +255,7 @@ describe('MPU with x-scal-s3-version-id header', () => { checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'content-md5', 'originOp', 'uploadId', 'microVersionId', - 'x-amz-restore', 'archive', 'dataStoreName']); + 'x-amz-restore', 'archive', 'dataStoreName', 'acl']); assert.deepStrictEqual(objMDAfter, objMDBefore); return done(); }); @@ -307,7 +307,7 @@ describe('MPU with x-scal-s3-version-id header', () => { checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'content-md5', 'originOp', 'uploadId', 'microVersionId', - 'x-amz-restore', 'archive', 'dataStoreName']); + 'x-amz-restore', 'archive', 'dataStoreName', 'acl']); assert.deepStrictEqual(objMDAfter, objMDBefore); return done(); }); @@ -414,7 +414,7 @@ describe('MPU with x-scal-s3-version-id header', () => { checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'content-md5', 'originOp', 'uploadId', 'microVersionId', - 'x-amz-restore', 'archive', 'dataStoreName']); + 'x-amz-restore', 'archive', 'dataStoreName', 'acl']); assert.deepStrictEqual(objMDAfter, objMDBefore); return done(); }); @@ -467,7 +467,7 @@ describe('MPU with x-scal-s3-version-id header', () => { checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'content-md5', 'originOp', 'uploadId', 'microVersionId', - 'x-amz-restore', 'archive', 'dataStoreName']); + 'x-amz-restore', 'archive', 'dataStoreName', 'acl']); assert.deepStrictEqual(objMDAfter, objMDBefore); return done(); }); @@ -523,7 +523,7 @@ describe('MPU with x-scal-s3-version-id header', () => { checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'content-md5', 'originOp', 'uploadId', 'microVersionId', - 'x-amz-restore', 'archive', 'dataStoreName']); + 'x-amz-restore', 'archive', 'dataStoreName', 'acl']); assert.deepStrictEqual(objMDAfter, objMDBefore); return done(); }); @@ -577,7 +577,7 @@ describe('MPU with x-scal-s3-version-id header', () => { checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'content-md5', 'originOp', 'uploadId', 'microVersionId', - 'x-amz-restore', 'archive', 'dataStoreName']); + 'x-amz-restore', 'archive', 'dataStoreName', 'acl']); assert.deepStrictEqual(objMDAfter, objMDBefore); return done(); }); @@ -630,7 +630,7 @@ describe('MPU with x-scal-s3-version-id header', () => { checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'content-md5', 'originOp', 'uploadId', 'microVersionId', - 'x-amz-restore', 'archive', 'dataStoreName']); + 'x-amz-restore', 'archive', 'dataStoreName', 'acl']); assert.deepStrictEqual(objMDAfter, objMDBefore); return done(); }); @@ -690,7 +690,7 @@ describe('MPU with x-scal-s3-version-id header', () => { checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'content-md5', 'originOp', 'uploadId', 'microVersionId', - 'x-amz-restore', 'archive', 'dataStoreName']); + 'x-amz-restore', 'archive', 'dataStoreName', 'acl']); assert.deepStrictEqual(objMDAfter, objMDBefore); return done(); }); @@ -738,7 +738,7 @@ describe('MPU with x-scal-s3-version-id header', () => { checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'content-md5', 'originOp', 'uploadId', 'microVersionId', - 'x-amz-restore', 'archive', 'dataStoreName']); + 'x-amz-restore', 'archive', 'dataStoreName', 'acl']); assert.deepStrictEqual(objMDAfter, objMDBefore); return done(); diff --git a/tests/functional/aws-node-sdk/test/object/objectCopy.js b/tests/functional/aws-node-sdk/test/object/objectCopy.js index a1c3a1ac39..13b692bfee 100644 --- a/tests/functional/aws-node-sdk/test/object/objectCopy.js +++ b/tests/functional/aws-node-sdk/test/object/objectCopy.js @@ -1279,6 +1279,7 @@ describe('Object Copy', () => { restoreCompletedAt: new Date(10), restoreWillExpireAt: new Date(10 + (5 * 24 * 60 * 60 * 1000)), }; + originalMetadata['custom-user-md'] = 'custom-md'; fakeMetadataArchive(sourceBucketName, sourceObjName, undefined, archiveCompleted, err => { assert.ifError(err); s3.copyObject({ diff --git a/tests/functional/aws-node-sdk/test/object/putVersion.js b/tests/functional/aws-node-sdk/test/object/putVersion.js index 9dfa93e167..6dd978cc00 100644 --- a/tests/functional/aws-node-sdk/test/object/putVersion.js +++ b/tests/functional/aws-node-sdk/test/object/putVersion.js @@ -89,8 +89,13 @@ describe('PUT object with x-scal-s3-version-id header', () => { next => s3.putObject(params, next), next => fakeMetadataArchive(bucketName, objectName, undefined, archive, next), next => getMetadata(bucketName, objectName, undefined, (err, objMD) => { + if (err) { + return next(err); + } + // eslint-disable-next-line no-param-reassign + objMD.legalHold = true; objMDBefore = objMD; - return next(err); + return metadata.putObjectMD(bucketName, objectName, objMD, params, log, err => next(err)); }), next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => { versionsBefore = res.Versions; @@ -114,7 +119,14 @@ describe('PUT object with x-scal-s3-version-id header', () => { checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'content-md5', 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName', 'originOp']); assert.deepStrictEqual(objMDAfter, objMDBefore); - return done(); + return getMetadata(bucketName, objectName, undefined, (err, objMD) => { + if (err) { + return done(err); + } + // eslint-disable-next-line no-param-reassign + objMD.legalHold = false; + return metadata.putObjectMD(bucketName, objectName, objMD, params, log, err => done(err)); + }); }); }); diff --git a/tests/functional/aws-node-sdk/test/utils/init.js b/tests/functional/aws-node-sdk/test/utils/init.js index 9816cbfe60..2465aedb51 100644 --- a/tests/functional/aws-node-sdk/test/utils/init.js +++ b/tests/functional/aws-node-sdk/test/utils/init.js @@ -79,6 +79,38 @@ function fakeMetadataArchive(bucketName, objectName, versionId, archive, cb) { /* eslint-disable no-param-reassign */ objMD.dataStoreName = 'location-dmf-v1'; objMD.archive = archive; + objMD['x-amz-meta-custom-user-md'] = 'custom-md'; + objMD.acl = { + 'Canned': '', + 'FULL_CONTROL': [ + 'cb676c668a4eb77a38c490548f9c95909353181a2ca8ec115d738920a7746a34' + ], + 'WRITE_ACP': [], + 'READ': [], + 'READ_ACP': [] + }; + objMD.tags = { tag1: 'value1', tag2: 'value2' }; + objMD['x-amz-website-redirect-location'] = 'https://scality.com/'; + objMD.replicationInfo = { + 'status': 'PENDING', + 'backends': [ + { + 'site': 'azure-normal', + 'status': 'PENDING', + 'dataStoreVersionId': '' + } + ], + 'content': [ + 'DATA', + 'METADATA' + ], + 'destination': 'arn:aws:s3:::versioned', + 'storageClass': 'azure-normal', + 'role': 'arn:aws:iam::root:role/s3-replication-role', + 'storageType': 'azure', + 'dataStoreVersionId': '', + 'isNFS': null + }; /* eslint-enable no-param-reassign */ return metadata.putObjectMD(bucketName, objectName, objMD, { versionId: decodeVersionId(versionId) }, log, err => cb(err)); diff --git a/tests/unit/api/apiUtils/versioning.js b/tests/unit/api/apiUtils/versioning.js index fca9e06c72..eefe0b8a82 100644 --- a/tests/unit/api/apiUtils/versioning.js +++ b/tests/unit/api/apiUtils/versioning.js @@ -3,9 +3,12 @@ const assert = require('assert'); const { versioning } = require('arsenal'); const { config } = require('../../../../lib/Config'); const INF_VID = versioning.VersionID.getInfVid(config.replicationGroupId); +const { scaledMsPerDay } = config.getTimeOptions(); +const sinon = require('sinon'); const { processVersioningState, getMasterState, - preprocessingVersioningDelete } = + preprocessingVersioningDelete, + overwritingVersioning } = require('../../../../lib/api/apiUtils/object/versioning'); describe('versioning helpers', () => { @@ -616,4 +619,312 @@ describe('versioning helpers', () => { assert.deepStrictEqual(options, testCase[expectedResAttr]); }))); }); + + describe('overwritingVersioning', () => { + const days = 3; + const archiveInfo = { + 'archiveID': '126783123678', + }; + const now = Date.now(); + let clock; + + beforeEach(() => { + clock = sinon.useFakeTimers(now); + }); + + afterEach(() => { + clock.restore(); + }); + + [ + { + description: 'Should update archive with restore infos', + objMD: { + 'versionId': '2345678', + 'creation-time': now, + 'last-modified': now, + 'originOp': 's3:PutObject', + archive: { + 'restoreRequestedDays': days, + 'restoreRequestedAt': now, + archiveInfo + } + }, + expectedRes: { + 'creationTime': now, + 'lastModifiedDate': now, + 'updateMicroVersionId': true, + 'originOp': 's3:ObjectRestore:Completed', + taggingCopy: undefined, + archive: { + archiveInfo, + 'restoreRequestedDays': 3, + 'restoreRequestedAt': now, + 'restoreCompletedAt': new Date(now), + 'restoreWillExpireAt': new Date(now + (days * scaledMsPerDay)), + } + } + }, + { + description: 'Should keep user mds and tags', + hasUserMD: true, + objMD: { + 'versionId': '2345678', + 'creation-time': now, + 'last-modified': now, + 'originOp': 's3:PutObject', + 'x-amz-meta-test': 'test', + 'x-amz-meta-test2': 'test2', + 'tags': { 'testtag': 'testtag', 'testtag2': 'testtag2' }, + archive: { + 'restoreRequestedDays': days, + 'restoreRequestedAt': now, + archiveInfo + } + }, + expectedRes: { + 'creationTime': now, + 'lastModifiedDate': now, + 'updateMicroVersionId': true, + 'originOp': 's3:ObjectRestore:Completed', + 'metaHeaders': { + 'x-amz-meta-test': 'test', + 'x-amz-meta-test2': 'test2', + }, + taggingCopy: { 'testtag': 'testtag', 'testtag2': 'testtag2' }, + archive: { + archiveInfo, + 'restoreRequestedDays': days, + 'restoreRequestedAt': now, + 'restoreCompletedAt': new Date(now), + 'restoreWillExpireAt': new Date(now + (days * scaledMsPerDay)), + } + }, + }, + { + description: 'Should not fail with a nullVersionId', + objMD: { + 'creation-time': now, + 'last-modified': now, + 'originOp': 's3:PutObject', + 'nullVersionId': 'vnull', + 'isNull': true, + archive: { + 'restoreRequestedDays': days, + 'restoreRequestedAt': now, + archiveInfo + } + }, + expectedRes: { + 'creationTime': now, + 'lastModifiedDate': now, + 'updateMicroVersionId': true, + 'originOp': 's3:ObjectRestore:Completed', + taggingCopy: undefined, + archive: { + archiveInfo, + 'restoreRequestedDays': 3, + 'restoreRequestedAt': now, + 'restoreCompletedAt': new Date(now), + 'restoreWillExpireAt': new Date(now + (days * scaledMsPerDay)), + } + } + }, + { + description: 'Should not keep x-amz-meta-scal-s3-restore-attempt user MD', + hasUserMD: true, + objMD: { + 'versionId': '2345678', + 'creation-time': now, + 'last-modified': now, + 'originOp': 's3:PutObject', + 'x-amz-meta-test': 'test', + 'x-amz-meta-scal-s3-restore-attempt': 14, + archive: { + 'restoreRequestedDays': days, + 'restoreRequestedAt': now, + archiveInfo + } + }, + expectedRes: { + 'creationTime': now, + 'lastModifiedDate': now, + 'updateMicroVersionId': true, + 'originOp': 's3:ObjectRestore:Completed', + 'metaHeaders': { + 'x-amz-meta-test': 'test', + }, + taggingCopy: undefined, + archive: { + archiveInfo, + 'restoreRequestedDays': 3, + 'restoreRequestedAt': now, + 'restoreCompletedAt': new Date(now), + 'restoreWillExpireAt': new Date(now + (days * scaledMsPerDay)), + } + } + }, + { + description: 'Should keep replication infos', + objMD: { + 'versionId': '2345678', + 'creation-time': now, + 'last-modified': now, + 'originOp': 's3:PutObject', + 'replicationInfo': { + 'status': 'COMPLETED', + 'backends': [ + { + 'site': 'azure-blob', + 'status': 'COMPLETED', + 'dataStoreVersionId': '' + } + ], + 'content': [ + 'DATA', + 'METADATA' + ], + 'destination': 'arn:aws:s3:::replicate-cold', + 'storageClass': 'azure-blob', + 'role': 'arn:aws:iam::root:role/s3-replication-role', + 'storageType': 'azure', + 'dataStoreVersionId': '', + }, + archive: { + 'restoreRequestedDays': days, + 'restoreRequestedAt': now, + archiveInfo + } + }, + expectedRes: { + 'creationTime': now, + 'lastModifiedDate': now, + 'updateMicroVersionId': true, + 'originOp': 's3:ObjectRestore:Completed', + 'replicationInfo': { + 'status': 'COMPLETED', + 'backends': [ + { + 'site': 'azure-blob', + 'status': 'COMPLETED', + 'dataStoreVersionId': '' + } + ], + 'content': [ + 'DATA', + 'METADATA' + ], + 'destination': 'arn:aws:s3:::replicate-cold', + 'storageClass': 'azure-blob', + 'role': 'arn:aws:iam::root:role/s3-replication-role', + 'storageType': 'azure', + 'dataStoreVersionId': '', + }, + taggingCopy: undefined, + archive: { + archiveInfo, + 'restoreRequestedDays': 3, + 'restoreRequestedAt': now, + 'restoreCompletedAt': new Date(now), + 'restoreWillExpireAt': new Date(now + (days * scaledMsPerDay)), + } + } + }, + { + description: 'Should keep legalHold', + objMD: { + 'versionId': '2345678', + 'creation-time': now, + 'last-modified': now, + 'originOp': 's3:PutObject', + 'legalHold': true, + archive: { + 'restoreRequestedDays': days, + 'restoreRequestedAt': now, + archiveInfo + } + }, + expectedRes: { + 'creationTime': now, + 'lastModifiedDate': now, + 'updateMicroVersionId': true, + 'originOp': 's3:ObjectRestore:Completed', + 'legalHold': true, + taggingCopy: undefined, + archive: { + archiveInfo, + 'restoreRequestedDays': 3, + 'restoreRequestedAt': now, + 'restoreCompletedAt': new Date(now), + 'restoreWillExpireAt': new Date(now + (days * scaledMsPerDay)), + } + } + }, + { + description: 'Should keep ACLs', + objMD: { + 'versionId': '2345678', + 'creation-time': now, + 'last-modified': now, + 'originOp': 's3:PutObject', + 'acl': { + 'Canned': '', + 'FULL_CONTROL': [ + '872c04772893deae2b48365752362cd92672eb80eb3deea50d89e834a10ce185' + ], + 'WRITE_ACP': [], + 'READ': [ + 'http://acs.amazonaws.com/groups/global/AllUsers' + ], + 'READ_ACP': [] + }, + archive: { + 'restoreRequestedDays': days, + 'restoreRequestedAt': now, + archiveInfo + } + }, + expectedRes: { + 'creationTime': now, + 'lastModifiedDate': now, + 'updateMicroVersionId': true, + 'originOp': 's3:ObjectRestore:Completed', + 'acl': { + 'Canned': '', + 'FULL_CONTROL': [ + '872c04772893deae2b48365752362cd92672eb80eb3deea50d89e834a10ce185' + ], + 'WRITE_ACP': [], + 'READ': [ + 'http://acs.amazonaws.com/groups/global/AllUsers' + ], + 'READ_ACP': [] + }, + taggingCopy: undefined, + archive: { + archiveInfo, + 'restoreRequestedDays': 3, + 'restoreRequestedAt': now, + 'restoreCompletedAt': new Date(now), + 'restoreWillExpireAt': new Date(now + (days * scaledMsPerDay)), + } + } + }, + ].forEach(testCase => { + it(testCase.description, () => { + const metadataStoreParams = {}; + if (testCase.hasUserMD) { + metadataStoreParams.metaHeaders = {}; + } + const options = overwritingVersioning(testCase.objMD, metadataStoreParams); + assert.deepStrictEqual(options.versionId, testCase.objMD.versionId); + assert.deepStrictEqual(metadataStoreParams, testCase.expectedRes); + + if (testCase.objMD.isNull) { + assert.deepStrictEqual(options.extraMD.nullVersionId, 'vnull'); + assert.deepStrictEqual(options.isNull, true); + } + }); + }); + }); });