Skip to content

Commit

Permalink
Merge branch 'improvement/CLDSRV-408-Fix-metadata-getting-deleted-whe…
Browse files Browse the repository at this point in the history 
…n-restoring' into tmp/octopus/w/8.8/improvement/CLDSRV-408-Fix-metadata-getting-deleted-when-restoring
  • Loading branch information
@bert-e
bert-e committed Jul 11, 2023
2 parents 0b58b3a + 89e9a8d commit 9cc3d7c
Show file tree
Hide file tree
Showing 7 changed files with 419 additions and 17 deletions.
46 changes: 43 additions & 3 deletions lib/api/apiUtils/object/versioning.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -434,6 +434,47 @@ function preprocessingVersioningDelete(bucketName, bucketMD, objectMD, reqVersio
return options;
}

/**
* Keep metadatas when the object is restored from cold storage
* but remove the specific ones we don't want to keep
* @param {object} objMD - obj metadata
* @param {object} metadataStoreParams - custom built object containing resource details.
* @return {undefined}
*/
function restoreMetadatas(objMD, metadataStoreParams) {
/* eslint-disable no-param-reassign */
const userMDToSkip = ['x-amz-meta-scal-s3-restore-attempt'];
// We need to keep user metadata and tags
Object.keys(objMD).forEach(key => {
if (key.startsWith('x-amz-meta-') && !userMDToSkip.includes(key)) {
metadataStoreParams.metaHeaders[key] = objMD[key];
}
});

if (objMD['x-amz-website-redirect-location']) {
if (!metadataStoreParams.headers) {
metadataStoreParams.headers = {};
}
metadataStoreParams.headers['x-amz-website-redirect-location'] = objMD['x-amz-website-redirect-location'];
}

if (objMD.replicationInfo) {
metadataStoreParams.replicationInfo = objMD.replicationInfo;
}

if (objMD.legalHold) {
metadataStoreParams.legalHold = objMD.legalHold;
}

if (objMD.acl) {
metadataStoreParams.acl = objMD.acl;
}

metadataStoreParams.creationTime = objMD['creation-time'];
metadataStoreParams.lastModifiedDate = objMD['last-modified'];
metadataStoreParams.taggingCopy = objMD.tags;
}

/** overwritingVersioning - return versioning information for S3 to handle
* storing version metadata with a specific version id.
* @param {object} objMD - obj metadata
Expand All @@ -445,9 +486,6 @@ function preprocessingVersioningDelete(bucketName, bucketMD, objectMD, reqVersio
* version id of the null version
*/
function overwritingVersioning(objMD, metadataStoreParams) {
/* eslint-disable no-param-reassign */
metadataStoreParams.creationTime = objMD['creation-time'];
metadataStoreParams.lastModifiedDate = objMD['last-modified'];
metadataStoreParams.updateMicroVersionId = true;

// set correct originOp
Expand Down Expand Up @@ -477,6 +515,8 @@ function overwritingVersioning(objMD, metadataStoreParams) {
};
}

restoreMetadatas(objMD, metadataStoreParams);

return options;
}

Expand Down
6 changes: 6 additions & 0 deletions lib/services.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -254,6 +254,12 @@ const services = {
if (multipart || md.getIsDeleteMarker()) {
return callback();
}
if (params.acl) {
// In case of a restore we dont pass ACL in the headers
// but we take them from the old metadata
md.setAcl(params.acl);
return callback();
}
const parseAclParams = {
headers,
resourceType: 'object',
Expand Down
22 changes: 11 additions & 11 deletions tests/functional/aws-node-sdk/test/object/mpuVersion.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,7 +160,7 @@ describe('MPU with x-scal-s3-version-id header', () => {

checkObjMdAndUpdate(objMDBefore, objMDAfter,
['location', 'uploadId', 'microVersionId', 'x-amz-restore',
'archive', 'dataStoreName', 'originOp']);
'archive', 'dataStoreName', 'originOp', 'acl']);

assert.deepStrictEqual(objMDAfter, objMDBefore);
return done();
Expand Down Expand Up @@ -202,7 +202,7 @@ describe('MPU with x-scal-s3-version-id header', () => {

checkObjMdAndUpdate(objMDBefore, objMDAfter,
['location', 'content-length', 'content-md5', 'originOp', 'uploadId', 'microVersionId',
'x-amz-restore', 'archive', 'dataStoreName']);
'x-amz-restore', 'archive', 'dataStoreName', 'acl']);

assert.deepStrictEqual(objMDAfter, objMDBefore);
return done();
Expand Down Expand Up @@ -255,7 +255,7 @@ describe('MPU with x-scal-s3-version-id header', () => {

checkObjMdAndUpdate(objMDBefore, objMDAfter,
['location', 'content-length', 'content-md5', 'originOp', 'uploadId', 'microVersionId',
'x-amz-restore', 'archive', 'dataStoreName']);
'x-amz-restore', 'archive', 'dataStoreName', 'acl']);
assert.deepStrictEqual(objMDAfter, objMDBefore);
return done();
});
Expand Down Expand Up @@ -307,7 +307,7 @@ describe('MPU with x-scal-s3-version-id header', () => {

checkObjMdAndUpdate(objMDBefore, objMDAfter,
['location', 'content-length', 'content-md5', 'originOp', 'uploadId', 'microVersionId',
'x-amz-restore', 'archive', 'dataStoreName']);
'x-amz-restore', 'archive', 'dataStoreName', 'acl']);
assert.deepStrictEqual(objMDAfter, objMDBefore);
return done();
});
Expand Down Expand Up @@ -414,7 +414,7 @@ describe('MPU with x-scal-s3-version-id header', () => {

checkObjMdAndUpdate(objMDBefore, objMDAfter,
['location', 'content-length', 'content-md5', 'originOp', 'uploadId', 'microVersionId',
'x-amz-restore', 'archive', 'dataStoreName']);
'x-amz-restore', 'archive', 'dataStoreName', 'acl']);
assert.deepStrictEqual(objMDAfter, objMDBefore);
return done();
});
Expand Down Expand Up @@ -467,7 +467,7 @@ describe('MPU with x-scal-s3-version-id header', () => {

checkObjMdAndUpdate(objMDBefore, objMDAfter,
['location', 'content-length', 'content-md5', 'originOp', 'uploadId', 'microVersionId',
'x-amz-restore', 'archive', 'dataStoreName']);
'x-amz-restore', 'archive', 'dataStoreName', 'acl']);
assert.deepStrictEqual(objMDAfter, objMDBefore);
return done();
});
Expand Down Expand Up @@ -523,7 +523,7 @@ describe('MPU with x-scal-s3-version-id header', () => {

checkObjMdAndUpdate(objMDBefore, objMDAfter,
['location', 'content-length', 'content-md5', 'originOp', 'uploadId', 'microVersionId',
'x-amz-restore', 'archive', 'dataStoreName']);
'x-amz-restore', 'archive', 'dataStoreName', 'acl']);
assert.deepStrictEqual(objMDAfter, objMDBefore);
return done();
});
Expand Down Expand Up @@ -577,7 +577,7 @@ describe('MPU with x-scal-s3-version-id header', () => {

checkObjMdAndUpdate(objMDBefore, objMDAfter,
['location', 'content-length', 'content-md5', 'originOp', 'uploadId', 'microVersionId',
'x-amz-restore', 'archive', 'dataStoreName']);
'x-amz-restore', 'archive', 'dataStoreName', 'acl']);
assert.deepStrictEqual(objMDAfter, objMDBefore);
return done();
});
Expand Down Expand Up @@ -630,7 +630,7 @@ describe('MPU with x-scal-s3-version-id header', () => {

checkObjMdAndUpdate(objMDBefore, objMDAfter,
['location', 'content-length', 'content-md5', 'originOp', 'uploadId', 'microVersionId',
'x-amz-restore', 'archive', 'dataStoreName']);
'x-amz-restore', 'archive', 'dataStoreName', 'acl']);
assert.deepStrictEqual(objMDAfter, objMDBefore);
return done();
});
Expand Down Expand Up @@ -690,7 +690,7 @@ describe('MPU with x-scal-s3-version-id header', () => {

checkObjMdAndUpdate(objMDBefore, objMDAfter,
['location', 'content-length', 'content-md5', 'originOp', 'uploadId', 'microVersionId',
'x-amz-restore', 'archive', 'dataStoreName']);
'x-amz-restore', 'archive', 'dataStoreName', 'acl']);
assert.deepStrictEqual(objMDAfter, objMDBefore);
return done();
});
Expand Down Expand Up @@ -738,7 +738,7 @@ describe('MPU with x-scal-s3-version-id header', () => {

checkObjMdAndUpdate(objMDBefore, objMDAfter,
['location', 'content-length', 'content-md5', 'originOp', 'uploadId', 'microVersionId',
'x-amz-restore', 'archive', 'dataStoreName']);
'x-amz-restore', 'archive', 'dataStoreName', 'acl']);

assert.deepStrictEqual(objMDAfter, objMDBefore);
return done();
Expand Down
1 change: 1 addition & 0 deletions tests/functional/aws-node-sdk/test/object/objectCopy.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1279,6 +1279,7 @@ describe('Object Copy', () => {
restoreCompletedAt: new Date(10),
restoreWillExpireAt: new Date(10 + (5 * 24 * 60 * 60 * 1000)),
};
originalMetadata['custom-user-md'] = 'custom-md';
fakeMetadataArchive(sourceBucketName, sourceObjName, undefined, archiveCompleted, err => {
assert.ifError(err);
s3.copyObject({
Expand Down
16 changes: 14 additions & 2 deletions tests/functional/aws-node-sdk/test/object/putVersion.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,8 +89,13 @@ describe('PUT object with x-scal-s3-version-id header', () => {
next => s3.putObject(params, next),
next => fakeMetadataArchive(bucketName, objectName, undefined, archive, next),
next => getMetadata(bucketName, objectName, undefined, (err, objMD) => {
if (err) {
return next(err);
}
// eslint-disable-next-line no-param-reassign
objMD.legalHold = true;
objMDBefore = objMD;
return next(err);
return metadata.putObjectMD(bucketName, objectName, objMD, params, log, err => next(err));
}),
next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => {
versionsBefore = res.Versions;
Expand All @@ -114,7 +119,14 @@ describe('PUT object with x-scal-s3-version-id header', () => {
checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'content-md5',
'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName', 'originOp']);
assert.deepStrictEqual(objMDAfter, objMDBefore);
return done();
return getMetadata(bucketName, objectName, undefined, (err, objMD) => {
if (err) {
return done(err);
}
// eslint-disable-next-line no-param-reassign
objMD.legalHold = false;
return metadata.putObjectMD(bucketName, objectName, objMD, params, log, err => done(err));
});
});
});

Expand Down
32 changes: 32 additions & 0 deletions tests/functional/aws-node-sdk/test/utils/init.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,38 @@ function fakeMetadataArchive(bucketName, objectName, versionId, archive, cb) {
/* eslint-disable no-param-reassign */
objMD.dataStoreName = 'location-dmf-v1';
objMD.archive = archive;
objMD['x-amz-meta-custom-user-md'] = 'custom-md';
objMD.acl = {
'Canned': '',
'FULL_CONTROL': [
'cb676c668a4eb77a38c490548f9c95909353181a2ca8ec115d738920a7746a34'
],
'WRITE_ACP': [],
'READ': [],
'READ_ACP': []
};
objMD.tags = { tag1: 'value1', tag2: 'value2' };
objMD['x-amz-website-redirect-location'] = 'https://scality.com/';
objMD.replicationInfo = {
'status': 'PENDING',
'backends': [
{
'site': 'azure-normal',
'status': 'PENDING',
'dataStoreVersionId': ''
}
],
'content': [
'DATA',
'METADATA'
],
'destination': 'arn:aws:s3:::versioned',
'storageClass': 'azure-normal',
'role': 'arn:aws:iam::root:role/s3-replication-role',
'storageType': 'azure',
'dataStoreVersionId': '',
'isNFS': null
};
/* eslint-enable no-param-reassign */
return metadata.putObjectMD(bucketName, objectName, objMD, { versionId: decodeVersionId(versionId) },
log, err => cb(err));
Expand Down
Loading

0 comments on commit 9cc3d7c

Please sign in to comment.