From 7e32503fb5e76eac083417f6c5d0420fc8b9b08f Mon Sep 17 00:00:00 2001 From: Jonathan Gramain Date: Tue, 4 Oct 2022 14:37:44 -0700 Subject: [PATCH 01/11] S3UTILS-110 revert crrExistingObjects to 1.4.1 version --- crrExistingObjects.js | 303 ++++++++++++++++++++++-------------------- 1 file changed, 157 insertions(+), 146 deletions(-) diff --git a/crrExistingObjects.js b/crrExistingObjects.js index 06312574..31c5372b 100644 --- a/crrExistingObjects.js +++ b/crrExistingObjects.js @@ -1,42 +1,50 @@ -const { - doWhilst, eachSeries, eachLimit, waterfall, series, -} = require('async'); -const werelogs = require('werelogs'); -const { ObjectMD } = require('arsenal').models; -const metadataUtil = require('./CrrExistingObjects/metadataUtils'); +const http = require('http'); -const logLevel = Number.parseInt(process.env.DEBUG, 10) === 1 - ? 'debug' : 'info'; -const loggerConfig = { - level: logLevel, - dump: 'error', -}; -werelogs.configure(loggerConfig); -const log = new werelogs.Logger('s3utils::crrExistingObjects'); +const AWS = require('aws-sdk'); +const { doWhilst, eachSeries, eachLimit, waterfall } = require('async'); -const BUCKETS = process.argv[2] ? process.argv[2].split(',') : null; -const { SITE_NAME } = process.env; -let { STORAGE_TYPE } = process.env; -let { TARGET_REPLICATION_STATUS } = process.env; -const { TARGET_PREFIX } = process.env; -const WORKERS = (process.env.WORKERS - && Number.parseInt(process.env.WORKERS, 10)) || 10; -const MAX_UPDATES = (process.env.MAX_UPDATES - && Number.parseInt(process.env.MAX_UPDATES, 10)); -const MAX_SCANNED = (process.env.MAX_SCANNED - && Number.parseInt(process.env.MAX_SCANNED, 10)); -let { KEY_MARKER } = process.env; -let { VERSION_ID_MARKER } = process.env; -const { GENERATE_INTERNAL_VERSION_ID } = process.env; +const { Logger } = require('werelogs'); -const LISTING_LIMIT = (process.env.LISTING_LIMIT - && Number.parseInt(process.env.LISTING_LIMIT, 10)) || 1000; +const BackbeatClient = require('./BackbeatClient'); +const log = new Logger('s3utils::crrExistingObjects'); +const BUCKETS = process.argv[2] ? process.argv[2].split(',') : null; +const ACCESS_KEY = process.env.ACCESS_KEY; +const SECRET_KEY = process.env.SECRET_KEY; +const ENDPOINT = process.env.ENDPOINT; +const SITE_NAME = process.env.SITE_NAME; +let STORAGE_TYPE = process.env.STORAGE_TYPE; +let TARGET_REPLICATION_STATUS = process.env.TARGET_REPLICATION_STATUS; +const TARGET_PREFIX = process.env.TARGET_PREFIX; +const WORKERS = (process.env.WORKERS && + Number.parseInt(process.env.WORKERS, 10)) || 10; +const MAX_UPDATES = (process.env.MAX_UPDATES && + Number.parseInt(process.env.MAX_UPDATES, 10)); +const MAX_SCANNED = (process.env.MAX_SCANNED && + Number.parseInt(process.env.MAX_SCANNED, 10)); +let KEY_MARKER = process.env.KEY_MARKER; +let VERSION_ID_MARKER = process.env.VERSION_ID_MARKER; + +const LISTING_LIMIT = 1000; const LOG_PROGRESS_INTERVAL_MS = 10000; +const AWS_SDK_REQUEST_RETRIES = 100; +const AWS_SDK_REQUEST_DELAY_MS = 30; if (!BUCKETS || BUCKETS.length === 0) { - log.fatal('No buckets given as input! Please provide ' - + 'a comma-separated list of buckets'); + log.fatal('No buckets given as input! Please provide ' + + 'a comma-separated list of buckets'); + process.exit(1); +} +if (!ENDPOINT) { + log.fatal('ENDPOINT not defined!'); + process.exit(1); +} +if (!ACCESS_KEY) { + log.fatal('ACCESS_KEY not defined'); + process.exit(1); +} +if (!SECRET_KEY) { + log.fatal('SECRET_KEY not defined'); process.exit(1); } if (!STORAGE_TYPE) { @@ -49,15 +57,47 @@ if (!TARGET_REPLICATION_STATUS) { const replicationStatusToProcess = TARGET_REPLICATION_STATUS.split(','); replicationStatusToProcess.forEach(state => { if (!['NEW', 'PENDING', 'COMPLETED', 'FAILED', 'REPLICA'].includes(state)) { - log.fatal('invalid TARGET_REPLICATION_STATUS environment: must be a ' - + 'comma-separated list of replication statuses to requeue, ' - + 'as NEW, PENDING, COMPLETED, FAILED or REPLICA.'); + log.fatal('invalid TARGET_REPLICATION_STATUS environment: must be a ' + + 'comma-separated list of replication statuses to requeue, ' + + 'as NEW, PENDING, COMPLETED, FAILED or REPLICA.'); process.exit(1); } }); -log.info('Objects with replication status ' - + `${replicationStatusToProcess.join(' or ')} ` - + 'will be reset to PENDING to trigger CRR'); +log.info('Objects with replication status ' + + `${replicationStatusToProcess.join(' or ')} ` + + 'will be reset to PENDING to trigger CRR'); + +const options = { + accessKeyId: ACCESS_KEY, + secretAccessKey: SECRET_KEY, + endpoint: ENDPOINT, + region: 'us-east-1', + sslEnabled: false, + s3ForcePathStyle: true, + apiVersions: { s3: '2006-03-01' }, + signatureVersion: 'v4', + signatureCache: false, + httpOptions: { + timeout: 0, + agent: new http.Agent({ keepAlive: true }), + }, +}; +/** + * Options specific to s3 requests + * `maxRetries` & `customBackoff` are set only to s3 requests + * default aws sdk retry count is 3 with an exponential delay of 2^n * 30 ms + */ +const s3Options = { + maxRetries: AWS_SDK_REQUEST_RETRIES, + customBackoff: (retryCount, error) => { + log.error('aws sdk request error', { error, retryCount }); + // computed delay is not truly exponential, it is reset to minimum after + // every 10 calls, with max delay of 15 seconds! + return AWS_SDK_REQUEST_DELAY_MS * Math.pow(2, retryCount % 10); + }, +}; +const s3 = new AWS.S3(Object.assign(options, s3Options)); +const bb = new BackbeatClient(options); let nProcessed = 0; let nSkipped = 0; @@ -83,39 +123,32 @@ const logProgressInterval = setInterval(_logProgress, LOG_PROGRESS_INTERVAL_MS); function _objectShouldBeUpdated(objMD) { return replicationStatusToProcess.some(filter => { if (filter === 'NEW') { - return (!objMD.getReplicationInfo() - || objMD.getReplicationInfo().status === ''); + return (!objMD.replicationInfo || + objMD.replicationInfo.status === ''); } - return (objMD.getReplicationInfo() - && objMD.getReplicationInfo().status === filter); + return (objMD.replicationInfo && + objMD.replicationInfo.status === filter); }); } -function _markObjectPending( - bucket, - key, - versionId, - storageClass, - repConfig, - cb, -) { +function _markObjectPending(bucket, key, versionId, storageClass, + repConfig, cb) { let objMD; let skip = false; return waterfall([ // get object blob - next => metadataUtil.getMetadata({ + next => bb.getMetadata({ Bucket: bucket, Key: key, VersionId: versionId, - }, log, next), + }, next), (mdRes, next) => { - objMD = new ObjectMD(mdRes); - const md = objMD.getValue(); + objMD = JSON.parse(mdRes.Body); if (!_objectShouldBeUpdated(objMD)) { skip = true; return next(); } - if (objMD.getVersionId()) { + if (objMD.versionId) { // The object already has an *internal* versionId, // which exists when the object has been put on // versioned or versioning-suspended bucket. Even if @@ -124,31 +157,26 @@ function _markObjectPending( // was versioning-suspended when the object was put. return next(); } - if (!GENERATE_INTERNAL_VERSION_ID) { - // When the GENERATE_INTERNAL_VERSION_ID env variable is set, - // matching objects with no *internal* versionId will get - // "updated" to get an internal versionId. The external versionId - // will still be "null". - return next(); - } // The object does not have an *internal* versionId, as it // was put on a nonversioned bucket: do a first metadata - // update to generate one, just passing on the existing metadata - // blob. Note that the resulting key will still be nonversioned, - // but the following update will be able to create a versioned key - // for this object, so that replication can happen. The externally - // visible version will stay "null". - return metadataUtil.putMetadata({ + // update to let cloudserver generate one, just passing on + // the existing metadata blob. Note that the resulting key + // will still be nonversioned, but the following update + // will be able to create a versioned key for this object, + // so that replication can happen. The externally visible + // version will stay "null". + return bb.putMetadata({ Bucket: bucket, Key: key, - Body: md, - }, log, (err, putRes) => { + ContentLength: Buffer.byteLength(mdRes.Body), + Body: mdRes.Body, + }, (err, putRes) => { if (err) { return next(err); } // No need to fetch the whole metadata again, simply // update the one we have with the generated versionId. - objMD.setVersionId(putRes.versionId); + objMD.versionId = putRes.versionId; return next(); }); }, @@ -157,41 +185,33 @@ function _markObjectPending( if (skip) { return next(); } - - // Initialize replication info, if missing - if (!objMD.getReplicationInfo() - || !objMD.getReplicationSiteStatus(storageClass)) { - const { Rules, Role } = repConfig; - const destination = Rules[0].Destination.Bucket; - // set replication properties - const ops = objMD.getContentLength() === 0 ? ['METADATA'] - : ['METADATA', 'DATA']; - const backends = [{ - site: storageClass, - status: 'PENDING', - dataStoreVersionId: '', - }]; - const replicationInfo = { - status: 'PENDING', - backends, - content: ops, - destination, - storageClass, - role: Role, - storageType: STORAGE_TYPE, - }; - objMD.setReplicationInfo(replicationInfo); - } - - objMD.setReplicationSiteStatus(storageClass, 'PENDING'); - objMD.setReplicationStatus('PENDING'); - objMD.updateMicroVersionId(); - const md = objMD.getValue(); - return metadataUtil.putMetadata({ + const { Rules, Role } = repConfig; + const destination = Rules[0].Destination.Bucket; + // set replication properties + const ops = objMD['content-length'] === 0 ? ['METADATA'] : + ['METADATA', 'DATA']; + const backends = [{ + site: storageClass, + status: 'PENDING', + dataStoreVersionId: '', + }]; + const replicationInfo = { + status: 'PENDING', + backends, + content: ops, + destination, + storageClass, + role: Role, + storageType: STORAGE_TYPE, + }; + objMD.replicationInfo = replicationInfo; + const mdBlob = JSON.stringify(objMD); + return bb.putMetadata({ Bucket: bucket, Key: key, - Body: md, - }, log, next); + ContentLength: Buffer.byteLength(mdBlob), + Body: mdBlob, + }, next); }, ], err => { ++nProcessed; @@ -213,19 +233,19 @@ function _markObjectPending( // list object versions function _listObjectVersions(bucket, VersionIdMarker, KeyMarker, cb) { - return metadataUtil.listObjectVersions({ + return s3.listObjectVersions({ Bucket: bucket, MaxKeys: LISTING_LIMIT, Prefix: TARGET_PREFIX, VersionIdMarker, KeyMarker, - }, log, cb); + }, cb); } function _markPending(bucket, versions, cb) { const options = { Bucket: bucket }; waterfall([ - next => metadataUtil.getBucketReplication(options, log, (err, res) => { + next => s3.getBucketReplication(options, (err, res) => { if (err) { log.error('error getting bucket replication', { error: err }); return next(err); @@ -236,14 +256,16 @@ function _markPending(bucket, versions, cb) { const { Rules } = repConfig; const storageClass = Rules[0].Destination.StorageClass || SITE_NAME; if (!storageClass) { - const errMsg = 'missing SITE_NAME environment variable, must be set to' - + ' the value of "site" property in the CRR configuration'; + const errMsg = + 'missing SITE_NAME environment variable, must be set to' + + ' the value of "site" property in the CRR configuration'; log.error(errMsg); return next(new Error(errMsg)); } return eachLimit(versions, WORKERS, (i, apply) => { const { Key, VersionId } = i; - _markObjectPending(bucket, Key, VersionId, storageClass, repConfig, apply); + _markObjectPending( + bucket, Key, VersionId, storageClass, repConfig, apply); }, next); }, ], cb); @@ -259,31 +281,26 @@ function triggerCRROnBucket(bucketName, cb) { VersionIdMarker = VERSION_ID_MARKER; KEY_MARKER = undefined; VERSION_ID_MARKER = undefined; - log.info(`resuming at: KeyMarker=${KeyMarker} ` - + `VersionIdMarker=${VersionIdMarker}`); + log.info(`resuming at: KeyMarker=${KeyMarker} ` + + `VersionIdMarker=${VersionIdMarker}`); } doWhilst( - done => _listObjectVersions( - bucket, - VersionIdMarker, - KeyMarker, + done => _listObjectVersions(bucket, VersionIdMarker, KeyMarker, (err, data) => { if (err) { log.error('error listing object versions', { error: err }); return done(err); } - const versions = data.DeleteMarkers - ? data.Versions.concat(data.DeleteMarkers) : data.Versions; - return _markPending(bucket, versions, err => { - if (err) { - return done(err); - } - VersionIdMarker = data.NextVersionIdMarker; - KeyMarker = data.NextKeyMarker; - return done(); - }); - }, - ), + return _markPending( + bucket, data.Versions.concat(data.DeleteMarkers), err => { + if (err) { + return done(err); + } + VersionIdMarker = data.NextVersionIdMarker; + KeyMarker = data.NextKeyMarker; + return done(); + }); + }), () => { if (nUpdated >= MAX_UPDATES || nProcessed >= MAX_SCANNED) { _logProgress(); @@ -291,23 +308,22 @@ function triggerCRROnBucket(bucketName, cb) { if (VersionIdMarker || KeyMarker) { // next bucket to process is still the current one remainingBuckets = BUCKETS.slice( - BUCKETS.findIndex(bucket => bucket === bucketName), - ); + BUCKETS.findIndex(bucket => bucket === bucketName)); } else { // next bucket to process is the next in bucket list remainingBuckets = BUCKETS.slice( - BUCKETS.findIndex(bucket => bucket === bucketName) + 1, - ); + BUCKETS.findIndex(bucket => bucket === bucketName) + 1); } - let message = 'reached ' - + `${nUpdated >= MAX_UPDATES ? 'update' : 'scanned'} ` - + 'count limit, resuming from this ' - + 'point can be achieved by re-running the script with ' - + `the bucket list "${remainingBuckets.join(',')}"`; + let message = + 'reached ' + + `${nUpdated >= MAX_UPDATES ? 'update' : 'scanned'} ` + + 'count limit, resuming from this ' + + 'point can be achieved by re-running the script with ' + + `the bucket list "${remainingBuckets.join(',')}"`; if (VersionIdMarker || KeyMarker) { message += ' and the following environment variables set: ' - + `KEY_MARKER=${KeyMarker} ` - + `VERSION_ID_MARKER=${VersionIdMarker}`; + + `KEY_MARKER=${KeyMarker} ` + + `VERSION_ID_MARKER=${VersionIdMarker}`; } log.info(message); process.exit(0); @@ -326,16 +342,11 @@ function triggerCRROnBucket(bucketName, cb) { _logProgress(); log.info(`completed task for bucket: ${bucket}`); return cb(); - }, - ); + }); } // trigger the calls to list objects and mark them for crr -series([ - next => metadataUtil.metadataClient.setup(next), - next => eachSeries(BUCKETS, triggerCRROnBucket, next), - next => metadataUtil.metadataClient.close(next), -], err => { +eachSeries(BUCKETS, triggerCRROnBucket, err => { clearInterval(logProgressInterval); if (err) { return log.error('error during task execution', { error: err }); From 7a851568e4a819cc64174d1441596fb614e3e7dc Mon Sep 17 00:00:00 2001 From: Jonathan Gramain Date: Thu, 9 Apr 2020 16:18:46 -0700 Subject: [PATCH 02/11] bugfix: ZENKO-2536 crrExistingObjects: update microVersionId - use ObjectMD model instead of raw parsed JSON object - update the new microVersionId field before putting the metadata changes, to force MongoDB to apply the change and generate an event in the oplog - side fix: call MetadataWrapper.close() so that the script exits after completion (cherry picked from commit 9e979589aaf290189dcc5ac9e77a984cf17d9f87) --- crrExistingObjects.js | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/crrExistingObjects.js b/crrExistingObjects.js index 31c5372b..4965cf27 100644 --- a/crrExistingObjects.js +++ b/crrExistingObjects.js @@ -6,6 +6,7 @@ const { doWhilst, eachSeries, eachLimit, waterfall } = require('async'); const { Logger } = require('werelogs'); const BackbeatClient = require('./BackbeatClient'); +const { ObjectMD } = require('arsenal').models; const log = new Logger('s3utils::crrExistingObjects'); const BUCKETS = process.argv[2] ? process.argv[2].split(',') : null; @@ -123,11 +124,11 @@ const logProgressInterval = setInterval(_logProgress, LOG_PROGRESS_INTERVAL_MS); function _objectShouldBeUpdated(objMD) { return replicationStatusToProcess.some(filter => { if (filter === 'NEW') { - return (!objMD.replicationInfo || - objMD.replicationInfo.status === ''); + return (!objMD.getReplicationInfo() || + objMD.getReplicationInfo().status === ''); } - return (objMD.replicationInfo && - objMD.replicationInfo.status === filter); + return (objMD.getReplicationInfo() && + objMD.getReplicationInfo().status === filter); }); } @@ -143,12 +144,12 @@ function _markObjectPending(bucket, key, versionId, storageClass, VersionId: versionId, }, next), (mdRes, next) => { - objMD = JSON.parse(mdRes.Body); + objMD = new ObjectMD(JSON.parse(mdRes.Body)); if (!_objectShouldBeUpdated(objMD)) { skip = true; return next(); } - if (objMD.versionId) { + if (objMD.getVersionId()) { // The object already has an *internal* versionId, // which exists when the object has been put on // versioned or versioning-suspended bucket. Even if @@ -176,7 +177,7 @@ function _markObjectPending(bucket, key, versionId, storageClass, } // No need to fetch the whole metadata again, simply // update the one we have with the generated versionId. - objMD.versionId = putRes.versionId; + objMD.setVersionId(putRes.versionId); return next(); }); }, @@ -188,7 +189,7 @@ function _markObjectPending(bucket, key, versionId, storageClass, const { Rules, Role } = repConfig; const destination = Rules[0].Destination.Bucket; // set replication properties - const ops = objMD['content-length'] === 0 ? ['METADATA'] : + const ops = objMD.getContentLength() === 0 ? ['METADATA'] : ['METADATA', 'DATA']; const backends = [{ site: storageClass, @@ -204,8 +205,9 @@ function _markObjectPending(bucket, key, versionId, storageClass, role: Role, storageType: STORAGE_TYPE, }; - objMD.replicationInfo = replicationInfo; - const mdBlob = JSON.stringify(objMD); + objMD.setReplicationInfo(replicationInfo); + objMD.updateMicroVersionId(); + const mdBlob = objMD.getSerialized(); return bb.putMetadata({ Bucket: bucket, Key: key, From 38a6bd42ac22c55d94ca6813a6e07833f9a1fa3c Mon Sep 17 00:00:00 2001 From: naren-scality Date: Mon, 13 Apr 2020 21:11:09 -0700 Subject: [PATCH 03/11] bugfix: ZENKO-2555 Fix bug on updating metadata Fix bug on updating metadata. Add 'DEBUG' option to output debug level information. (cherry picked from commit 99a81802eab27487490370e2c9ca28a7ba806f1b) --- crrExistingObjects.js | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/crrExistingObjects.js b/crrExistingObjects.js index 4965cf27..cdbe28eb 100644 --- a/crrExistingObjects.js +++ b/crrExistingObjects.js @@ -3,12 +3,20 @@ const http = require('http'); const AWS = require('aws-sdk'); const { doWhilst, eachSeries, eachLimit, waterfall } = require('async'); -const { Logger } = require('werelogs'); +const werelogs = require('werelogs'); const BackbeatClient = require('./BackbeatClient'); const { ObjectMD } = require('arsenal').models; -const log = new Logger('s3utils::crrExistingObjects'); +const logLevel = Number.parseInt(process.env.DEBUG, 10) === 1 + ? 'debug' : 'info'; +const loggerConfig = { + level: logLevel, + dump: 'error', +}; +werelogs.configure(loggerConfig); +const log = new werelogs.Logger('s3utils::crrExistingObjects'); + const BUCKETS = process.argv[2] ? process.argv[2].split(',') : null; const ACCESS_KEY = process.env.ACCESS_KEY; const SECRET_KEY = process.env.SECRET_KEY; @@ -125,10 +133,10 @@ function _objectShouldBeUpdated(objMD) { return replicationStatusToProcess.some(filter => { if (filter === 'NEW') { return (!objMD.getReplicationInfo() || - objMD.getReplicationInfo().status === ''); + objMD.getReplicationInfo().status === ''); } return (objMD.getReplicationInfo() && - objMD.getReplicationInfo().status === filter); + objMD.getReplicationInfo().status === filter); }); } @@ -145,6 +153,7 @@ function _markObjectPending(bucket, key, versionId, storageClass, }, next), (mdRes, next) => { objMD = new ObjectMD(JSON.parse(mdRes.Body)); + const mdBlob = objMD.getSerialized(); if (!_objectShouldBeUpdated(objMD)) { skip = true; return next(); @@ -160,17 +169,16 @@ function _markObjectPending(bucket, key, versionId, storageClass, } // The object does not have an *internal* versionId, as it // was put on a nonversioned bucket: do a first metadata - // update to let cloudserver generate one, just passing on - // the existing metadata blob. Note that the resulting key - // will still be nonversioned, but the following update - // will be able to create a versioned key for this object, - // so that replication can happen. The externally visible - // version will stay "null". + // update to generate one, just passing on the existing metadata + // blob. Note that the resulting key will still be nonversioned, + // but the following update will be able to create a versioned key + // for this object, so that replication can happen. The externally + // visible version will stay "null". return bb.putMetadata({ Bucket: bucket, Key: key, - ContentLength: Buffer.byteLength(mdRes.Body), - Body: mdRes.Body, + ContentLength: Buffer.byteLength(mdBlob), + Body: mdBlob, }, (err, putRes) => { if (err) { return next(err); From ba452906edb8683247be2256e22fa42720cc700e Mon Sep 17 00:00:00 2001 From: Gregoire Doumergue Date: Thu, 27 Feb 2020 16:09:45 +0100 Subject: [PATCH 04/11] S3C-2662: LISTING_LIMIT configurable via env var (cherry picked from commit ac87ae2592e1a605f75ab38f3bbe97470352c389) --- crrExistingObjects.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crrExistingObjects.js b/crrExistingObjects.js index cdbe28eb..9ed3b25b 100644 --- a/crrExistingObjects.js +++ b/crrExistingObjects.js @@ -34,7 +34,9 @@ const MAX_SCANNED = (process.env.MAX_SCANNED && let KEY_MARKER = process.env.KEY_MARKER; let VERSION_ID_MARKER = process.env.VERSION_ID_MARKER; -const LISTING_LIMIT = 1000; +const LISTING_LIMIT = (process.env.LISTING_LIMIT && + Number.parseInt(process.env.LISTING_LIMIT, 10)) || 1000; + const LOG_PROGRESS_INTERVAL_MS = 10000; const AWS_SDK_REQUEST_RETRIES = 100; const AWS_SDK_REQUEST_DELAY_MS = 30; From 927c1acdb2a7cdd1e3397510d5f9717326cc573b Mon Sep 17 00:00:00 2001 From: Ronnie Smith Date: Thu, 3 Feb 2022 12:34:23 -0800 Subject: [PATCH 05/11] feature: S3UTILS-29 huge lint overhaul from upgraded eslint (cherry picked from commit 724008d576af2431b79a5066e8ee2e0b2cc67504) --- crrExistingObjects.js | 122 +++++++++++++++++++++++------------------- 1 file changed, 66 insertions(+), 56 deletions(-) diff --git a/crrExistingObjects.js b/crrExistingObjects.js index 9ed3b25b..9c537eaf 100644 --- a/crrExistingObjects.js +++ b/crrExistingObjects.js @@ -1,7 +1,9 @@ const http = require('http'); const AWS = require('aws-sdk'); -const { doWhilst, eachSeries, eachLimit, waterfall } = require('async'); +const { + doWhilst, eachSeries, eachLimit, waterfall, +} = require('async'); const werelogs = require('werelogs'); @@ -18,32 +20,32 @@ werelogs.configure(loggerConfig); const log = new werelogs.Logger('s3utils::crrExistingObjects'); const BUCKETS = process.argv[2] ? process.argv[2].split(',') : null; -const ACCESS_KEY = process.env.ACCESS_KEY; -const SECRET_KEY = process.env.SECRET_KEY; -const ENDPOINT = process.env.ENDPOINT; -const SITE_NAME = process.env.SITE_NAME; -let STORAGE_TYPE = process.env.STORAGE_TYPE; -let TARGET_REPLICATION_STATUS = process.env.TARGET_REPLICATION_STATUS; -const TARGET_PREFIX = process.env.TARGET_PREFIX; +const { + ACCESS_KEY, SECRET_KEY, ENDPOINT, SITE_NAME, +} = process.env; +let { + STORAGE_TYPE, TARGET_REPLICATION_STATUS, +} = process.env; +const { TARGET_PREFIX } = process.env; const WORKERS = (process.env.WORKERS && Number.parseInt(process.env.WORKERS, 10)) || 10; const MAX_UPDATES = (process.env.MAX_UPDATES && Number.parseInt(process.env.MAX_UPDATES, 10)); const MAX_SCANNED = (process.env.MAX_SCANNED && Number.parseInt(process.env.MAX_SCANNED, 10)); -let KEY_MARKER = process.env.KEY_MARKER; -let VERSION_ID_MARKER = process.env.VERSION_ID_MARKER; +let { KEY_MARKER } = process.env; +let { VERSION_ID_MARKER } = process.env; -const LISTING_LIMIT = (process.env.LISTING_LIMIT && - Number.parseInt(process.env.LISTING_LIMIT, 10)) || 1000; +const LISTING_LIMIT = (process.env.LISTING_LIMIT + && Number.parseInt(process.env.LISTING_LIMIT, 10)) || 1000; const LOG_PROGRESS_INTERVAL_MS = 10000; const AWS_SDK_REQUEST_RETRIES = 100; const AWS_SDK_REQUEST_DELAY_MS = 30; if (!BUCKETS || BUCKETS.length === 0) { - log.fatal('No buckets given as input! Please provide ' + - 'a comma-separated list of buckets'); + log.fatal('No buckets given as input! Please provide ' + + 'a comma-separated list of buckets'); process.exit(1); } if (!ENDPOINT) { @@ -68,15 +70,15 @@ if (!TARGET_REPLICATION_STATUS) { const replicationStatusToProcess = TARGET_REPLICATION_STATUS.split(','); replicationStatusToProcess.forEach(state => { if (!['NEW', 'PENDING', 'COMPLETED', 'FAILED', 'REPLICA'].includes(state)) { - log.fatal('invalid TARGET_REPLICATION_STATUS environment: must be a ' + - 'comma-separated list of replication statuses to requeue, ' + - 'as NEW, PENDING, COMPLETED, FAILED or REPLICA.'); + log.fatal('invalid TARGET_REPLICATION_STATUS environment: must be a ' + + 'comma-separated list of replication statuses to requeue, ' + + 'as NEW, PENDING, COMPLETED, FAILED or REPLICA.'); process.exit(1); } }); -log.info('Objects with replication status ' + - `${replicationStatusToProcess.join(' or ')} ` + - 'will be reset to PENDING to trigger CRR'); +log.info('Objects with replication status ' + + `${replicationStatusToProcess.join(' or ')} ` + + 'will be reset to PENDING to trigger CRR'); const options = { accessKeyId: ACCESS_KEY, @@ -134,16 +136,22 @@ const logProgressInterval = setInterval(_logProgress, LOG_PROGRESS_INTERVAL_MS); function _objectShouldBeUpdated(objMD) { return replicationStatusToProcess.some(filter => { if (filter === 'NEW') { - return (!objMD.getReplicationInfo() || - objMD.getReplicationInfo().status === ''); + return (!objMD.getReplicationInfo() + || objMD.getReplicationInfo().status === ''); } - return (objMD.getReplicationInfo() && - objMD.getReplicationInfo().status === filter); + return (objMD.getReplicationInfo() + && objMD.getReplicationInfo().status === filter); }); } -function _markObjectPending(bucket, key, versionId, storageClass, - repConfig, cb) { +function _markObjectPending( + bucket, + key, + versionId, + storageClass, + repConfig, + cb, +) { let objMD; let skip = false; return waterfall([ @@ -199,8 +207,8 @@ function _markObjectPending(bucket, key, versionId, storageClass, const { Rules, Role } = repConfig; const destination = Rules[0].Destination.Bucket; // set replication properties - const ops = objMD.getContentLength() === 0 ? ['METADATA'] : - ['METADATA', 'DATA']; + const ops = objMD.getContentLength() === 0 ? ['METADATA'] + : ['METADATA', 'DATA']; const backends = [{ site: storageClass, status: 'PENDING', @@ -268,16 +276,14 @@ function _markPending(bucket, versions, cb) { const { Rules } = repConfig; const storageClass = Rules[0].Destination.StorageClass || SITE_NAME; if (!storageClass) { - const errMsg = - 'missing SITE_NAME environment variable, must be set to' + - ' the value of "site" property in the CRR configuration'; + const errMsg = 'missing SITE_NAME environment variable, must be set to' + + ' the value of "site" property in the CRR configuration'; log.error(errMsg); return next(new Error(errMsg)); } return eachLimit(versions, WORKERS, (i, apply) => { const { Key, VersionId } = i; - _markObjectPending( - bucket, Key, VersionId, storageClass, repConfig, apply); + _markObjectPending(bucket, Key, VersionId, storageClass, repConfig, apply); }, next); }, ], cb); @@ -293,25 +299,27 @@ function triggerCRROnBucket(bucketName, cb) { VersionIdMarker = VERSION_ID_MARKER; KEY_MARKER = undefined; VERSION_ID_MARKER = undefined; - log.info(`resuming at: KeyMarker=${KeyMarker} ` + - `VersionIdMarker=${VersionIdMarker}`); + log.info(`resuming at: KeyMarker=${KeyMarker} ` + + `VersionIdMarker=${VersionIdMarker}`); } doWhilst( - done => _listObjectVersions(bucket, VersionIdMarker, KeyMarker, + done => _listObjectVersions( + bucket, + VersionIdMarker, + KeyMarker, (err, data) => { if (err) { log.error('error listing object versions', { error: err }); return done(err); } - return _markPending( - bucket, data.Versions.concat(data.DeleteMarkers), err => { - if (err) { - return done(err); - } - VersionIdMarker = data.NextVersionIdMarker; - KeyMarker = data.NextKeyMarker; - return done(); - }); + return _markPending(bucket, data.Versions.concat(data.DeleteMarkers), err => { + if (err) { + return done(err); + } + VersionIdMarker = data.NextVersionIdMarker; + KeyMarker = data.NextKeyMarker; + return done(); + }); }), () => { if (nUpdated >= MAX_UPDATES || nProcessed >= MAX_SCANNED) { @@ -320,22 +328,23 @@ function triggerCRROnBucket(bucketName, cb) { if (VersionIdMarker || KeyMarker) { // next bucket to process is still the current one remainingBuckets = BUCKETS.slice( - BUCKETS.findIndex(bucket => bucket === bucketName)); + BUCKETS.findIndex(bucket => bucket === bucketName), + ); } else { // next bucket to process is the next in bucket list remainingBuckets = BUCKETS.slice( - BUCKETS.findIndex(bucket => bucket === bucketName) + 1); + BUCKETS.findIndex(bucket => bucket === bucketName) + 1, + ); } - let message = - 'reached ' + - `${nUpdated >= MAX_UPDATES ? 'update' : 'scanned'} ` + - 'count limit, resuming from this ' + - 'point can be achieved by re-running the script with ' + - `the bucket list "${remainingBuckets.join(',')}"`; + let message = 'reached ' + + `${nUpdated >= MAX_UPDATES ? 'update' : 'scanned'} ` + + 'count limit, resuming from this ' + + 'point can be achieved by re-running the script with ' + + `the bucket list "${remainingBuckets.join(',')}"`; if (VersionIdMarker || KeyMarker) { message += ' and the following environment variables set: ' - + `KEY_MARKER=${KeyMarker} ` + - `VERSION_ID_MARKER=${VersionIdMarker}`; + + `KEY_MARKER=${KeyMarker} ` + + `VERSION_ID_MARKER=${VersionIdMarker}`; } log.info(message); process.exit(0); @@ -354,7 +363,8 @@ function triggerCRROnBucket(bucketName, cb) { _logProgress(); log.info(`completed task for bucket: ${bucket}`); return cb(); - }); + }, + ); } // trigger the calls to list objects and mark them for crr From 5152f11a217c6db6c616ddb7b178d7088a6a1c0c Mon Sep 17 00:00:00 2001 From: Francois Ferrand Date: Mon, 11 Jul 2022 11:45:26 +0200 Subject: [PATCH 06/11] Update the replication status directly The code used to re-create the replication state, which could break some fields (esp. dataLocation) or remove some replication backends. We now simply update the status, which is safer (more future-proof) and simpler. Issue: S3UTILS-77 (cherry picked from commit 9df72074129d8267ed4dbadb6e094046c82cb837) --- crrExistingObjects.js | 23 +++-------------------- 1 file changed, 3 insertions(+), 20 deletions(-) diff --git a/crrExistingObjects.js b/crrExistingObjects.js index 9c537eaf..9b19f919 100644 --- a/crrExistingObjects.js +++ b/crrExistingObjects.js @@ -204,26 +204,9 @@ function _markObjectPending( if (skip) { return next(); } - const { Rules, Role } = repConfig; - const destination = Rules[0].Destination.Bucket; - // set replication properties - const ops = objMD.getContentLength() === 0 ? ['METADATA'] - : ['METADATA', 'DATA']; - const backends = [{ - site: storageClass, - status: 'PENDING', - dataStoreVersionId: '', - }]; - const replicationInfo = { - status: 'PENDING', - backends, - content: ops, - destination, - storageClass, - role: Role, - storageType: STORAGE_TYPE, - }; - objMD.setReplicationInfo(replicationInfo); + + objMD.setReplicationSiteStatus(storageClass, 'PENDING'); + objMD.setReplicationStatus('PENDING'); objMD.updateMicroVersionId(); const mdBlob = objMD.getSerialized(); return bb.putMetadata({ From 02f2669f429bd7c4ec028c4c13894e5097883599 Mon Sep 17 00:00:00 2001 From: Francois Ferrand Date: Wed, 13 Jul 2022 19:47:14 +0200 Subject: [PATCH 07/11] Initialize replicationInfo if not present This is especially needed in case the object was created before replication is enabled on the bucket. Issue: S3UTILS-76 (cherry picked from commit ab227232e9ecb5851b7430f45cd205aff35fc6c9) --- crrExistingObjects.js | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/crrExistingObjects.js b/crrExistingObjects.js index 9b19f919..6e706b73 100644 --- a/crrExistingObjects.js +++ b/crrExistingObjects.js @@ -205,6 +205,31 @@ function _markObjectPending( return next(); } + // Initialize replication info, if missing + if (!objMD.getReplicationInfo() + || !objMD.getReplicationSiteStatus(storageClass)) { + const { Rules, Role } = repConfig; + const destination = Rules[0].Destination.Bucket; + // set replication properties + const ops = objMD.getContentLength() === 0 ? ['METADATA'] + : ['METADATA', 'DATA']; + const backends = [{ + site: storageClass, + status: 'PENDING', + dataStoreVersionId: '', + }]; + const replicationInfo = { + status: 'PENDING', + backends, + content: ops, + destination, + storageClass, + role: Role, + storageType: STORAGE_TYPE, + }; + objMD.setReplicationInfo(replicationInfo); + } + objMD.setReplicationSiteStatus(storageClass, 'PENDING'); objMD.setReplicationStatus('PENDING'); objMD.updateMicroVersionId(); From fd7fa677df4aa4cd0666432530a7c70efaaa48e6 Mon Sep 17 00:00:00 2001 From: Jonathan Gramain Date: Tue, 4 Oct 2022 18:20:53 -0700 Subject: [PATCH 08/11] S3UTILS-110 remove left-over CrrExistingObjects helpers --- CrrExistingObjects/listingParser.js | 24 --- CrrExistingObjects/metadataClient.js | 20 --- CrrExistingObjects/metadataUtils.js | 211 --------------------------- 3 files changed, 255 deletions(-) delete mode 100644 CrrExistingObjects/listingParser.js delete mode 100644 CrrExistingObjects/metadataClient.js delete mode 100644 CrrExistingObjects/metadataUtils.js diff --git a/CrrExistingObjects/listingParser.js b/CrrExistingObjects/listingParser.js deleted file mode 100644 index 5d47b9bd..00000000 --- a/CrrExistingObjects/listingParser.js +++ /dev/null @@ -1,24 +0,0 @@ -function listingParser(entries) { - if (!entries) { - return entries; - } - return entries.map(entry => { - const tmp = JSON.parse(entry.value); - return { - Key: entry.key, - Size: tmp['content-length'], - ETag: tmp['content-md5'], - VersionId: tmp.versionId, - IsNull: tmp.isNull, - IsDeleteMarker: tmp.isDeleteMarker, - LastModified: tmp['last-modified'], - Owner: { - DisplayName: tmp['owner-display-name'], - ID: tmp['owner-id'], - }, - StorageClass: tmp['x-amz-storage-class'], - }; - }); -} - -module.exports = listingParser; diff --git a/CrrExistingObjects/metadataClient.js b/CrrExistingObjects/metadataClient.js deleted file mode 100644 index 58d20c86..00000000 --- a/CrrExistingObjects/metadataClient.js +++ /dev/null @@ -1,20 +0,0 @@ -const { MetadataWrapper } = require('arsenal').storage.metadata; -const werelogs = require('werelogs'); -const createMongoParams = require('../utils/createMongoParams'); -const listingParser = require('./listingParser'); - -const loggerConfig = { - level: 'info', - dump: 'error', -}; -werelogs.configure(loggerConfig); - -const log = new werelogs.Logger('s3utils::crrExistingObjects'); -const implName = 'mongodb'; -const params = { - customListingParser: listingParser, - mongodb: createMongoParams(log, { readPreference: 'primary' }), -}; -const metadata = new MetadataWrapper(implName, params, null, log); - -module.exports = metadata; diff --git a/CrrExistingObjects/metadataUtils.js b/CrrExistingObjects/metadataUtils.js deleted file mode 100644 index 8c197baa..00000000 --- a/CrrExistingObjects/metadataUtils.js +++ /dev/null @@ -1,211 +0,0 @@ -const { errors, versioning } = require('arsenal'); -const metadataClient = require('./metadataClient'); - -const versionIdUtils = versioning.VersionID; - -const { GENERATE_INTERNAL_VERSION_ID } = process.env; -const REPLICATION_GROUP_ID = process.env.REPLICATION_GROUP_ID || 'RG001'; -// Use Arsenal function to generate a version ID used internally by metadata -// for null versions that are created before bucket versioning is configured -const nonVersionedObjId = versionIdUtils.getInfVid(REPLICATION_GROUP_ID); - -function _processVersions(list) { - /* eslint-disable no-param-reassign */ - list.NextVersionIdMarker = list.NextVersionIdMarker - ? versionIdUtils.encode(list.NextVersionIdMarker) - : list.NextVersionIdMarker; - - list.Versions.forEach(v => { - v.VersionId = v.VersionId - ? versionIdUtils.encode(v.VersionId) : v.VersionId; - }); - /* eslint-enable no-param-reassign */ - return list; -} - -function listObjectVersions(params, log, cb) { - const bucketName = params.Bucket; - const listingParams = { - listingType: 'DelimiterVersions', - maxKeys: params.MaxKeys, - prefix: params.Prefix, - keyMarker: params.KeyMarker, - versionIdMarker: params.VersionIdMarker, - }; - log.debug('listing object versions', { - method: 'metadataUtils.listObjectVersions', - listingParams, - }); - return metadataClient.listObject( - bucketName, - listingParams, - log, - (err, list) => { - if (err) { - return cb(err); - } - return cb(null, _processVersions(list)); - }, - ); -} - -function _formatConfig(config) { - const { role, destination, rules } = config; - const Rules = rules.map(rule => { - const { - prefix, enabled, storageClass, id, - } = rule; - return { - ID: id, - Prefix: prefix, - Status: enabled ? 'Enabled' : 'Disabled', - Destination: { - Bucket: destination, - StorageClass: (storageClass || ''), - }, - }; - }); - return { - ReplicationConfiguration: { - Role: role, - Rules, - }, - }; -} - -function getBucketReplication(options, log, cb) { - const bucketName = options.Bucket; - log.debug('getting bucket replication', { - method: 'metadataUtils.getBucketReplication', - bucket: bucketName, - }); - return metadataClient.getBucket(bucketName, log, (err, data) => { - if (err) { - return cb(err); - } - const replConf = _formatConfig(data._replicationConfiguration); - return cb(null, replConf); - }); -} - -function _getNullVersion(objMD, bucketName, objectKey, log, cb) { - const options = {}; - if (objMD.isNull || !objMD.versionId) { - log.debug('found null version'); - return process.nextTick(() => cb(null, objMD)); - } - if (objMD.nullVersionId) { - log.debug('null version exists, get the null version'); - options.versionId = objMD.nullVersionId; - return metadataClient.getObjectMD( - bucketName, - objectKey, - options, - log, - cb, - ); - } - return process.nextTick(() => cb()); -} - -function getMetadata(params, log, cb) { - const { Bucket, Key } = params; - let versionId = params.VersionId; - log.debug('getting object metadata', { - method: 'metadataUtils.getMetadata', - bucket: Bucket, - objectKey: Key, - versionId, - }); - if (versionId && versionId !== 'null') { - versionId = versionIdUtils.decode(versionId); - } - if (versionId instanceof Error) { - const errMsg = 'Invalid version id specified'; - return cb(errors.InvalidArgument.customizeDescription(errMsg)); - } - const mdParams = { - versionId, - }; - return metadataClient.getObjectMD( - Bucket, - Key, - mdParams, - log, - (err, data) => { - if (err) { - return cb(err); - } - if (data && versionId === 'null') { - return _getNullVersion( - data, - Bucket, - Key, - log, - (err, nullVer) => { - if (err) { - return cb(err); - } - return cb(null, nullVer); - }, - ); - } - return cb(null, data); - }, - ); -} - -function getOptions(objMD) { - const options = {}; - - if (objMD.versionId === undefined) { - if (!GENERATE_INTERNAL_VERSION_ID) { - return options; - } - - objMD.setIsNull(true); - objMD.setVersionId(nonVersionedObjId); - - options.nullVersionId = objMD.versionId; - // non-versioned (non-null) MPU objects don't have a - // replay ID, so don't reference their uploadId - if (objMD.uploadId) { - options.nullUploadId = objMD.uploadId; - } - } - - // specify both 'versioning' and 'versionId' to create a "new" - // version (updating master as well) but with specified versionId - options.versioning = true; - options.versionId = objMD.versionId; - return options; -} - -function putMetadata(params, log, cb) { - const { Bucket, Key, Body: objMD } = params; - const options = getOptions(objMD); - - log.debug('updating object metadata', { - method: 'metadataUtils.putMetadata', - bucket: Bucket, - objectKey: Key, - versionId: objMD.versionId, - }); - // If the object is from a source bucket without versioning (i.e. NFS), - // then we want to create a version for the replica object even though - // none was provided in the object metadata value. - if (objMD.replicationInfo.isNFS) { - const isReplica = objMD.replicationInfo.status === 'REPLICA'; - options.versioning = isReplica; - objMD.replicationInfo.isNFS = !isReplica; - } - return metadataClient.putObjectMD(Bucket, Key, objMD, options, log, cb); -} - -module.exports = { - metadataClient, - listObjectVersions, - getBucketReplication, - getMetadata, - putMetadata, -}; From 86351da3ffe1e8815ca942dfb13ece71f9b397d5 Mon Sep 17 00:00:00 2001 From: Jonathan Gramain Date: Tue, 4 Oct 2022 18:44:05 -0700 Subject: [PATCH 09/11] S3UTILS-110 lint --- crrExistingObjects.js | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/crrExistingObjects.js b/crrExistingObjects.js index 6e706b73..e7f072cd 100644 --- a/crrExistingObjects.js +++ b/crrExistingObjects.js @@ -5,10 +5,10 @@ const { doWhilst, eachSeries, eachLimit, waterfall, } = require('async'); +const { ObjectMD } = require('arsenal').models; const werelogs = require('werelogs'); const BackbeatClient = require('./BackbeatClient'); -const { ObjectMD } = require('arsenal').models; const logLevel = Number.parseInt(process.env.DEBUG, 10) === 1 ? 'debug' : 'info'; @@ -27,12 +27,12 @@ let { STORAGE_TYPE, TARGET_REPLICATION_STATUS, } = process.env; const { TARGET_PREFIX } = process.env; -const WORKERS = (process.env.WORKERS && - Number.parseInt(process.env.WORKERS, 10)) || 10; -const MAX_UPDATES = (process.env.MAX_UPDATES && - Number.parseInt(process.env.MAX_UPDATES, 10)); -const MAX_SCANNED = (process.env.MAX_SCANNED && - Number.parseInt(process.env.MAX_SCANNED, 10)); +const WORKERS = (process.env.WORKERS + && Number.parseInt(process.env.WORKERS, 10)) || 10; +const MAX_UPDATES = (process.env.MAX_UPDATES + && Number.parseInt(process.env.MAX_UPDATES, 10)); +const MAX_SCANNED = (process.env.MAX_SCANNED + && Number.parseInt(process.env.MAX_SCANNED, 10)); let { KEY_MARKER } = process.env; let { VERSION_ID_MARKER } = process.env; @@ -106,7 +106,7 @@ const s3Options = { log.error('aws sdk request error', { error, retryCount }); // computed delay is not truly exponential, it is reset to minimum after // every 10 calls, with max delay of 15 seconds! - return AWS_SDK_REQUEST_DELAY_MS * Math.pow(2, retryCount % 10); + return AWS_SDK_REQUEST_DELAY_MS * 2 ** (retryCount % 10); }, }; const s3 = new AWS.S3(Object.assign(options, s3Options)); @@ -328,7 +328,8 @@ function triggerCRROnBucket(bucketName, cb) { KeyMarker = data.NextKeyMarker; return done(); }); - }), + }, + ), () => { if (nUpdated >= MAX_UPDATES || nProcessed >= MAX_SCANNED) { _logProgress(); From 4c41db02e6930ed031b82de40892e6546e3a3705 Mon Sep 17 00:00:00 2001 From: Jonathan Gramain Date: Tue, 4 Oct 2022 18:30:46 -0700 Subject: [PATCH 10/11] S3UTILS-110 BackbeatClient update Update BackbeatClient with latest schema from BackbeatAPI 8.4 branch --- BackbeatClient/backbeat-2017-07-01.api.json | 334 +++++++++++++++++++- 1 file changed, 318 insertions(+), 16 deletions(-) diff --git a/BackbeatClient/backbeat-2017-07-01.api.json b/BackbeatClient/backbeat-2017-07-01.api.json index d658a2a9..2d97f085 100644 --- a/BackbeatClient/backbeat-2017-07-01.api.json +++ b/BackbeatClient/backbeat-2017-07-01.api.json @@ -16,7 +16,7 @@ "PutData": { "http": { "method": "PUT", - "requestUri": "/_/backbeat/data/{Bucket}/{Key+}" + "requestUri": "/_/backbeat/data/{Bucket}/{Key+}?v2" }, "input": { "type": "structure", @@ -66,9 +66,27 @@ }, "dataStoreName": { "type": "string" + }, + "cryptoScheme": { + "type": "long" + }, + "cipheredDataKey": { + "type": "string" } } } + }, + "ServerSideEncryption": { + "location": "header", + "locationName": "x-amz-server-side-encryption" + }, + "SSECustomerAlgorithm": { + "location": "header", + "locationName": "x-amz-server-side-encryption-customer-algorithm" + }, + "SSEKMSKeyId": { + "location": "header", + "locationName": "x-amz-server-side-encryption-aws-kms-key-id" } }, "payload": "Location" @@ -140,6 +158,10 @@ "location": "header", "locationName": "X-Scal-Version-Id" }, + "Tags": { + "location": "header", + "locationName": "X-Scal-Tags" + }, "Body": { "streaming": true, "type": "blob" @@ -152,6 +174,12 @@ "members": { "versionId": { "type": "string" + }, + "location": { + "type": "list", + "member": { + "shape": "LocationMDObj" + } } } } @@ -200,6 +228,53 @@ } } }, + "MultipleBackendHeadObject": { + "http": { + "method": "GET", + "requestUri": "/_/backbeat/multiplebackendmetadata/{Bucket}/{Key+}" + }, + "input": { + "type": "structure", + "required": [ + "Bucket", + "Key", + "Locations" + ], + "members": { + "Bucket": { + "location": "uri", + "locationName": "Bucket" + }, + "Key": { + "location": "uri", + "locationName": "Key" + }, + "Locations": { + "location": "header", + "locationName": "X-Scal-Locations", + "type": "string", + "member": { + "type": "structure", + "required": [ + "key", + "dataStoreName" + ], + "member": { + "shape": "LocationMDObj" + } + } + } + } + }, + "output": { + "type": "structure", + "members": { + "lastModified": { + "type": "string" + } + } + } + }, "MultipleBackendPutMPUPart": { "http": { "method": "PUT", @@ -318,6 +393,10 @@ "location": "header", "locationName": "X-Scal-Content-Encoding" }, + "Tags": { + "location": "header", + "locationName": "X-Scal-Tags" + }, "Body": { "type": "blob" } @@ -333,6 +412,46 @@ } } }, + "MultipleBackendAbortMPU": { + "http": { + "method": "DELETE", + "requestUri": "/_/backbeat/multiplebackenddata/{Bucket}/{Key+}?operation=abortmpu" + }, + "input": { + "type": "structure", + "required": [ + "Bucket", + "Key", + "StorageClass" + ], + "members": { + "Bucket": { + "location": "uri", + "locationName": "Bucket" + }, + "Key": { + "location": "uri", + "locationName": "Key" + }, + "StorageType": { + "location": "header", + "locationName": "X-Scal-Storage-Type" + }, + "StorageClass": { + "location": "header", + "locationName": "X-Scal-Storage-Class" + }, + "UploadId": { + "location": "header", + "locationName": "X-Scal-Upload-Id" + } + } + }, + "output": { + "type": "structure", + "members": {} + } + }, "MultipleBackendCompleteMPU": { "http": { "method": "POST", @@ -390,6 +509,10 @@ "location": "header", "locationName": "X-Scal-Upload-Id" }, + "Tags": { + "location": "header", + "locationName": "X-Scal-Tags" + }, "Body": { "type": "blob" } @@ -401,6 +524,12 @@ "members": { "versionId": { "type": "string" + }, + "location": { + "type": "list", + "member": { + "shape": "LocationMDObj" + } } } } @@ -549,6 +678,12 @@ "location": "uri", "locationName": "Key" }, + "VersionId": { + "type": "string", + "documentation": "VersionId used to reference a specific version of the object.", + "location": "querystring", + "locationName": "versionId" + }, "ContentLength": { "location": "header", "locationName": "Content-Length", @@ -845,13 +980,34 @@ "BatchDelete": { "http": { "method": "POST", - "requestUri": "/_/backbeat/batchdelete" + "requestUri": "/_/backbeat/batchdelete/{Bucket}/{Key+}" }, "input": { "type": "structure", "required": [ ], "members": { + "Bucket": { + "location": "uri", + "locationName": "Bucket" + }, + "Key": { + "location": "uri", + "locationName": "Key" + }, + "IfUnmodifiedSince": { + "location": "header", + "locationName": "If-Unmodified-Since", + "type": "string" + }, + "StorageClass": { + "location": "header", + "locationName": "X-Scal-Storage-Class" + }, + "Tags": { + "location": "header", + "locationName": "X-Scal-Tags" + }, "ContentType": { "location": "header", "locationName": "X-Scal-Content-Type" @@ -873,6 +1029,9 @@ }, "size": { "type": "integer" + }, + "dataStoreVersionId": { + "type": "string" } } } @@ -888,7 +1047,7 @@ "GetRaftBuckets": { "http": { "method": "GET", - "requestUri": "/_/metadata/listbuckets/{LogId}" + "requestUri": "/_/metadata/admin/raft_sessions/{LogId}/bucket" }, "input": { "type": "structure", @@ -909,10 +1068,67 @@ } } }, + "GetRaftId": { + "http": { + "method": "GET", + "requestUri": "/_/metadata/admin/buckets/{Bucket}/id" + }, + "input": { + "type": "structure", + "required": [ + "Bucket" + ], + "members": { + "Bucket": { + "location": "uri", + "locationName": "Bucket" + } + } + }, + "output": { + "type": "string" + } + }, + "GetRaftLog": { + "http": { + "method": "GET", + "requestUri": "/_/metadata/admin/raft_sessions/{LogId}/log" + }, + "input": { + "type": "structure", + "required": [ + "LogId" + ], + "members": { + "LogId": { + "location": "uri", + "locationName": "LogId" + }, + "Begin": { + "type": "integer", + "location": "querystring", + "locationName": "begin" + }, + "Limit": { + "type": "integer", + "location": "querystring", + "locationName": "limit" + }, + "TargetLeader": { + "type": "boolean", + "location": "querystring", + "locationName": "targetLeader" + } + } + }, + "output": { + "shape": "RaftLogOutput" + } + }, "GetBucketMetadata": { "http": { "method": "GET", - "requestUri": "/_/metadata/getbucket/{Bucket}" + "requestUri": "/_/metadata/default/attributes/{Bucket}" }, "input": { "type": "structure", @@ -933,7 +1149,7 @@ "GetObjectList": { "http": { "method": "GET", - "requestUri": "/_/metadata/listobjects/{Bucket}" + "requestUri": "/_/metadata/default/bucket/{Bucket}" }, "input": { "type": "structure", @@ -951,32 +1167,33 @@ "shape": "ObjectMDListResponse" } }, - "GetObjectMetadata": { + "GetBucketCseq": { "http": { "method": "GET", - "requestUri": "/_/metadata/getobject/{Bucket}/{Key+}" + "requestUri": "/_/metadata/default/informations/{Bucket}" }, "input": { "type": "structure", "required": [ - "Bucket", - "Key" + "Bucket" ], "members": { "Bucket": { "location": "uri", "locationName": "Bucket" - }, - "Key": { - "location": "uri", - "locationName": "Key" } } }, "output": { - "type": "map", - "key": {}, - "value": {} + "type": "list", + "member": { + "type": "structure", + "members": { + "cseq": { + "type": "integer" + } + } + } } } }, @@ -999,6 +1216,18 @@ "members": { "Contents": { "shape": "ObjectMDList" + }, + "CommonPrefixes": { + "type": "list", + "members": { + "type": "string" + } + }, + "IsTruncated": { + "type": "boolean" + }, + "Delimiter": { + "type": "string" } } }, @@ -1198,6 +1427,79 @@ } } } + }, + "LocationMDObj": { + "type": "structure", + "members": { + "key": { + "type": "string" + }, + "size": { + "type": "integer" + }, + "start": { + "type": "integer" + }, + "dataStoreName": { + "type": "string" + }, + "dataStoreType": { + "type": "string" + }, + "dataStoreETag": { + "type": "string" + }, + "dataStoreVersionId": { + "type": "string" + } + } + }, + "RaftLogOutput": { + "type": "structure", + "members": { + "info": { + "type": "structure", + "members": { + "start": { + "type": "integer" + }, + "cseq": { + "type": "integer" + }, + "prune": { + "type": "integer" + } + } + }, + "log": { + "type": "list", + "member": { + "type": "structure", + "members": { + "db": { + "type": "string" + }, + "entries": { + "type": "list", + "member": { + "type": "structure", + "members": { + "key": { + "type": "string" + }, + "value": { + "type": "string" + } + } + } + }, + "method": { + "type": "integer" + } + } + } + } + } } } } From a6fac041cf1d3e2f0f8bff7db5152c733167c39e Mon Sep 17 00:00:00 2001 From: Jonathan Gramain Date: Tue, 4 Oct 2022 18:39:09 -0700 Subject: [PATCH 11/11] bugfix: S3UTILS-110 pass VersionId to PutMetadata route Make sure the `PUT /_/backbeat/metadata` route gets the VersionId attribute, so that it is not confused in thinking we are attempting to update the master version. --- crrExistingObjects.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crrExistingObjects.js b/crrExistingObjects.js index e7f072cd..2c10941d 100644 --- a/crrExistingObjects.js +++ b/crrExistingObjects.js @@ -187,6 +187,7 @@ function _markObjectPending( return bb.putMetadata({ Bucket: bucket, Key: key, + VersionId: versionId, ContentLength: Buffer.byteLength(mdBlob), Body: mdBlob, }, (err, putRes) => { @@ -237,6 +238,7 @@ function _markObjectPending( return bb.putMetadata({ Bucket: bucket, Key: key, + VersionId: versionId, ContentLength: Buffer.byteLength(mdBlob), Body: mdBlob, }, next);