From 57fb5f14038c6bef804f7b21d25055230508a8f6 Mon Sep 17 00:00:00 2001 From: gaspardmoindrot Date: Wed, 24 May 2023 22:35:45 +0000 Subject: [PATCH 01/14] =?UTF-8?q?[CLDSRV-388]=20=E2=9C=A8=20Implement=20GH?= =?UTF-8?q?AS?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/codeql.yaml | 25 ++++++++++++++++++++++++ .github/workflows/dependency-review.yaml | 16 +++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 .github/workflows/codeql.yaml create mode 100644 .github/workflows/dependency-review.yaml diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql.yaml new file mode 100644 index 0000000000..9a2fc7d704 --- /dev/null +++ b/.github/workflows/codeql.yaml @@ -0,0 +1,25 @@ +--- +name: codeQL + +on: + push: + branches: [development/*, stabilization/*, hotfix/*] + pull_request: + branches: [development/*, stabilization/*, hotfix/*] + workflow_dispatch: + +jobs: + analyze: + name: Static analysis with CodeQL + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: javascript, python, ruby + + - name: Build and analyze + uses: github/codeql-action/analyze@v2 diff --git a/.github/workflows/dependency-review.yaml b/.github/workflows/dependency-review.yaml new file mode 100644 index 0000000000..4716cdfd1f --- /dev/null +++ b/.github/workflows/dependency-review.yaml @@ -0,0 +1,16 @@ +--- +name: dependency review + +on: + pull_request: + branches: [development/*, stabilization/*, hotfix/*] + +jobs: + dependency-review: + runs-on: ubuntu-latest + steps: + - name: 'Checkout Repository' + uses: actions/checkout@v3 + + - name: 'Dependency Review' + uses: actions/dependency-review-action@v3 From f889100798f9394b230f06f93a6b9ba10b0471a9 Mon Sep 17 00:00:00 2001 From: Nicolas Humbert Date: Thu, 25 May 2023 17:37:19 -0400 Subject: [PATCH 02/14] CLDSRV-396 putMetadata API route is not updating null version properly --- lib/routes/routeBackbeat.js | 57 +- .../raw-node/test/routes/routeBackbeat.js | 1002 ++++++++++++++++- 2 files changed, 1049 insertions(+), 10 deletions(-) diff --git a/lib/routes/routeBackbeat.js b/lib/routes/routeBackbeat.js index 7b2c4b0924..2822e5f2bc 100644 --- a/lib/routes/routeBackbeat.js +++ b/lib/routes/routeBackbeat.js @@ -472,22 +472,61 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) { omVal[headerName] = objMd[headerName]; }); } - const versionId = decodeVersionId(request.query); - // specify both 'versioning' and 'versionId' to create a "new" - // version (updating master as well) but with specified - // versionId - const options = { - versioning: bucketInfo.isVersioningEnabled(), - versionId, - }; + + let versionId = decodeVersionId(request.query); + let versioning = bucketInfo.isVersioningEnabled(); + + if (versionId === 'null') { + // Retrieve the null version id from the object metadata. + versionId = objMd && objMd.versionId; + if (!versionId) { + if (versioning) { + // If the null version does not have a version id, it is a current null version. + // To update the metadata of a current version, versioning is set to false. + + // This condition is to handle the case where a single null version looks like a master + // key and will not have a duplicate versioned key and no version ID. + // They are created when you have a non-versioned bucket with objects, + // and then convert bucket to versioned. + // If no new versioned objects are added for given object(s), they look like + // standalone master keys. + versioning = false; + } else { + const versioningConf = bucketInfo.getVersioningConfiguration(); + // The purpose of this condition is to address situations in which + // - versioning is "suspended" and + // - no existing object or no null version. + // In such scenarios, we generate a new null version and designate it as the master version. + if (versioningConf && versioningConf.Status === 'Suspended') { + versionId = ''; + omVal.isNull = true; + } + } + } + } + // If the object is from a source bucket without versioning (i.e. NFS), // then we want to create a version for the replica object even though // none was provided in the object metadata value. if (omVal.replicationInfo.isNFS) { const isReplica = omVal.replicationInfo.status === 'REPLICA'; - options.versioning = isReplica; + versioning = isReplica; omVal.replicationInfo.isNFS = !isReplica; } + + const options = { + versionId, + }; + + // NOTE: When 'versioning' is set to true and no 'versionId' is specified, + // it results in the creation of a "new" version, which also updates the master. + // NOTE: Since option fields are converted to strings when they're sent to Metadata via the query string, + // Metadata interprets the value "false" as if it were true. + // Therefore, to avoid this confusion, we don't pass the versioning parameter at all if its value is false. + if (versioning) { + options.versioning = true; + } + log.trace('putting object version', { objectKey: request.objectKey, omVal, options }); return metadata.putObjectMD(bucketName, objectKey, omVal, options, log, diff --git a/tests/functional/raw-node/test/routes/routeBackbeat.js b/tests/functional/raw-node/test/routes/routeBackbeat.js index 294f81efa8..3133c98eed 100644 --- a/tests/functional/raw-node/test/routes/routeBackbeat.js +++ b/tests/functional/raw-node/test/routes/routeBackbeat.js @@ -1,8 +1,9 @@ const assert = require('assert'); const async = require('async'); const crypto = require('crypto'); -const { versioning } = require('arsenal'); +const { models, versioning } = require('arsenal'); const versionIdUtils = versioning.VersionID; +const { ObjectMD } = models; const { makeRequest } = require('../../utils/makeRequest'); const BucketUtility = require('../../../aws-node-sdk/lib/utility/bucket-util'); @@ -19,6 +20,7 @@ const TEST_BUCKET = 'backbeatbucket'; const TEST_ENCRYPTED_BUCKET = 'backbeatbucket-encrypted'; const TEST_KEY = 'fookey'; const NONVERSIONED_BUCKET = 'backbeatbucket-non-versioned'; +const BUCKET_FOR_NULL_VERSION = 'backbeatbucket-null-version'; const testArn = 'aws::iam:123456789012:user/bart'; const testKey = 'testkey'; @@ -72,6 +74,18 @@ function checkObjectData(s3, objectKey, dataValue, done) { }); } +function checkVersionData(s3, bucket, objectKey, versionId, dataValue, done) { + return s3.getObject({ + Bucket: bucket, + Key: objectKey, + VersionId: versionId, + }, (err, data) => { + assert.ifError(err); + assert.strictEqual(data.Body.toString(), dataValue); + return done(); + }); +} + /** makeBackbeatRequest - utility function to generate a request going * through backbeat route * @param {object} params - params for making request @@ -105,6 +119,21 @@ function makeBackbeatRequest(params, callback) { makeRequest(options, callback); } +function updateStorageClass(data, storageClass) { + let parsedBody; + try { + parsedBody = JSON.parse(data.body); + } catch (err) { + return { error: err }; + } + const { result, error } = ObjectMD.createFromBlob(parsedBody.Body); + if (error) { + return { error }; + } + result.setAmzStorageClass(storageClass); + return { result }; +} + function getMetadataToPut(putDataResponse) { const mdToPut = Object.assign({}, testMd); // Reproduce what backbeat does to update target metadata @@ -171,6 +200,977 @@ describeSkipIfAWS('backbeat routes', () => { .then(() => done()); }); + describe('null version', () => { + const bucket = BUCKET_FOR_NULL_VERSION; + const keyName = 'key0'; + const storageClass = 'foo'; + + function assertVersionIsNullAndUpdated(version) { + const { Key, VersionId, StorageClass } = version; + assert.strictEqual(Key, keyName); + assert.strictEqual(VersionId, 'null'); + assert.strictEqual(StorageClass, storageClass); + } + + function assertVersionHasNotBeenUpdated(version, expectedVersionId) { + const { Key, VersionId, StorageClass } = version; + assert.strictEqual(Key, keyName); + assert.strictEqual(VersionId, expectedVersionId); + assert.strictEqual(StorageClass, 'STANDARD'); + } + + beforeEach(done => s3.createBucket({ Bucket: BUCKET_FOR_NULL_VERSION }, done)); + afterEach(done => { + bucketUtil.empty(BUCKET_FOR_NULL_VERSION) + .then(() => s3.deleteBucket({ Bucket: BUCKET_FOR_NULL_VERSION }).promise()) + .then(() => done()); + }); + + it('should update metadata of a current null version', done => { + let objMD; + return async.series([ + next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next), + next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } }, + next), + next => makeBackbeatRequest({ + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + }, (err, data) => { + if (err) { + return next(err); + } + const { error, result } = updateStorageClass(data, storageClass); + if (error) { + return next(error); + } + objMD = result; + return next(); + }), + next => makeBackbeatRequest({ + method: 'PUT', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + requestBody: objMD.getSerialized(), + }, next), + next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next), + next => s3.listObjectVersions({ Bucket: bucket }, next), + ], (err, data) => { + if (err) { + return done(err); + } + const headObjectRes = data[4]; + assert.strictEqual(headObjectRes.VersionId, 'null'); + assert.strictEqual(headObjectRes.StorageClass, storageClass); + + const listObjectVersionsRes = data[5]; + const { Versions } = listObjectVersionsRes; + + assert.strictEqual(Versions.length, 1); + + const [currentVersion] = Versions; + assertVersionIsNullAndUpdated(currentVersion); + return done(); + }); + }); + + it('should update metadata of a non-current null version', done => { + let objMD; + let expectedVersionId; + return async.series([ + next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next), + next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } }, + next), + next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, (err, data) => { + if (err) { + return next(err); + } + expectedVersionId = data.VersionId; + return next(); + }), + next => makeBackbeatRequest({ + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + }, (err, data) => { + if (err) { + return next(err); + } + const { error, result } = updateStorageClass(data, storageClass); + if (error) { + return next(error); + } + objMD = result; + return next(); + }), + next => makeBackbeatRequest({ + method: 'PUT', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + requestBody: objMD.getSerialized(), + }, next), + next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next), + next => s3.listObjectVersions({ Bucket: bucket }, next), + ], (err, data) => { + if (err) { + return done(err); + } + const headObjectRes = data[5]; + assert.strictEqual(headObjectRes.VersionId, 'null'); + assert.strictEqual(headObjectRes.StorageClass, storageClass); + + const listObjectVersionsRes = data[6]; + const { Versions } = listObjectVersionsRes; + + assert.strictEqual(Versions.length, 2); + + const currentVersion = Versions.find(v => v.IsLatest); + assertVersionHasNotBeenUpdated(currentVersion, expectedVersionId); + + const nonCurrentVersion = Versions.find(v => !v.IsLatest); + assertVersionIsNullAndUpdated(nonCurrentVersion); + return done(); + }); + }); + + // Skipping is necessary because non-versioned buckets are not supported by S3C backbeat routes. + it.skip('should update metadata of a non-version object', done => { + let objMD; + return async.series([ + next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next), + next => makeBackbeatRequest({ + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + }, (err, data) => { + if (err) { + return next(err); + } + const { error, result } = updateStorageClass(data, storageClass); + if (error) { + return next(error); + } + objMD = result; + return next(); + }), + next => makeBackbeatRequest({ + method: 'PUT', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + requestBody: objMD.getSerialized(), + }, next), + next => s3.headObject({ Bucket: bucket, Key: keyName }, next), + next => s3.listObjectVersions({ Bucket: bucket }, next), + ], (err, data) => { + if (err) { + return done(err); + } + + const headObjectRes = data[3]; + assert(!headObjectRes.VersionId); + assert.strictEqual(headObjectRes.StorageClass, storageClass); + + const listObjectVersionsRes = data[4]; + const { DeleteMarkers, Versions } = listObjectVersionsRes; + + assert.strictEqual(DeleteMarkers.length, 0); + assert.strictEqual(Versions.length, 1); + + const currentVersion = Versions[0]; + assert(currentVersion.IsLatest); + assertVersionIsNullAndUpdated(currentVersion); + return done(); + }); + }); + + it.skip('should create a new null version if versioning suspended and no version', done => { + let objMD; + return async.series([ + next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Suspended' } }, + next), + next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next), + next => makeBackbeatRequest({ + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + }, (err, data) => { + if (err) { + return next(err); + } + const { error, result } = updateStorageClass(data, storageClass); + if (error) { + return next(error); + } + objMD = result; + return next(); + }), + next => s3.deleteObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next), + next => makeBackbeatRequest({ + method: 'PUT', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + requestBody: objMD.getSerialized(), + }, next), + next => s3.headObject({ Bucket: bucket, Key: keyName }, next), + next => s3.listObjectVersions({ Bucket: bucket }, next), + ], (err, data) => { + if (err) { + return done(err); + } + const headObjectRes = data[5]; + assert.strictEqual(headObjectRes.VersionId, 'null'); + assert.strictEqual(headObjectRes.StorageClass, storageClass); + + const listObjectVersionsRes = data[6]; + const { DeleteMarkers, Versions } = listObjectVersionsRes; + + assert.strictEqual(DeleteMarkers.length, 0); + assert.strictEqual(Versions.length, 1); + + const currentVersion = Versions[0]; + assert(currentVersion.IsLatest); + + assertVersionIsNullAndUpdated(currentVersion); + + return done(); + }); + }); + + it.skip('should create a new null version if versioning suspended and delete marker null version', done => { + let objMD; + return async.series([ + next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Suspended' } }, + next), + next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next), + next => makeBackbeatRequest({ + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + }, (err, data) => { + if (err) { + return next(err); + } + const { error, result } = updateStorageClass(data, storageClass); + if (error) { + return next(error); + } + objMD = result; + return next(); + }), + next => s3.deleteObject({ Bucket: bucket, Key: keyName }, next), + next => makeBackbeatRequest({ + method: 'PUT', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + requestBody: objMD.getSerialized(), + }, next), + next => s3.headObject({ Bucket: bucket, Key: keyName }, next), + next => s3.listObjectVersions({ Bucket: bucket }, next), + ], (err, data) => { + if (err) { + return done(err); + } + const headObjectRes = data[5]; + assert.strictEqual(headObjectRes.VersionId, 'null'); + assert.strictEqual(headObjectRes.StorageClass, storageClass); + + const listObjectVersionsRes = data[6]; + const { DeleteMarkers, Versions } = listObjectVersionsRes; + + assert.strictEqual(DeleteMarkers.length, 0); + assert.strictEqual(Versions.length, 1); + + const currentVersion = Versions[0]; + assert(currentVersion.IsLatest); + assertVersionIsNullAndUpdated(currentVersion); + return done(); + }); + }); + + it.skip('should create a new null version if versioning suspended and version has version id', done => { + let expectedVersionId; + let objMD; + return async.series([ + next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } }, + next), + next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, (err, data) => { + if (err) { + return next(err); + } + expectedVersionId = data.VersionId; + return next(); + }), + next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Suspended' } }, + next), + next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next), + next => makeBackbeatRequest({ + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: null, + }, + authCredentials: backbeatAuthCredentials, + }, (err, data) => { + if (err) { + return next(err); + } + const { error, result } = updateStorageClass(data, storageClass); + if (error) { + return next(error); + } + objMD = result; + return next(); + }), + next => s3.deleteObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next), + next => makeBackbeatRequest({ + method: 'PUT', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + requestBody: objMD.getSerialized(), + }, next), + next => s3.headObject({ Bucket: bucket, Key: keyName }, next), + next => s3.listObjectVersions({ Bucket: bucket }, next), + ], (err, data) => { + if (err) { + return done(err); + } + const headObjectRes = data[7]; + assert.strictEqual(headObjectRes.VersionId, 'null'); + assert.strictEqual(headObjectRes.StorageClass, storageClass); + + const listObjectVersionsRes = data[8]; + const { DeleteMarkers, Versions } = listObjectVersionsRes; + + assert.strictEqual(DeleteMarkers.length, 0); + assert.strictEqual(Versions.length, 2); + + const currentVersion = Versions.find(v => v.IsLatest); + assertVersionIsNullAndUpdated(currentVersion); + + const nonCurrentVersion = Versions.find(v => !v.IsLatest); + assertVersionHasNotBeenUpdated(nonCurrentVersion, expectedVersionId); + + // give some time for the async deletes to complete + return setTimeout(() => checkVersionData(s3, bucket, keyName, expectedVersionId, testData, done), + 1000); + }); + }); + + it.skip('should update null version with no version id and versioning suspended', done => { + let objMD; + return async.series([ + next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next), + next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Suspended' } }, + next), + next => makeBackbeatRequest({ + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + }, (err, data) => { + if (err) { + return next(err); + } + const { error, result } = updateStorageClass(data, storageClass); + if (error) { + return next(error); + } + objMD = result; + return next(); + }), + next => makeBackbeatRequest({ + method: 'PUT', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + requestBody: objMD.getSerialized(), + }, next), + next => s3.headObject({ Bucket: bucket, Key: keyName }, next), + next => s3.listObjectVersions({ Bucket: bucket }, next), + ], (err, data) => { + if (err) { + return done(err); + } + const headObjectRes = data[4]; + assert.strictEqual(headObjectRes.VersionId, 'null'); + assert.strictEqual(headObjectRes.StorageClass, storageClass); + + const listObjectVersionsRes = data[5]; + const { DeleteMarkers, Versions } = listObjectVersionsRes; + assert.strictEqual(DeleteMarkers.length, 0); + assert.strictEqual(Versions.length, 1); + + const currentVersion = Versions[0]; + assert(currentVersion.IsLatest); + assertVersionIsNullAndUpdated(currentVersion); + + return done(); + }); + }); + + it.skip('should update null version if versioning suspended and null version has a version id', done => { + let objMD; + return async.series([ + next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Suspended' } }, + next), + next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next), + next => makeBackbeatRequest({ + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + }, (err, data) => { + if (err) { + return next(err); + } + const { error, result } = updateStorageClass(data, storageClass); + if (error) { + return next(error); + } + objMD = result; + return next(); + }), + next => makeBackbeatRequest({ + method: 'PUT', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + requestBody: objMD.getSerialized(), + }, next), + next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next), + next => s3.listObjectVersions({ Bucket: bucket }, next), + ], (err, data) => { + if (err) { + return done(err); + } + + const headObjectRes = data[4]; + assert.strictEqual(headObjectRes.VersionId, 'null'); + assert.strictEqual(headObjectRes.StorageClass, storageClass); + + const listObjectVersionsRes = data[5]; + const { DeleteMarkers, Versions } = listObjectVersionsRes; + assert.strictEqual(Versions.length, 1); + assert.strictEqual(DeleteMarkers.length, 0); + + const currentVersion = Versions[0]; + assert(currentVersion.IsLatest); + assertVersionIsNullAndUpdated(currentVersion); + return done(); + }); + }); + + it.skip('should update null version if versioning suspended and null version has a version id and' + + 'put object afterward', done => { + let objMD; + return async.series([ + next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Suspended' } }, + next), + next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next), + next => makeBackbeatRequest({ + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + }, (err, data) => { + if (err) { + return next(err); + } + const { error, result } = updateStorageClass(data, storageClass); + if (error) { + return next(error); + } + objMD = result; + return next(); + }), + next => makeBackbeatRequest({ + method: 'PUT', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + requestBody: objMD.getSerialized(), + }, next), + next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next), + next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next), + next => s3.listObjectVersions({ Bucket: bucket }, next), + ], (err, data) => { + if (err) { + return done(err); + } + + const headObjectRes = data[5]; + assert.strictEqual(headObjectRes.VersionId, 'null'); + assert(!headObjectRes.StorageClass); + + const listObjectVersionsRes = data[6]; + const { DeleteMarkers, Versions } = listObjectVersionsRes; + assert.strictEqual(DeleteMarkers.length, 0); + assert.strictEqual(Versions.length, 1); + + const currentVersion = Versions[0]; + assert(currentVersion.IsLatest); + assertVersionHasNotBeenUpdated(currentVersion, 'null'); + return done(); + }); + }); + + it.skip('should update null version if versioning suspended and null version has a version id and' + + 'put version afterward', done => { + let objMD; + let expectedVersionId; + return async.series([ + next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Suspended' } }, + next), + next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next), + next => makeBackbeatRequest({ + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + }, (err, data) => { + if (err) { + return next(err); + } + const { error, result } = updateStorageClass(data, storageClass); + if (error) { + return next(error); + } + objMD = result; + return next(); + }), + next => makeBackbeatRequest({ + method: 'PUT', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + requestBody: objMD.getSerialized(), + }, next), + next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } }, + next), + next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, (err, data) => { + if (err) { + return next(err); + } + expectedVersionId = data.VersionId; + return next(); + }), + next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next), + next => s3.listObjectVersions({ Bucket: bucket }, next), + ], (err, data) => { + if (err) { + return done(err); + } + + const headObjectRes = data[6]; + assert.strictEqual(headObjectRes.VersionId, 'null'); + assert.strictEqual(headObjectRes.StorageClass, storageClass); + + const listObjectVersionsRes = data[7]; + const { Versions } = listObjectVersionsRes; + assert.strictEqual(Versions.length, 2); + + const [currentVersion] = Versions.filter(v => v.IsLatest); + assertVersionHasNotBeenUpdated(currentVersion, expectedVersionId); + + const [nonCurrentVersion] = Versions.filter(v => !v.IsLatest); + assertVersionIsNullAndUpdated(nonCurrentVersion); + return done(); + }); + }); + + it.skip('should update non-current null version if versioning suspended', done => { + let expectedVersionId; + let objMD; + return async.series([ + next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next), + next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } }, + next), + next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, (err, data) => { + if (err) { + return next(err); + } + expectedVersionId = data.VersionId; + return next(); + }), + next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Suspended' } }, + next), + next => makeBackbeatRequest({ + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + }, (err, data) => { + if (err) { + return next(err); + } + const { error, result } = updateStorageClass(data, storageClass); + if (error) { + return next(error); + } + objMD = result; + return next(); + }), + next => makeBackbeatRequest({ + method: 'PUT', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + requestBody: objMD.getSerialized(), + }, next), + next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next), + next => s3.listObjectVersions({ Bucket: bucket }, next), + ], (err, data) => { + if (err) { + return done(err); + } + + const headObjectRes = data[6]; + assert.strictEqual(headObjectRes.VersionId, 'null'); + assert.strictEqual(headObjectRes.StorageClass, storageClass); + + const listObjectVersionsRes = data[7]; + const deleteMarkers = listObjectVersionsRes.DeleteMarkers; + assert.strictEqual(deleteMarkers.length, 0); + const { Versions } = listObjectVersionsRes; + assert.strictEqual(Versions.length, 2); + + const [currentVersion] = Versions.filter(v => v.IsLatest); + assertVersionHasNotBeenUpdated(currentVersion, expectedVersionId); + + const [nonCurrentVersion] = Versions.filter(v => !v.IsLatest); + assertVersionIsNullAndUpdated(nonCurrentVersion); + + return done(); + }); + }); + + it.skip('should update current null version if versioning suspended', done => { + let objMD; + let expectedVersionId; + return async.series([ + next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next), + next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } }, + next), + next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, (err, data) => { + if (err) { + return next(err); + } + expectedVersionId = data.VersionId; + return next(); + }), + next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Suspended' } }, + next), + next => s3.deleteObject({ Bucket: bucket, Key: keyName, VersionId: expectedVersionId }, next), + next => makeBackbeatRequest({ + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + }, (err, data) => { + if (err) { + return next(err); + } + const { error, result } = updateStorageClass(data, storageClass); + if (error) { + return next(error); + } + objMD = result; + return next(); + }), + next => makeBackbeatRequest({ + method: 'PUT', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + requestBody: objMD.getSerialized(), + }, next), + next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next), + next => s3.listObjectVersions({ Bucket: bucket }, next), + ], (err, data) => { + if (err) { + return done(err); + } + + const headObjectRes = data[7]; + assert.strictEqual(headObjectRes.VersionId, 'null'); + assert.strictEqual(headObjectRes.StorageClass, storageClass); + + const listObjectVersionsRes = data[8]; + const { DeleteMarkers, Versions } = listObjectVersionsRes; + assert.strictEqual(Versions.length, 1); + assert.strictEqual(DeleteMarkers.length, 0); + + const currentVersion = Versions[0]; + assert(currentVersion.IsLatest); + assertVersionIsNullAndUpdated(currentVersion); + return done(); + }); + }); + + it.skip('should update current null version if versioning suspended and put a null version ' + + 'afterwards', done => { + let objMD; + let deletedVersionId; + return async.series([ + next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next), + next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } }, + next), + next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, (err, data) => { + if (err) { + return next(err); + } + deletedVersionId = data.VersionId; + return next(); + }), + next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Suspended' } }, + next), + next => s3.deleteObject({ Bucket: bucket, Key: keyName, VersionId: deletedVersionId }, next), + next => makeBackbeatRequest({ + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + }, (err, data) => { + if (err) { + return next(err); + } + const { error, result } = updateStorageClass(data, storageClass); + if (error) { + return next(error); + } + objMD = result; + return next(); + }), + next => makeBackbeatRequest({ + method: 'PUT', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + requestBody: objMD.getSerialized(), + }, next), + next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next), + next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next), + next => s3.listObjectVersions({ Bucket: bucket }, next), + ], (err, data) => { + if (err) { + return done(err); + } + + const headObjectRes = data[8]; + assert.strictEqual(headObjectRes.VersionId, 'null'); + assert(!headObjectRes.StorageClass); + + const listObjectVersionsRes = data[9]; + const { DeleteMarkers, Versions } = listObjectVersionsRes; + assert.strictEqual(DeleteMarkers.length, 0); + assert.strictEqual(Versions.length, 1); + + const currentVersion = Versions[0]; + assert(currentVersion.IsLatest); + assertVersionHasNotBeenUpdated(currentVersion, 'null'); + + return done(); + }); + }); + + it.skip('should update current null version if versioning suspended and put a version afterwards', done => { + let objMD; + let deletedVersionId; + let expectedVersionId; + return async.series([ + next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, next), + next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } }, + next), + next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, (err, data) => { + if (err) { + return next(err); + } + deletedVersionId = data.VersionId; + return next(); + }), + next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Suspended' } }, + next), + next => s3.deleteObject({ Bucket: bucket, Key: keyName, VersionId: deletedVersionId }, next), + next => makeBackbeatRequest({ + method: 'GET', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + }, (err, data) => { + if (err) { + return next(err); + } + const { error, result } = updateStorageClass(data, storageClass); + if (error) { + return next(error); + } + objMD = result; + return next(); + }), + next => makeBackbeatRequest({ + method: 'PUT', + resourceType: 'metadata', + bucket, + objectKey: keyName, + queryObj: { + versionId: 'null', + }, + authCredentials: backbeatAuthCredentials, + requestBody: objMD.getSerialized(), + }, next), + next => s3.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { Status: 'Enabled' } }, + next), + next => s3.putObject({ Bucket: bucket, Key: keyName, Body: new Buffer(testData) }, (err, data) => { + if (err) { + return next(err); + } + expectedVersionId = data.VersionId; + return next(); + }), + next => s3.headObject({ Bucket: bucket, Key: keyName, VersionId: 'null' }, next), + next => s3.listObjectVersions({ Bucket: bucket }, next), + ], (err, data) => { + if (err) { + return done(err); + } + + const headObjectRes = data[9]; + assert.strictEqual(headObjectRes.VersionId, 'null'); + assert.strictEqual(headObjectRes.StorageClass, storageClass); + + const listObjectVersionsRes = data[10]; + const { DeleteMarkers, Versions } = listObjectVersionsRes; + assert.strictEqual(DeleteMarkers.length, 0); + assert.strictEqual(Versions.length, 2); + + const [currentVersion] = Versions.filter(v => v.IsLatest); + assertVersionHasNotBeenUpdated(currentVersion, expectedVersionId); + + const [nonCurrentVersion] = Versions.filter(v => !v.IsLatest); + assertVersionIsNullAndUpdated(nonCurrentVersion); + + return done(); + }); + }); + }); + describe('backbeat PUT routes', () => { describe('PUT data + metadata should create a new complete object', () => { From 1ed32b2cae3c80d99619a8d7303edb870e1e2e46 Mon Sep 17 00:00:00 2001 From: Nicolas Humbert Date: Fri, 2 Jun 2023 11:55:35 -0400 Subject: [PATCH 03/14] CLDSRV-396 If put metadata for a null version, set options.isNull to true --- lib/routes/routeBackbeat.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/routes/routeBackbeat.js b/lib/routes/routeBackbeat.js index 2822e5f2bc..565d12e31a 100644 --- a/lib/routes/routeBackbeat.js +++ b/lib/routes/routeBackbeat.js @@ -475,8 +475,10 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) { let versionId = decodeVersionId(request.query); let versioning = bucketInfo.isVersioningEnabled(); + let isNull = false; if (versionId === 'null') { + isNull = true; // Retrieve the null version id from the object metadata. versionId = objMd && objMd.versionId; if (!versionId) { @@ -516,6 +518,7 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) { const options = { versionId, + isNull, }; // NOTE: When 'versioning' is set to true and no 'versionId' is specified, From 27cacc9552db569b31f37fa29ce32e697e9f3f8e Mon Sep 17 00:00:00 2001 From: Nicolas Humbert Date: Fri, 2 Jun 2023 14:14:05 -0400 Subject: [PATCH 04/14] CLDSRV-396 add nullVersionCompatMode condition --- lib/routes/routeBackbeat.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/routes/routeBackbeat.js b/lib/routes/routeBackbeat.js index 565d12e31a..d017007720 100644 --- a/lib/routes/routeBackbeat.js +++ b/lib/routes/routeBackbeat.js @@ -478,7 +478,9 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) { let isNull = false; if (versionId === 'null') { - isNull = true; + if (!config.nullVersionCompatMode) { + isNull = true; + } // Retrieve the null version id from the object metadata. versionId = objMd && objMd.versionId; if (!versionId) { From 0fa264693db11652430e162ece4f0f0cdcb879a8 Mon Sep 17 00:00:00 2001 From: Nicolas Humbert Date: Fri, 2 Jun 2023 15:05:00 -0400 Subject: [PATCH 05/14] CLDSRV-398 bump version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 1520f14580..f56d7be7f6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "s3", - "version": "7.10.27", + "version": "7.10.28", "description": "S3 connector", "main": "index.js", "engines": { From 31d1734d5c66b4794a6e9d4074dad46810cbd992 Mon Sep 17 00:00:00 2001 From: williamlardier Date: Thu, 22 Jun 2023 13:52:14 +0200 Subject: [PATCH 06/14] CLDSRV-409: use latest s3cmd with python3 --- .github/workflows/tests.yaml | 7 ++---- tests/functional/s3cmd/tests.js | 42 +++++++++++++++++---------------- 2 files changed, 24 insertions(+), 25 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 9387ad0a7f..532796b3a4 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -222,13 +222,10 @@ jobs: 3.9 - name: Setup CI environment uses: ./.github/actions/setup-ci - - name: Setup python2 test environment + - name: Setup python test environment run: | sudo apt-get install -y libdigest-hmac-perl - pip install virtualenv==20.21.0 - virtualenv -p $(which python2) ~/.virtualenv/py2 - source ~/.virtualenv/py2/bin/activate - pip install 's3cmd==1.6.1' + pip install 's3cmd==2.3.0' - name: Setup CI services run: docker-compose up -d working-directory: .github/docker diff --git a/tests/functional/s3cmd/tests.js b/tests/functional/s3cmd/tests.js index a0c09feade..a1bff602bc 100644 --- a/tests/functional/s3cmd/tests.js +++ b/tests/functional/s3cmd/tests.js @@ -165,7 +165,9 @@ function readJsonFromChild(child, lineFinder, cb) { const findBrace = data.indexOf('{', findLine); const findEnd = findEndJson(data, findBrace); const endJson = data.substring(findBrace, findEnd + 1) - .replace(/"/g, '\\"').replace(/'/g, '"'); + .replace(/"/g, '\\"').replace(/'/g, '"') + .replace(/b'/g, '\'') + .replace(/b"/g, '"'); return cb(JSON.parse(endJson)); }); } @@ -262,7 +264,7 @@ describe('s3cmd putBucket', () => { exec([ 'mb', `s3://${bucket}`, '--bucket-location=scality-us-west-1', - ], done, 13); + ], done, 11); }); it('put an invalid bucket, should fail', done => { @@ -344,18 +346,18 @@ describe('s3cmd getService', () => { it("should have response headers matching AWS's response headers", done => { - provideLineOfInterest(['ls', '--debug'], 'DEBUG: Response: {', + provideLineOfInterest(['ls', '--debug'], '\'headers\': {', parsedObject => { - assert(parsedObject.headers['x-amz-id-2']); - assert(parsedObject.headers['transfer-encoding']); - assert(parsedObject.headers['x-amz-request-id']); - const gmtDate = new Date(parsedObject.headers.date) + assert(parsedObject['x-amz-id-2']); + assert(parsedObject['transfer-encoding']); + assert(parsedObject['x-amz-request-id']); + const gmtDate = new Date(parsedObject.date) .toUTCString(); - assert.strictEqual(parsedObject.headers.date, gmtDate); + assert.strictEqual(parsedObject.date, gmtDate); assert.strictEqual(parsedObject - .headers['content-type'], 'application/xml'); + ['content-type'], 'application/xml'); assert.strictEqual(parsedObject - .headers['set-cookie'], undefined); + ['set-cookie'], undefined); done(); }); }); @@ -395,11 +397,11 @@ describe('s3cmd getObject', function toto() { }); it('get non existing file in existing bucket, should fail', done => { - exec(['get', `s3://${bucket}/${nonexist}`, 'fail'], done, 12); + exec(['get', `s3://${bucket}/${nonexist}`, 'fail'], done, 64); }); it('get file in non existing bucket, should fail', done => { - exec(['get', `s3://${nonexist}/${nonexist}`, 'fail2'], done, 12); + exec(['get', `s3://${nonexist}/${nonexist}`, 'fail2'], done, 64); }); }); @@ -511,7 +513,7 @@ describe('s3cmd delObject', () => { it('delete an already deleted object, should return a 204', done => { provideLineOfInterest(['rm', `s3://${bucket}/${upload}`, '--debug'], - 'DEBUG: Response: {', parsedObject => { + 'DEBUG: Response:\n{', parsedObject => { assert.strictEqual(parsedObject.status, 204); done(); }); @@ -519,14 +521,14 @@ describe('s3cmd delObject', () => { it('delete non-existing object, should return a 204', done => { provideLineOfInterest(['rm', `s3://${bucket}/${nonexist}`, '--debug'], - 'DEBUG: Response: {', parsedObject => { + 'DEBUG: Response:\n{', parsedObject => { assert.strictEqual(parsedObject.status, 204); done(); }); }); it('try to get the deleted object, should fail', done => { - exec(['get', `s3://${bucket}/${upload}`, download], done, 12); + exec(['get', `s3://${bucket}/${upload}`, download], done, 64); }); }); @@ -621,7 +623,7 @@ describe('s3cmd multipart upload', function titi() { }); it('should not be able to get deleted object', done => { - exec(['get', `s3://${bucket}/${MPUpload}`, download], done, 12); + exec(['get', `s3://${bucket}/${MPUpload}`, download], done, 64); }); }); @@ -660,7 +662,7 @@ MPUploadSplitter.forEach(file => { }); it('should not be able to get deleted object', done => { - exec(['get', `s3://${bucket}/${file}`, download], done, 12); + exec(['get', `s3://${bucket}/${file}`, download], done, 64); }); }); }); @@ -728,7 +730,7 @@ describe('s3cmd info', () => { // test that POLICY and CORS are returned as 'none' it('should find that policy has a value of none', done => { - checkRawOutput(['info', `s3://${bucket}`], 'policy', 'none', + checkRawOutput(['info', `s3://${bucket}`], 'Policy', 'none', 'stdout', foundIt => { assert(foundIt); done(); @@ -736,7 +738,7 @@ describe('s3cmd info', () => { }); it('should find that cors has a value of none', done => { - checkRawOutput(['info', `s3://${bucket}`], 'cors', 'none', + checkRawOutput(['info', `s3://${bucket}`], 'CORS', 'none', 'stdout', foundIt => { assert(foundIt); done(); @@ -762,7 +764,7 @@ describe('s3cmd info', () => { }); it('should find that cors has a value', done => { - checkRawOutput(['info', `s3://${bucket}`], 'cors', corsConfig, + checkRawOutput(['info', `s3://${bucket}`], 'CORS', corsConfig, 'stdout', foundIt => { assert(foundIt, 'Did not find value for cors'); done(); From aef272ea3c2576e21bb49b9f0da7b39f7d6cd26b Mon Sep 17 00:00:00 2001 From: williamlardier Date: Thu, 22 Jun 2023 14:30:43 +0200 Subject: [PATCH 07/14] CLDSRV-409: remove python 2.7 --- .github/workflows/tests.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 532796b3a4..c436ced948 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -217,9 +217,7 @@ jobs: uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: - python-version: | - 2.7 - 3.9 + python-version: 3.9 - name: Setup CI environment uses: ./.github/actions/setup-ci - name: Setup python test environment From 507782bd17d5e804658ac1cde7d9fa0b2dc132e2 Mon Sep 17 00:00:00 2001 From: williamlardier Date: Thu, 22 Jun 2023 14:42:09 +0200 Subject: [PATCH 08/14] CLDSRV-409: remove virtualenv --- .github/workflows/tests.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index c436ced948..9e9dc091f4 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -231,7 +231,6 @@ jobs: run: |- set -o pipefail; bash wait_for_local_port.bash 8000 40 - source ~/.virtualenv/py2/bin/activate yarn run ft_test | tee /tmp/artifacts/${{ github.job }}/tests.log - name: Upload logs to artifacts uses: scality/action-artifacts@v3 From c57a6e3c57728dad133e0b6364bb402e88618329 Mon Sep 17 00:00:00 2001 From: williamlardier Date: Thu, 22 Jun 2023 15:14:24 +0200 Subject: [PATCH 09/14] CLDSRV-409: fix s3cmd test --- tests/functional/s3cmd/tests.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/functional/s3cmd/tests.js b/tests/functional/s3cmd/tests.js index a1bff602bc..26ad51bd9d 100644 --- a/tests/functional/s3cmd/tests.js +++ b/tests/functional/s3cmd/tests.js @@ -264,7 +264,7 @@ describe('s3cmd putBucket', () => { exec([ 'mb', `s3://${bucket}`, '--bucket-location=scality-us-west-1', - ], done, 11); + ], done, 13); }); it('put an invalid bucket, should fail', done => { From 9774d31b034af2bf6aff91df5f783aa99f854386 Mon Sep 17 00:00:00 2001 From: williamlardier Date: Wed, 12 Jul 2023 16:51:12 +0200 Subject: [PATCH 10/14] CLDSRV-402: optimize multideleteobject API - Parallelism is increased to reduce the latency - If the backend supports it, batching is used - Batch the deletion of objects from storage - Flag to disable or enable the optimization, as well as a way to tune it. --- constants.js | 1 + lib/Config.js | 11 + lib/api/apiUtils/object/deleteObject.js | 18 ++ lib/api/multiObjectDelete.js | 348 +++++++++++++++--------- lib/api/objectDelete.js | 7 +- lib/metadata/metadataUtils.js | 42 ++- lib/routes/routeBackbeat.js | 4 +- lib/services.js | 17 +- 8 files changed, 312 insertions(+), 136 deletions(-) create mode 100644 lib/api/apiUtils/object/deleteObject.js diff --git a/constants.js b/constants.js index e0e04f85c3..044530490f 100644 --- a/constants.js +++ b/constants.js @@ -177,6 +177,7 @@ const constants = { assumedRoleArnResourceType: 'assumed-role', // Session name of the backbeat lifecycle assumed role session. backbeatLifecycleSessionName: 'backbeat-lifecycle', + multiObjectDeleteConcurrency: 50, }; module.exports = constants; diff --git a/lib/Config.js b/lib/Config.js index 5b2b28675b..5fdda38405 100644 --- a/lib/Config.js +++ b/lib/Config.js @@ -17,6 +17,7 @@ const { azureAccountNameRegex, base64Regex, } = require('../constants'); const { utapiVersion } = require('utapi'); const { versioning } = require('arsenal'); +const constants = require('../constants'); const versionIdUtils = versioning.VersionID; @@ -1320,6 +1321,16 @@ class Config extends EventEmitter { } this.lifecycleRoleName = config.lifecycleRoleName || null; + this.multiObjectDeleteConcurrency = constants.multiObjectDeleteConcurrency; + const extractedNumber = Number.parseInt(config.multiObjectDeleteConcurrency, 10); + if (!isNaN(extractedNumber) && extractedNumber > 0 && extractedNumber < 1000) { + this.multiObjectDeleteConcurrency = extractedNumber; + } + + this.multiObjectDeleteEnableOptimizations = true; + if (config.multiObjectDeleteEnableOptimizations === false) { + this.multiObjectDeleteEnableOptimizations = false; + } } _configureBackends() { diff --git a/lib/api/apiUtils/object/deleteObject.js b/lib/api/apiUtils/object/deleteObject.js new file mode 100644 index 0000000000..214c5f2bc2 --- /dev/null +++ b/lib/api/apiUtils/object/deleteObject.js @@ -0,0 +1,18 @@ +/** + * _bucketRequiresOplogUpdate - DELETE an object from a bucket + * @param {BucketInfo} bucket - bucket object + * @return {boolean} whether objects require oplog updates on deletion, or not + */ +function _bucketRequiresOplogUpdate(bucket) { + // Default behavior is to require an oplog update + if (!bucket || !bucket.getLifecycleConfiguration || !bucket.getNotificationConfiguration) { + return true; + } + // If the bucket has lifecycle configuration or notification configuration + // set, we also require an oplog update + return bucket.getLifecycleConfiguration() || bucket.getNotificationConfiguration(); +} + +module.exports = { + _bucketRequiresOplogUpdate, +}; diff --git a/lib/api/multiObjectDelete.js b/lib/api/multiObjectDelete.js index 9f2dd31a44..53abea1e55 100644 --- a/lib/api/multiObjectDelete.js +++ b/lib/api/multiObjectDelete.js @@ -17,11 +17,14 @@ const { preprocessingVersioningDelete } = require('./apiUtils/object/versioning'); const createAndStoreObject = require('./apiUtils/object/createAndStoreObject'); const monitoring = require('../utilities/metrics'); -const { metadataGetObject } = require('../metadata/metadataUtils'); +const metadataUtils = require('../metadata/metadataUtils'); const { config } = require('../Config'); const { hasGovernanceBypassHeader, checkUserGovernanceBypass, ObjectLockInfo } = require('./apiUtils/object/objectLockHelpers'); const requestUtils = policies.requestUtils; +const { data } = require('../data/wrapper'); +const logger = require('../utilities/logger'); +const { _bucketRequiresOplogUpdate } = require('./apiUtils/object/deleteObject'); const versionIdUtils = versioning.VersionID; @@ -166,6 +169,63 @@ function _parseXml(xmlToParse, next) { }); } +/** + * decodeObjectVersion - decode object version to be deleted + * @param {object} entry - entry from data model + * @param {function} next - callback to call with error or decoded version + * @return {undefined} + **/ +function decodeObjectVersion(entry) { + let decodedVersionId; + if (entry.versionId) { + decodedVersionId = entry.versionId === 'null' ? + 'null' : versionIdUtils.decode(entry.versionId); + } + if (decodedVersionId instanceof Error) { + return [errors.NoSuchVersion]; + } + return [null, decodedVersionId]; +} + +/** + * Initialization function for the MultiObjectDelete API that will, based on the + * current metadata backend, assess if metadata READ batching is supported. If + * yes, the initialization step will call the metadataGetObjects function from + * the MetadataWrapper. + * @param {string} bucketName - bucket name + * @param {string []} inPlay - list of object keys still in play + * @param {object} log - logger object + * @param {function} callback - callback to call with error or list of objects + * @return {undefined} + */ +function initializeMultiObjectDeleteWithBatchingSupport(bucketName, inPlay, log, callback) { + if (config.multiObjectDeleteEnableOptimizations === false) { + return callback(null, {}); + } + // If the backend supports batching, we want to optimize the API latency by + // first getting all the objects metadata, stored in memory, for later use + // in the API. This approach does not change the API architecture, but + // transplants an additional piece of code that can greatly improve the API + // latency when the database supports batching. + const objectKeys = Object.values(inPlay).map(entry => { + const [err, versionId] = decodeObjectVersion(entry, bucketName); + if (err) { + return null; + } + return { + versionId, + inPlay: entry, + }; + }); + return metadataUtils.metadataGetObjects(bucketName, objectKeys, log, (err, cache) => { + // This optional step is read-only, so any error can be safely ignored + if (err) { + return callback(null, {}); + } + return callback(null, cache); + }); +} + /** * gets object metadata and deletes object * @param {AuthInfo} authInfo - Instance of AuthInfo class with requester's info @@ -191,34 +251,18 @@ function getObjMetadataAndDelete(authInfo, canonicalID, request, let numOfObjectsRemoved = 0; const skipError = new Error('skip'); const objectLockedError = new Error('object locked'); + let deleteFromStorage = []; - // doing 5 requests at a time. note that the data wrapper - // will do 5 parallel requests to data backend to delete parts - return async.forEachLimit(inPlay, 5, (entry, moveOn) => { - async.waterfall([ - callback => { - let decodedVersionId; - if (entry.versionId) { - decodedVersionId = entry.versionId === 'null' ? - 'null' : versionIdUtils.decode(entry.versionId); - } - if (decodedVersionId instanceof Error) { - monitoring.promMetrics('DELETE', bucketName, 404, - 'multiObjectDelete'); - return callback(errors.NoSuchVersion); - } - return callback(null, decodedVersionId); - }, - // for obj deletes, no need to check acl's at object level - // (authority is at the bucket level for obj deletes) - (versionId, callback) => metadataGetObject(bucketName, entry.key, - versionId, log, (err, objMD) => { - // if general error from metadata return error - if (err) { - monitoring.promMetrics('DELETE', bucketName, err.code, - 'multiObjectDelete'); - return callback(err); - } + return async.waterfall([ + callback => initializeMultiObjectDeleteWithBatchingSupport(bucketName, inPlay, log, callback), + (cache, callback) => async.forEachLimit(inPlay, config.multiObjectDeleteConcurrency, (entry, moveOn) => { + async.waterfall([ + callback => callback(...decodeObjectVersion(entry, bucketName)), + // for obj deletes, no need to check acl's at object level + // (authority is at the bucket level for obj deletes) + (versionId, callback) => metadataUtils.metadataGetObject(bucketName, entry.key, + versionId, cache, log, (err, objMD) => callback(err, objMD, versionId)), + (objMD, versionId, callback) => { if (!objMD) { const verCfg = bucket.getVersioningConfiguration(); // To adhere to AWS behavior, create a delete marker @@ -226,7 +270,7 @@ function getObjMetadataAndDelete(authInfo, canonicalID, request, // when versioning has been configured if (verCfg && !entry.versionId) { log.debug('trying to delete specific version ' + - ' that does not exist'); + 'that does not exist'); return callback(null, objMD, versionId); } // otherwise if particular key does not exist, AWS @@ -242,114 +286,160 @@ function getObjMetadataAndDelete(authInfo, canonicalID, request, objMD.location[0].deleteVersion = true; } return callback(null, objMD, versionId); - }), - (objMD, versionId, callback) => { - // AWS only returns an object lock error if a version id - // is specified, else continue to create a delete marker - if (!versionId || !bucket.isObjectLockEnabled()) { - return callback(null, null, objMD, versionId); - } - const hasGovernanceBypass = hasGovernanceBypassHeader(request.headers); - if (hasGovernanceBypass && authInfo.isRequesterAnIAMUser()) { - return checkUserGovernanceBypass(request, authInfo, bucket, entry.key, log, error => { - if (error && error.is.AccessDenied) { - log.debug('user does not have BypassGovernanceRetention and object is locked', { error }); - return callback(objectLockedError); + }, + (objMD, versionId, callback) => { + // AWS only returns an object lock error if a version id + // is specified, else continue to create a delete marker + if (!versionId || !bucket.isObjectLockEnabled()) { + return callback(null, null, objMD, versionId); + } + const hasGovernanceBypass = hasGovernanceBypassHeader(request.headers); + if (hasGovernanceBypass && authInfo.isRequesterAnIAMUser()) { + return checkUserGovernanceBypass(request, authInfo, bucket, entry.key, log, error => { + if (error && error.is.AccessDenied) { + log.debug('user does not have BypassGovernanceRetention and object is locked', + { error }); + return callback(objectLockedError); + } + if (error) { + return callback(error); + } + return callback(null, hasGovernanceBypass, objMD, versionId); + }); + } + return callback(null, hasGovernanceBypass, objMD, versionId); + }, + (hasGovernanceBypass, objMD, versionId, callback) => { + // AWS only returns an object lock error if a version id + // is specified, else continue to create a delete marker + if (!versionId || !bucket.isObjectLockEnabled()) { + return callback(null, objMD, versionId); + } + const objLockInfo = new ObjectLockInfo({ + mode: objMD.retentionMode, + date: objMD.retentionDate, + legalHold: objMD.legalHold || false, + }); + + // If the object can not be deleted raise an error + if (!objLockInfo.canModifyObject(hasGovernanceBypass)) { + log.debug('trying to delete locked object'); + return callback(objectLockedError); + } + + return callback(null, objMD, versionId); + }, + (objMD, versionId, callback) => { + const options = preprocessingVersioningDelete( + bucketName, bucket, objMD, versionId, config.nullVersionCompatMode); + const deleteInfo = {}; + if (options && options.deleteData) { + deleteInfo.deleted = true; + if (!_bucketRequiresOplogUpdate(bucket)) { + options.doesNotNeedOpogUpdate = true; } - if (error) { - return callback(error); + if (objMD.uploadId) { + // eslint-disable-next-line + options.replayId = objMD.uploadId; } - return callback(null, hasGovernanceBypass, objMD, versionId); - }); + return services.deleteObject(bucketName, objMD, + entry.key, options, config.multiObjectDeleteEnableOptimizations, log, (err, toDelete) => { + if (err) { + return callback(err); + } + if (toDelete) { + deleteFromStorage = deleteFromStorage.concat(toDelete); + } + return callback(null, objMD, deleteInfo); + }); + } + deleteInfo.newDeleteMarker = true; + // This call will create a delete-marker + return createAndStoreObject(bucketName, bucket, entry.key, + objMD, authInfo, canonicalID, null, request, + deleteInfo.newDeleteMarker, null, log, (err, result) => + callback(err, objMD, deleteInfo, result.versionId)); + }, + ], (err, objMD, deleteInfo, versionId) => { + if (err === skipError) { + return moveOn(); + } else if (err === objectLockedError) { + errorResults.push({ entry, error: errors.AccessDenied, objectLocked: true }); + return moveOn(); + } else if (err) { + log.error('error deleting object', { error: err, entry }); + errorResults.push({ entry, error: err }); + return moveOn(); } - return callback(null, hasGovernanceBypass, objMD, versionId); - }, - (hasGovernanceBypass, objMD, versionId, callback) => { - // AWS only returns an object lock error if a version id - // is specified, else continue to create a delete marker - if (!versionId || !bucket.isObjectLockEnabled()) { - return callback(null, objMD, versionId); + if (deleteInfo.deleted && objMD['content-length']) { + numOfObjectsRemoved++; + totalContentLengthDeleted += objMD['content-length']; + } + let isDeleteMarker; + let deleteMarkerVersionId; + // - If trying to delete an object that does not exist (if a new + // delete marker was created) + // - Or if an object exists but no version was specified + // return DeleteMarkerVersionId equals the versionID of the marker + // you just generated and DeleteMarker tag equals true + if (deleteInfo.newDeleteMarker) { + isDeleteMarker = true; + deleteMarkerVersionId = versionIdUtils.encode( + versionId, config.versionIdEncodingType); + // In this case we are putting a new object (i.e., the delete + // marker), so we decrement the numOfObjectsRemoved value. + numOfObjectsRemoved--; + // If trying to delete a delete marker, DeleteMarkerVersionId equals + // deleteMarker's versionID and DeleteMarker equals true + } else if (objMD && objMD.isDeleteMarker) { + isDeleteMarker = true; + deleteMarkerVersionId = entry.versionId; } - const objLockInfo = new ObjectLockInfo({ - mode: objMD.retentionMode, - date: objMD.retentionDate, - legalHold: objMD.legalHold || false, + successfullyDeleted.push({ + entry, isDeleteMarker, + deleteMarkerVersionId, }); + return moveOn(); + }); + }, + // end of forEach func + err => { + // Batch delete all objects + const onDone = () => callback(err, quietSetting, errorResults, numOfObjectsRemoved, + successfullyDeleted, totalContentLengthDeleted, bucket); - // If the object can not be deleted raise an error - if (!objLockInfo.canModifyObject(hasGovernanceBypass)) { - log.debug('trying to delete locked object'); - return callback(objectLockedError); + if (err && deleteFromStorage.length === 0) { + log.trace('no objects to delete from data backend'); + return onDone(); } + // If error but we have objects in the list, delete them to ensure + // consistent state. + log.trace('deleting objects from data backend'); - return callback(null, objMD, versionId); - }, - (objMD, versionId, callback) => { - const options = preprocessingVersioningDelete( - bucketName, bucket, objMD, versionId, config.nullVersionCompatMode); - const deleteInfo = {}; - if (options && options.deleteData) { - deleteInfo.deleted = true; - if (objMD.uploadId) { - // eslint-disable-next-line - options.replayId = objMD.uploadId; - } - return services.deleteObject(bucketName, objMD, - entry.key, options, log, err => - callback(err, objMD, deleteInfo)); + // Split the array into chunks + const chunks = []; + while (deleteFromStorage.length > 0) { + chunks.push(deleteFromStorage.splice(0, config.multiObjectDeleteConcurrency)); } - deleteInfo.newDeleteMarker = true; - // This call will create a delete-marker - return createAndStoreObject(bucketName, bucket, entry.key, - objMD, authInfo, canonicalID, null, request, - deleteInfo.newDeleteMarker, null, log, (err, result) => - callback(err, objMD, deleteInfo, result.versionId)); - }, - ], (err, objMD, deleteInfo, versionId) => { - if (err === skipError) { - return moveOn(); - } else if (err === objectLockedError) { - errorResults.push({ entry, error: errors.AccessDenied, objectLocked: true }); - return moveOn(); - } else if (err) { - log.error('error deleting object', { error: err, entry }); - errorResults.push({ entry, error: err }); - return moveOn(); - } - if (deleteInfo.deleted && objMD['content-length']) { - numOfObjectsRemoved++; - totalContentLengthDeleted += objMD['content-length']; - } - let isDeleteMarker; - let deleteMarkerVersionId; - // - If trying to delete an object that does not exist (if a new - // delete marker was created) - // - Or if an object exists but no version was specified - // return DeleteMarkerVersionId equals the versionID of the marker - // you just generated and DeleteMarker tag equals true - if (deleteInfo.newDeleteMarker) { - isDeleteMarker = true; - deleteMarkerVersionId = versionIdUtils.encode( - versionId, config.versionIdEncodingType); - // In this case we are putting a new object (i.e., the delete - // marker), so we decrement the numOfObjectsRemoved value. - numOfObjectsRemoved--; - // If trying to delete a delete marker, DeleteMarkerVersionId equals - // deleteMarker's versionID and DeleteMarker equals true - } else if (objMD && objMD.isDeleteMarker) { - isDeleteMarker = true; - deleteMarkerVersionId = entry.versionId; - } - successfullyDeleted.push({ entry, isDeleteMarker, - deleteMarkerVersionId }); - return moveOn(); - }); - }, - // end of forEach func - err => { - log.trace('finished deleting objects', { numOfObjectsRemoved }); - return next(err, quietSetting, errorResults, numOfObjectsRemoved, - successfullyDeleted, totalContentLengthDeleted, bucket); + + return async.each(chunks, (chunk, done) => data.batchDelete(chunk, null, null, + logger.newRequestLoggerFromSerializedUids(log.getSerializedUids()), done), + err => { + if (err) { + log.error('error deleting objects from data backend', { error: err }); + return onDone(err); + } + return onDone(); + }); + }), + ], (err, ...results) => { + // if general error from metadata return error + if (err) { + monitoring.promMetrics('DELETE', bucketName, err.code, + 'multiObjectDelete'); + return next(err); + } + return next(null, ...results); }); } @@ -576,4 +666,6 @@ function multiObjectDelete(authInfo, request, log, callback) { module.exports = { getObjMetadataAndDelete, multiObjectDelete, + decodeObjectVersion, + initializeMultiObjectDeleteWithBatchingSupport, }; diff --git a/lib/api/objectDelete.js b/lib/api/objectDelete.js index beee7c64e0..68eed7e977 100644 --- a/lib/api/objectDelete.js +++ b/lib/api/objectDelete.js @@ -13,6 +13,7 @@ const monitoring = require('../utilities/metrics'); const { hasGovernanceBypassHeader, checkUserGovernanceBypass, ObjectLockInfo } = require('./apiUtils/object/objectLockHelpers'); const { config } = require('../Config'); +const { _bucketRequiresOplogUpdate } = require('./apiUtils/object/deleteObject'); const versionIdUtils = versioning.VersionID; const objectLockedError = new Error('object locked'); @@ -155,8 +156,12 @@ function objectDelete(authInfo, request, log, cb) { delOptions.replayId = objectMD.uploadId; } + if (!_bucketRequiresOplogUpdate(bucketMD)) { + delOptions.doesNotNeedOpogUpdate = true; + } + return services.deleteObject(bucketName, objectMD, objectKey, - delOptions, log, (err, delResult) => next(err, bucketMD, + delOptions, false, log, (err, delResult) => next(err, bucketMD, objectMD, delResult, deleteInfo)); } // putting a new delete marker diff --git a/lib/metadata/metadataUtils.js b/lib/metadata/metadataUtils.js index d931d5d5c9..ef93f24b23 100644 --- a/lib/metadata/metadataUtils.js +++ b/lib/metadata/metadataUtils.js @@ -60,13 +60,18 @@ function getNullVersionFromMaster(bucketName, objectKey, log, cb) { * @param {string} bucketName - name of bucket * @param {string} objectKey - name of object key * @param {string} [versionId] - version of object to retrieve + * @param {object} cachedDocuments - cached version of the documents used for + * abstraction purposes * @param {RequestLogger} log - request logger * @param {function} cb - callback * @return {undefined} - and call callback with err, bucket md and object md */ -function metadataGetObject(bucketName, objectKey, versionId, log, cb) { +function metadataGetObject(bucketName, objectKey, versionId, cachedDocuments, log, cb) { // versionId may be 'null', which asks metadata to fetch the null key specifically const options = { versionId, getDeleteMarker: true }; + if (cachedDocuments && cachedDocuments[objectKey]) { + return cb(null, cachedDocuments[objectKey]); + } return metadata.getObjectMD(bucketName, objectKey, options, log, (err, objMD) => { if (err) { @@ -84,6 +89,40 @@ function metadataGetObject(bucketName, objectKey, versionId, log, cb) { }); } +/** metadataGetObjects - retrieves specified object or version from metadata. This + * method uses cursors, hence is only compatible with a MongoDB DB backend. + * @param {string} bucketName - name of bucket + * @param {string} objectsKeys - name of object key + * @param {RequestLogger} log - request logger + * @param {function} cb - callback + * @return {undefined} - and call callback with err, bucket md and object md + */ +function metadataGetObjects(bucketName, objectsKeys, log, cb) { + const options = { getDeleteMarker: true }; + const objects = objectsKeys.map(objectKey => ({ + key: objectKey ? objectKey.inPlay.key : null, + params: options, + versionId: objectKey ? objectKey.versionId : null, + })); + + // Returned objects are following the following format: { key, doc, versionId } + // That is required with batching to properly map the objects + return metadata.getObjectsMD(bucketName, objects, log, (err, objMds) => { + if (err) { + log.debug('error getting objects MD from metadata', { error: err }); + return cb(err); + } + + const result = {}; + objMds.forEach(objMd => { + if (objMd.doc) { + result[`${objMd.doc.key}${objMd.versionId}`] = objMd.doc; + } + }); + + return cb(null, result); + }); +} /** * Validate that a bucket is accessible and authorized to the user, * return a specific error code otherwise @@ -215,6 +254,7 @@ function metadataValidateBucket(params, log, callback) { module.exports = { validateBucket, metadataGetObject, + metadataGetObjects, metadataValidateBucketAndObj, metadataValidateBucket, }; diff --git a/lib/routes/routeBackbeat.js b/lib/routes/routeBackbeat.js index d017007720..560e593bd9 100644 --- a/lib/routes/routeBackbeat.js +++ b/lib/routes/routeBackbeat.js @@ -909,7 +909,7 @@ function putObjectTagging(request, response, log, callback) { // retrieve it from metadata here. if (dataStoreVersionId === '') { return metadataGetObject(sourceBucket, request.objectKey, - sourceVersionId, log, (err, objMD) => { + sourceVersionId, null, log, (err, objMD) => { if (err) { return callback(err); } @@ -941,7 +941,7 @@ function deleteObjectTagging(request, response, log, callback) { // retrieve it from metadata here. if (dataStoreVersionId === '') { return metadataGetObject(sourceBucket, request.objectKey, - sourceVersionId, log, (err, objMD) => { + sourceVersionId, null, log, (err, objMD) => { if (err) { return callback(err); } diff --git a/lib/services.js b/lib/services.js index 10609836b1..c6103f288d 100644 --- a/lib/services.js +++ b/lib/services.js @@ -284,11 +284,13 @@ const services = { * @param {string} objectKey - object key name * @param {object} options - other instructions, such as { versionId } to * delete a specific version of the object + * @param {boolean} deferLocationDeletion - true if the object should not + * be removed from the storage, but be returned instead. * @param {Log} log - logger instance * @param {function} cb - callback from async.waterfall in objectGet * @return {undefined} */ - deleteObject(bucketName, objectMD, objectKey, options, log, cb) { + deleteObject(bucketName, objectMD, objectKey, options, deferLocationDeletion, log, cb) { log.trace('deleting object from bucket'); assert.strictEqual(typeof bucketName, 'string'); assert.strictEqual(typeof objectMD, 'object'); @@ -305,12 +307,19 @@ const services = { log.getSerializedUids()); if (objectMD.location === null) { return cb(null, res); - } else if (!Array.isArray(objectMD.location)) { + } + + if (deferLocationDeletion) { + return cb(null, Array.isArray(objectMD.location) + ? objectMD.location : [objectMD.location]); + } + + if (!Array.isArray(objectMD.location)) { data.delete(objectMD.location, deleteLog); return cb(null, res); } - return data.batchDelete(objectMD.location, null, null, - deleteLog, err => { + + return data.batchDelete(objectMD.location, null, null, deleteLog, err => { if (err) { return cb(err); } From 84068b205ec385470d9e386047ab8c5e92463f5e Mon Sep 17 00:00:00 2001 From: williamlardier Date: Wed, 12 Jul 2023 16:52:29 +0200 Subject: [PATCH 11/14] CLDSRV-402: test multi object delete optimization --- tests/unit/api/multiObjectDelete.js | 146 +++++++++++++++++++++- tests/unit/api/objectDelete.js | 3 +- tests/unit/metadata/metadataUtils.spec.js | 95 +++++++++++++- tests/utilities/objectLock-util.js | 2 +- 4 files changed, 241 insertions(+), 5 deletions(-) diff --git a/tests/unit/api/multiObjectDelete.js b/tests/unit/api/multiObjectDelete.js index 31cd1f1128..c3dd768ded 100644 --- a/tests/unit/api/multiObjectDelete.js +++ b/tests/unit/api/multiObjectDelete.js @@ -1,15 +1,16 @@ const assert = require('assert'); const { errors } = require('arsenal'); -const { getObjMetadataAndDelete } +const { decodeObjectVersion, getObjMetadataAndDelete, initializeMultiObjectDeleteWithBatchingSupport } = require('../../../lib/api/multiObjectDelete'); +const multiObjectDelete = require('../../../lib/api/multiObjectDelete'); const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../helpers'); const { ds } = require('arsenal').storage.data.inMemory.datastore; const { metadata } = require('arsenal').storage.metadata.inMemory.metadata; const DummyRequest = require('../DummyRequest'); const { bucketPut } = require('../../../lib/api/bucketPut'); const objectPut = require('../../../lib/api/objectPut'); - +const sinon = require('sinon'); const log = new DummyRequestLogger(); const canonicalID = 'accessKey1'; const authInfo = makeAuthInfo(canonicalID); @@ -19,6 +20,8 @@ const postBody = Buffer.from('I am a body', 'utf8'); const contentLength = 2 * postBody.length; const objectKey1 = 'objectName1'; const objectKey2 = 'objectName2'; +const metadataUtils = require('../../../lib/metadata/metadataUtils'); +const services = require('../../../lib/services'); const testBucketPutRequest = new DummyRequest({ bucketName, namespace, @@ -68,6 +71,10 @@ describe('getObjMetadataAndDelete function for multiObjectDelete', () => { }); }); + afterEach(() => { + sinon.restore(); + }); + it('should successfully get object metadata and then ' + 'delete metadata and data', done => { getObjMetadataAndDelete(authInfo, 'foo', request, bucketName, bucket, @@ -184,4 +191,139 @@ describe('getObjMetadataAndDelete function for multiObjectDelete', () => { done(); }); }); + + it('should properly batch delete data even if there are errors in other objects', done => { + const deleteObjectStub = sinon.stub(services, 'deleteObject'); + deleteObjectStub.onCall(0).callsArgWith(6, errors.InternalError); + deleteObjectStub.onCall(1).callsArgWith(6, null); + + getObjMetadataAndDelete(authInfo, 'foo', request, bucketName, bucket, + true, [], [{ key: objectKey1 }, { key: objectKey2 }], log, + (err, quietSetting, errorResults, numOfObjects, + successfullyDeleted, totalContentLengthDeleted) => { + assert.ifError(err); + assert.strictEqual(quietSetting, true); + assert.deepStrictEqual(errorResults, [ + { + entry: { + key: objectKey1, + }, + error: errors.InternalError, + }, + ]); + assert.strictEqual(numOfObjects, 1); + assert.strictEqual(totalContentLengthDeleted, contentLength / 2); + // Expect still in memory as we stubbed the function + assert.strictEqual(metadata.keyMaps.get(bucketName).has(objectKey1), true); + assert.strictEqual(metadata.keyMaps.get(bucketName).has(objectKey2), true); + // ensure object 2 only is in the list of successful deletions + assert.strictEqual(successfullyDeleted.length, 1); + assert.deepStrictEqual(successfullyDeleted[0].entry.key, objectKey2); + return done(); + }); + }); +}); + +describe('initializeMultiObjectDeleteWithBatchingSupport', () => { + let bucketName; + let inPlay; + let log; + let callback; + + beforeEach(() => { + bucketName = 'myBucket'; + inPlay = { one: 'object1', two: 'object2' }; + log = {}; + callback = sinon.spy(); + }); + + afterEach(() => { + sinon.restore(); + }); + + it('should not throw if the decodeObjectVersion function fails', done => { + const metadataGetObjectsStub = sinon.stub(metadataUtils, 'metadataGetObjects').yields(null, {}); + sinon.stub(multiObjectDelete, 'decodeObjectVersion').returns([new Error('decode error')]); + + initializeMultiObjectDeleteWithBatchingSupport(bucketName, inPlay, log, callback); + + assert.strictEqual(metadataGetObjectsStub.callCount, 1); + sinon.assert.calledOnce(callback); + assert.strictEqual(callback.getCall(0).args[0], null); + assert.deepStrictEqual(callback.getCall(0).args[1], {}); + done(); + }); + + it('should call the batching method if the backend supports it', done => { + const metadataGetObjectsStub = sinon.stub(metadataUtils, 'metadataGetObjects').yields(null, {}); + const objectVersion = 'someVersionId'; + sinon.stub(multiObjectDelete, 'decodeObjectVersion').returns([null, objectVersion]); + + initializeMultiObjectDeleteWithBatchingSupport(bucketName, inPlay, log, callback); + + assert.strictEqual(metadataGetObjectsStub.callCount, 1); + sinon.assert.calledOnce(callback); + assert.strictEqual(callback.getCall(0).args[0], null); + done(); + }); + + it('should not return an error if the metadataGetObjects function fails', done => { + const metadataGetObjectsStub = + sinon.stub(metadataUtils, 'metadataGetObjects').yields(new Error('metadata error'), null); + const objectVersion = 'someVersionId'; + sinon.stub(multiObjectDelete, 'decodeObjectVersion').returns([null, objectVersion]); + + initializeMultiObjectDeleteWithBatchingSupport(bucketName, inPlay, log, callback); + + assert.strictEqual(metadataGetObjectsStub.callCount, 1); + sinon.assert.calledOnce(callback); + assert.strictEqual(callback.getCall(0).args[0] instanceof Error, false); + assert.deepStrictEqual(callback.getCall(0).args[1], {}); + done(); + }); + + it('should populate the cache when the backend supports it', done => { + const expectedOutput = { + one: { + value: 'object1', + }, + two: { + value: 'object2', + }, + }; + const metadataGetObjectsStub = sinon.stub(metadataUtils, 'metadataGetObjects').yields(null, expectedOutput); + const objectVersion = 'someVersionId'; + sinon.stub(multiObjectDelete, 'decodeObjectVersion').returns([null, objectVersion]); + + initializeMultiObjectDeleteWithBatchingSupport(bucketName, inPlay, log, callback); + + assert.strictEqual(metadataGetObjectsStub.callCount, 1); + sinon.assert.calledOnce(callback); + assert.strictEqual(callback.getCall(0).args[0], null); + assert.deepStrictEqual(callback.getCall(0).args[1], expectedOutput); + done(); + }); +}); + +describe('decodeObjectVersion function helper', () => { + it('should throw error for invalid version IDs', () => { + const ret = decodeObjectVersion({ + versionId: '\0', + }); + assert(ret[0].is.NoSuchVersion); + }); + + it('should return "null" for null versionId', () => { + const ret = decodeObjectVersion({ + versionId: 'null', + }); + assert.strictEqual(ret[0], null); + assert.strictEqual(ret[1], 'null'); + }); + + it('should return null error on success', () => { + const ret = decodeObjectVersion({}); + assert.ifError(ret[0]); + assert.deepStrictEqual(ret[1], undefined); + }); }); diff --git a/tests/unit/api/objectDelete.js b/tests/unit/api/objectDelete.js index b1ae2e77c6..5da43260eb 100644 --- a/tests/unit/api/objectDelete.js +++ b/tests/unit/api/objectDelete.js @@ -139,7 +139,8 @@ describe('objectDelete API', () => { any, any, any, { deleteData: true, replayId: testUploadId, - }, any, any); + doesNotNeedOpogUpdate: true, + }, any, any, any); done(); }); }); diff --git a/tests/unit/metadata/metadataUtils.spec.js b/tests/unit/metadata/metadataUtils.spec.js index 574c90e41e..ef83c607bb 100644 --- a/tests/unit/metadata/metadataUtils.spec.js +++ b/tests/unit/metadata/metadataUtils.spec.js @@ -1,4 +1,5 @@ const assert = require('assert'); +const sinon = require('sinon'); const { models } = require('arsenal'); const { BucketInfo } = models; @@ -13,7 +14,8 @@ const bucket = new BucketInfo('niftyBucket', ownerCanonicalId, authInfo.getAccountDisplayName(), creationDate); const log = new DummyRequestLogger(); -const { validateBucket } = require('../../../lib/metadata/metadataUtils'); +const { validateBucket, metadataGetObjects, metadataGetObject } = require('../../../lib/metadata/metadataUtils'); +const metadata = require('../../../lib/metadata/wrapper'); describe('validateBucket', () => { it('action bucketPutPolicy by bucket owner', () => { @@ -53,3 +55,94 @@ describe('validateBucket', () => { assert(validationResult.is.AccessDenied); }); }); + +describe('metadataGetObjects', () => { + let sandbox; + const objectsKeys = [ + { inPlay: { key: 'objectKey1' }, versionId: 'versionId1' }, + { inPlay: { key: 'objectKey2' }, versionId: 'versionId2' }, + ]; + + beforeEach(() => { + sandbox = sinon.createSandbox(); + }); + + afterEach(() => { + sandbox.restore(); + }); + + it('should return error if metadata.getObjectsMD fails', done => { + const error = new Error('Failed to get object metadata'); + sandbox.stub(metadata, 'getObjectsMD').yields(error); + + metadataGetObjects('bucketName', objectsKeys, log, err => { + assert(err); + assert.strictEqual(err, error); + done(); + }); + }); + + it('should return object metadata if successful', done => { + const metadataObjs = [ + { doc: { key: 'objectKey1' }, versionId: 'versionId1' }, + { doc: { key: 'objectKey2' }, versionId: 'versionId2' }, + ]; + sandbox.stub(metadata, 'getObjectsMD').yields(null, metadataObjs); + + metadataGetObjects('bucketName', objectsKeys, log, (err, result) => { + assert.ifError(err); + assert(result); + assert.strictEqual(result.objectKey1versionId1, metadataObjs[0].doc); + assert.strictEqual(result.objectKey2versionId2, metadataObjs[1].doc); + done(); + }); + }); +}); + +describe('metadataGetObject', () => { + let sandbox; + const objectKey = { inPlay: { key: 'objectKey1' }, versionId: 'versionId1' }; + + beforeEach(() => { + sandbox = sinon.createSandbox(); + }); + + afterEach(() => { + sandbox.restore(); + }); + + it('should return the cached document if provided', done => { + const cachedDoc = { + [objectKey.inPlay.key]: { + key: 'objectKey1', versionId: 'versionId1', + }, + }; + metadataGetObject('bucketName', objectKey.inPlay.key, objectKey.versionId, cachedDoc, log, (err, result) => { + assert.ifError(err); + assert.deepStrictEqual(result, cachedDoc[objectKey.inPlay.key]); + done(); + }); + }); + + it('should return error if metadata.getObjectMD fails', done => { + const error = new Error('Failed to get object metadata'); + sandbox.stub(metadata, 'getObjectMD').yields(error); + + metadataGetObject('bucketName', objectKey.inPlay.key, objectKey.versionId, null, log, err => { + assert(err); + assert.strictEqual(err, error); + done(); + }); + }); + + it('should return object metadata if successful', done => { + const metadataObj = { doc: { key: 'objectKey1', versionId: 'versionId1' } }; + sandbox.stub(metadata, 'getObjectMD').yields(null, metadataObj); + + metadataGetObject('bucketName', objectKey.inPlay.key, objectKey.versionId, null, log, (err, result) => { + assert.ifError(err); + assert.deepStrictEqual(result, metadataObj); + done(); + }); + }); +}); diff --git a/tests/utilities/objectLock-util.js b/tests/utilities/objectLock-util.js index e7a1778a4f..92a223e6cd 100644 --- a/tests/utilities/objectLock-util.js +++ b/tests/utilities/objectLock-util.js @@ -12,7 +12,7 @@ const log = new DummyRequestLogger(); function changeObjectLock(objects, newConfig, cb) { async.each(objects, (object, next) => { const { bucket, key, versionId } = object; - metadataGetObject(bucket, key, versionIdUtils.decode(versionId), log, (err, objMD) => { + metadataGetObject(bucket, key, versionIdUtils.decode(versionId), null, log, (err, objMD) => { assert.ifError(err); // set newConfig as empty string to remove object lock /* eslint-disable no-param-reassign */ From f7593d385eca7b32df64eced68df50abd3173173 Mon Sep 17 00:00:00 2001 From: williamlardier Date: Wed, 12 Jul 2023 16:52:39 +0200 Subject: [PATCH 12/14] CLDSRV-402: bump arsenal dependency --- package.json | 2 +- yarn.lock | 142 +++++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 133 insertions(+), 11 deletions(-) diff --git a/package.json b/package.json index 56f011c840..9772bdc3ff 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ "homepage": "https://github.com/scality/S3#readme", "dependencies": { "@hapi/joi": "^17.1.0", - "arsenal": "git+https://github.com/scality/arsenal#7.70.4", + "arsenal": "git+https://github.com/scality/arsenal#7.70.7", "async": "~2.5.0", "aws-sdk": "2.905.0", "azure-storage": "^2.1.0", diff --git a/yarn.lock b/yarn.lock index 3d1f57eb0e..35e3b3b3f0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -129,6 +129,11 @@ resolved "https://registry.yarnpkg.com/@sinonjs/text-encoding/-/text-encoding-0.7.2.tgz#5981a8db18b56ba38ef0efb7d995b12aa7b51918" integrity sha512-sXXKG+uL9IrKqViTtao2Ws6dy0znu9sOaP1di/jKGW1M6VssO8vlpXCQcpZ+jisQ1tTFAC5Jo/EOzFbggBagFQ== +"@socket.io/component-emitter@~3.1.0": + version "3.1.0" + resolved "https://registry.yarnpkg.com/@socket.io/component-emitter/-/component-emitter-3.1.0.tgz#96116f2a912e0c02817345b3c10751069920d553" + integrity sha512-+9jVqKhRSpsc591z5vX+X5Yyw+he/HCB4iQ/RYxw35CEPaY1gnsNE43nf9n9AaYjAQrTiI/mOwKUKdUs9vf7Xg== + "@tootallnate/once@1": version "1.1.2" resolved "https://registry.yarnpkg.com/@tootallnate/once/-/once-1.1.2.tgz#ccb91445360179a04e7fe6aff78c00ffc1eeaf82" @@ -139,6 +144,23 @@ resolved "https://registry.yarnpkg.com/@types/async/-/async-3.2.15.tgz#26d4768fdda0e466f18d6c9918ca28cc89a4e1fe" integrity sha512-PAmPfzvFA31mRoqZyTVsgJMsvbynR429UTTxhmfsUCrWGh3/fxOrzqBtaTPJsn4UtzTv4Vb0+/O7CARWb69N4g== +"@types/cookie@^0.4.1": + version "0.4.1" + resolved "https://registry.yarnpkg.com/@types/cookie/-/cookie-0.4.1.tgz#bfd02c1f2224567676c1545199f87c3a861d878d" + integrity sha512-XW/Aa8APYr6jSVVA1y/DEIZX0/GMKLEVekNG727R8cs56ahETkRAy/3DR7+fJyh7oUgGwNQaRfXCun0+KbWY7Q== + +"@types/cors@^2.8.12": + version "2.8.13" + resolved "https://registry.yarnpkg.com/@types/cors/-/cors-2.8.13.tgz#b8ade22ba455a1b8cb3b5d3f35910fd204f84f94" + integrity sha512-RG8AStHlUiV5ysZQKq97copd2UmVYw3/pRMLefISZ3S1hK104Cwm7iLQ3fTKx+lsUH2CE8FlLaYeEA2LSeqYUA== + dependencies: + "@types/node" "*" + +"@types/node@*", "@types/node@>=10.0.0": + version "20.3.1" + resolved "https://registry.yarnpkg.com/@types/node/-/node-20.3.1.tgz#e8a83f1aa8b649377bb1fb5d7bac5cb90e784dfe" + integrity sha512-EhcH/wvidPy1WeML3TtYFGR83UzjxeWRen9V402T8aUGYsCHOmfoisV3ZSg03gAFIbLq8TnWOJ0f4cALtnSEUg== + "@types/utf8@^3.0.1": version "3.0.1" resolved "https://registry.yarnpkg.com/@types/utf8/-/utf8-3.0.1.tgz#bf081663d4fff05ee63b41f377a35f8b189f7e5b" @@ -426,9 +448,9 @@ arraybuffer.slice@~0.0.7: optionalDependencies: ioctl "^2.0.2" -"arsenal@git+https://github.com/scality/arsenal#7.70.4": - version "7.70.4" - resolved "git+https://github.com/scality/arsenal#c4cc5a2c3dfa4a8d6d565c4029ec05cbb0bf1a3e" +"arsenal@git+https://github.com/scality/arsenal#7.70.7": + version "7.70.7" + resolved "git+https://github.com/scality/arsenal#2938bb0c886c2cd34cbb2a2ceaf3107db7012213" dependencies: "@types/async" "^3.2.12" "@types/utf8" "^3.0.1" @@ -444,7 +466,7 @@ arraybuffer.slice@~0.0.7: bson "4.0.0" debug "~2.6.9" diskusage "^1.1.1" - fcntl "github:scality/node-fcntl#0.2.0" + fcntl "github:scality/node-fcntl#0.2.2" hdclient scality/hdclient#1.1.0 https-proxy-agent "^2.2.0" ioredis "^4.28.5" @@ -456,8 +478,8 @@ arraybuffer.slice@~0.0.7: node-forge "^0.7.1" prom-client "14.2.0" simple-glob "^0.2" - socket.io "~2.3.0" - socket.io-client "~2.3.0" + socket.io "~4.6.1" + socket.io-client "~4.6.1" sproxydclient "github:scality/sproxydclient#8.0.4" utf8 "2.1.2" uuid "^3.0.1" @@ -712,7 +734,7 @@ base64-js@^1.0.2, base64-js@^1.3.1: resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== -base64id@2.0.0: +base64id@2.0.0, base64id@~2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/base64id/-/base64id-2.0.0.tgz#2770ac6bc47d312af97a8bf9a634342e0cd25cb6" integrity sha512-lGe34o6EHj9y3Kts9R4ZYs/Gr+6N7MCaMlIFA3F1R2O5/m7K06AxfSeO5530PEERE6/WyEg3lsuyw4GHlPZHog== @@ -1150,6 +1172,11 @@ cookie@0.5.0: resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.5.0.tgz#d1f5d71adec6558c58f389987c366aa47e994f8b" integrity sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw== +cookie@~0.4.1: + version "0.4.2" + resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.4.2.tgz#0e41f24de5ecf317947c82fc789e06a884824432" + integrity sha512-aSWTXFzaKWkvHO1Ny/s+ePFpvKsPnjc551iI41v3ny/ow6tBG5Vd+FuqGNhh1LxOmVzOlGUriIlOaokOvhaStA== + core-js@^2.4.0: version "2.6.12" resolved "https://registry.yarnpkg.com/core-js/-/core-js-2.6.12.tgz#d9333dfa7b065e347cc5682219d6f690859cc2ec" @@ -1165,6 +1192,14 @@ core-util-is@~1.0.0: resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.3.tgz#a6042d3634c2b27e9328f837b965fac83808db85" integrity sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ== +cors@~2.8.5: + version "2.8.5" + resolved "https://registry.yarnpkg.com/cors/-/cors-2.8.5.tgz#eac11da51592dd86b9f06f6e7ac293b3df875d29" + integrity sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g== + dependencies: + object-assign "^4" + vary "^1" + cron-parser@^2.11.0, cron-parser@^2.15.0, cron-parser@^2.18.0: version "2.18.0" resolved "https://registry.yarnpkg.com/cron-parser/-/cron-parser-2.18.0.tgz#de1bb0ad528c815548371993f81a54e5a089edcf" @@ -1228,7 +1263,7 @@ debug@2.6.9, debug@^2.1.1, debug@^2.2.0, debug@^2.6.8, debug@~2.6.9: dependencies: ms "2.0.0" -debug@4, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.3: +debug@4, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.3, debug@~4.3.1, debug@~4.3.2: version "4.3.4" resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.4.tgz#1319f6579357f2338d3337d2cdd4914bb5dcc865" integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ== @@ -1423,6 +1458,17 @@ engine.io-client@~3.4.0: xmlhttprequest-ssl "~1.5.4" yeast "0.1.2" +engine.io-client@~6.4.0: + version "6.4.0" + resolved "https://registry.yarnpkg.com/engine.io-client/-/engine.io-client-6.4.0.tgz#88cd3082609ca86d7d3c12f0e746d12db4f47c91" + integrity sha512-GyKPDyoEha+XZ7iEqam49vz6auPnNJ9ZBfy89f+rMMas8AuiMWOZ9PVzu8xb9ZC6rafUqiGHSCfu22ih66E+1g== + dependencies: + "@socket.io/component-emitter" "~3.1.0" + debug "~4.3.1" + engine.io-parser "~5.0.3" + ws "~8.11.0" + xmlhttprequest-ssl "~2.0.0" + engine.io-parser@~2.2.0: version "2.2.1" resolved "https://registry.yarnpkg.com/engine.io-parser/-/engine.io-parser-2.2.1.tgz#57ce5611d9370ee94f99641b589f94c97e4f5da7" @@ -1434,6 +1480,11 @@ engine.io-parser@~2.2.0: blob "0.0.5" has-binary2 "~1.0.2" +engine.io-parser@~5.0.3: + version "5.0.7" + resolved "https://registry.yarnpkg.com/engine.io-parser/-/engine.io-parser-5.0.7.tgz#ed5eae76c71f398284c578ab6deafd3ba7e4e4f6" + integrity sha512-P+jDFbvK6lE3n1OL+q9KuzdOFWkkZ/cMV9gol/SbVfpyqfvrfrFTOFJ6fQm2VC3PZHlU3QPhVwmbsCnauHF2MQ== + engine.io@~3.4.0: version "3.4.2" resolved "https://registry.yarnpkg.com/engine.io/-/engine.io-3.4.2.tgz#8fc84ee00388e3e228645e0a7d3dfaeed5bd122c" @@ -1446,6 +1497,22 @@ engine.io@~3.4.0: engine.io-parser "~2.2.0" ws "^7.1.2" +engine.io@~6.4.2: + version "6.4.2" + resolved "https://registry.yarnpkg.com/engine.io/-/engine.io-6.4.2.tgz#ffeaf68f69b1364b0286badddf15ff633476473f" + integrity sha512-FKn/3oMiJjrOEOeUub2WCox6JhxBXq/Zn3fZOMCBxKnNYtsdKjxhl7yR3fZhM9PV+rdE75SU5SYMc+2PGzo+Tg== + dependencies: + "@types/cookie" "^0.4.1" + "@types/cors" "^2.8.12" + "@types/node" ">=10.0.0" + accepts "~1.3.4" + base64id "2.0.0" + cookie "~0.4.1" + cors "~2.8.5" + debug "~4.3.1" + engine.io-parser "~5.0.3" + ws "~8.11.0" + entities@~2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/entities/-/entities-2.1.0.tgz#992d3129cf7df6870b96c57858c249a120f8b8b5" @@ -1811,6 +1878,14 @@ fast-levenshtein@~2.0.6: nan "^2.3.2" node-gyp "^8.0.0" +"fcntl@github:scality/node-fcntl#0.2.2": + version "0.2.1" + resolved "https://codeload.github.com/scality/node-fcntl/tar.gz/b1335ca204c6265cedc50c26020c4d63aabe920e" + dependencies: + bindings "^1.1.1" + nan "^2.3.2" + node-gyp "^8.0.0" + fecha@^4.2.0: version "4.2.3" resolved "https://registry.yarnpkg.com/fecha/-/fecha-4.2.3.tgz#4d9ccdbc61e8629b259fdca67e65891448d569fd" @@ -3839,7 +3914,7 @@ oauth-sign@~0.9.0: resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.9.0.tgz#47a7b016baa68b5fa0ecf3dee08a85c679ac6455" integrity sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ== -object-assign@^4.0.1, object-assign@^4.1.0: +object-assign@^4, object-assign@^4.0.1, object-assign@^4.1.0: version "4.1.1" resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" integrity sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg== @@ -4585,6 +4660,13 @@ socket.io-adapter@~1.1.0: resolved "https://registry.yarnpkg.com/socket.io-adapter/-/socket.io-adapter-1.1.2.tgz#ab3f0d6f66b8fc7fca3959ab5991f82221789be9" integrity sha512-WzZRUj1kUjrTIrUKpZLEzFZ1OLj5FwLlAFQs9kuZJzJi5DKdU7FsWc36SNmA8iDOtwBQyT8FkrriRM8vXLYz8g== +socket.io-adapter@~2.5.2: + version "2.5.2" + resolved "https://registry.yarnpkg.com/socket.io-adapter/-/socket.io-adapter-2.5.2.tgz#5de9477c9182fdc171cd8c8364b9a8894ec75d12" + integrity sha512-87C3LO/NOMc+eMcpcxUBebGjkpMDkNBS9tf7KJqcDsmL936EChtVva71Dw2q4tQcuVC+hAUy4an2NO/sYXmwRA== + dependencies: + ws "~8.11.0" + socket.io-client@2.3.0: version "2.3.0" resolved "https://registry.yarnpkg.com/socket.io-client/-/socket.io-client-2.3.0.tgz#14d5ba2e00b9bcd145ae443ab96b3f86cbcc1bb4" @@ -4622,6 +4704,16 @@ socket.io-client@~2.3.0: socket.io-parser "~3.3.0" to-array "0.1.4" +socket.io-client@~4.6.1: + version "4.6.2" + resolved "https://registry.yarnpkg.com/socket.io-client/-/socket.io-client-4.6.2.tgz#2bfde952e74625d54e622718a7cb1d591ee62fd6" + integrity sha512-OwWrMbbA8wSqhBAR0yoPK6EdQLERQAYjXb3A0zLpgxfM1ZGLKoxHx8gVmCHA6pcclRX5oA/zvQf7bghAS11jRA== + dependencies: + "@socket.io/component-emitter" "~3.1.0" + debug "~4.3.2" + engine.io-client "~6.4.0" + socket.io-parser "~4.2.4" + socket.io-parser@~3.3.0: version "3.3.2" resolved "https://registry.yarnpkg.com/socket.io-parser/-/socket.io-parser-3.3.2.tgz#ef872009d0adcf704f2fbe830191a14752ad50b6" @@ -4640,6 +4732,14 @@ socket.io-parser@~3.4.0: debug "~4.1.0" isarray "2.0.1" +socket.io-parser@~4.2.4: + version "4.2.4" + resolved "https://registry.yarnpkg.com/socket.io-parser/-/socket.io-parser-4.2.4.tgz#c806966cf7270601e47469ddeec30fbdfda44c83" + integrity sha512-/GbIKmo8ioc+NIWIhwdecY0ge+qVBSMdgxGygevmdHj24bsfgtCmcUUcQ5ZzcylGFHsN3k4HB4Cgkl96KVnuew== + dependencies: + "@socket.io/component-emitter" "~3.1.0" + debug "~4.3.1" + socket.io@~2.3.0: version "2.3.0" resolved "https://registry.yarnpkg.com/socket.io/-/socket.io-2.3.0.tgz#cd762ed6a4faeca59bc1f3e243c0969311eb73fb" @@ -4652,6 +4752,18 @@ socket.io@~2.3.0: socket.io-client "2.3.0" socket.io-parser "~3.4.0" +socket.io@~4.6.1: + version "4.6.2" + resolved "https://registry.yarnpkg.com/socket.io/-/socket.io-4.6.2.tgz#d597db077d4df9cbbdfaa7a9ed8ccc3d49439786" + integrity sha512-Vp+lSks5k0dewYTfwgPT9UeGGd+ht7sCpB7p0e83VgO4X/AHYWhXITMrNk/pg8syY2bpx23ptClCQuHhqi2BgQ== + dependencies: + accepts "~1.3.4" + base64id "~2.0.0" + debug "~4.3.2" + engine.io "~6.4.2" + socket.io-adapter "~2.5.2" + socket.io-parser "~4.2.4" + socks-proxy-agent@^6.0.0: version "6.2.1" resolved "https://registry.yarnpkg.com/socks-proxy-agent/-/socks-proxy-agent-6.2.1.tgz#2687a31f9d7185e38d530bef1944fe1f1496d6ce" @@ -5251,7 +5363,7 @@ validator@^13.0.0, validator@^13.6.0, validator@^13.7.0: resolved "https://registry.yarnpkg.com/validator/-/validator-13.7.0.tgz#4f9658ba13ba8f3d82ee881d3516489ea85c0857" integrity sha512-nYXQLCBkpJ8X6ltALua9dRrZDHVYxjJ1wgskNt1lH9fzGjs3tgojGSCBjmEPwkWS1y29+DrizMTW19Pr9uB2nw== -vary@~1.1.2: +vary@^1, vary@~1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/vary/-/vary-1.1.2.tgz#2299f02c6ded30d4a5961b0b9f74524a18f634fc" integrity sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg== @@ -5414,6 +5526,11 @@ ws@~6.1.0: dependencies: async-limiter "~1.0.0" +ws@~8.11.0: + version "8.11.0" + resolved "https://registry.yarnpkg.com/ws/-/ws-8.11.0.tgz#6a0d36b8edfd9f96d8b25683db2f8d7de6e8e143" + integrity sha512-HPG3wQd9sNQoT9xHyNCXoDUa+Xw/VevmY9FoHyQ+g+rrMn4j6FB4np7Z0OhdTgjx6MgQLK7jwSy1YecU1+4Asg== + xml2js@0.4.19: version "0.4.19" resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.19.tgz#686c20f213209e94abf0d1bcf1efaa291c7827a7" @@ -5457,6 +5574,11 @@ xmlhttprequest-ssl@~1.5.4: resolved "https://registry.yarnpkg.com/xmlhttprequest-ssl/-/xmlhttprequest-ssl-1.5.5.tgz#c2876b06168aadc40e57d97e81191ac8f4398b3e" integrity sha512-/bFPLUgJrfGUL10AIv4Y7/CUt6so9CLtB/oFxQSHseSDNNCdC6vwwKEqwLN6wNPBg9YWXAiMu8jkf6RPRS/75Q== +xmlhttprequest-ssl@~2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/xmlhttprequest-ssl/-/xmlhttprequest-ssl-2.0.0.tgz#91360c86b914e67f44dce769180027c0da618c67" + integrity sha512-QKxVRxiRACQcVuQEYFsI1hhkrMlrXHPegbbd1yn9UHOmRxY+si12nQYzri3vbzt8VdTTRviqcKxcyllFas5z2A== + xtend@^4.0.0, xtend@^4.0.2, xtend@~4.0.0: version "4.0.2" resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54" From af0436f1cdcced0b16440550306d465084609c12 Mon Sep 17 00:00:00 2001 From: williamlardier Date: Wed, 12 Jul 2023 16:52:51 +0200 Subject: [PATCH 13/14] CLDSRV-402: bump project version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 9772bdc3ff..6977560475 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "s3", - "version": "7.70.21", + "version": "7.70.22", "description": "S3 connector", "main": "index.js", "engines": { From 68e4b0610a3bd56ca1016ee9096786ca0f44a587 Mon Sep 17 00:00:00 2001 From: williamlardier Date: Thu, 13 Jul 2023 17:10:06 +0200 Subject: [PATCH 14/14] CLDSRV-402: bump project version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 6977560475..cb16c76cf4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "s3", - "version": "7.70.22", + "version": "7.70.23", "description": "S3 connector", "main": "index.js", "engines": {