From a26ce5f5a84932fd80a1858776330f3d0a48d27b Mon Sep 17 00:00:00 2001 From: Nicolas Humbert Date: Fri, 5 Jul 2024 12:51:58 +0200 Subject: [PATCH 1/4] CLDSRV-554 Allow backbeat/metadata API to be tested E2E --- .../aws-node-sdk/test/support/config.js | 12 ++--------- .../aws-node-sdk/test/support/credentials.js | 21 +++++++++++++++++++ .../raw-node/test/routes/routeBackbeat.js | 7 +++++-- .../routes/routeBackbeatForReplication.js | 7 +++++-- .../raw-node/test/routes/routeMetadata.js | 7 +++++-- 5 files changed, 38 insertions(+), 16 deletions(-) create mode 100644 tests/functional/aws-node-sdk/test/support/credentials.js diff --git a/tests/functional/aws-node-sdk/test/support/config.js b/tests/functional/aws-node-sdk/test/support/config.js index 2c66fec471..ae4df173a2 100644 --- a/tests/functional/aws-node-sdk/test/support/config.js +++ b/tests/functional/aws-node-sdk/test/support/config.js @@ -1,7 +1,7 @@ const https = require('https'); const AWS = require('aws-sdk'); -const memCredentials = require('../../lib/json/mem_credentials.json'); +const { getCredentials } = require('./credentials'); const { getAwsCredentials } = require('./awsConfig'); const conf = require('../../../../../lib/Config').config; @@ -29,16 +29,8 @@ const DEFAULT_MEM_OPTIONS = { }; const DEFAULT_AWS_OPTIONS = {}; -if (!memCredentials || Object.is(memCredentials, {})) { - throw new Error('Credential info is missing in mem_credentials.json'); -} - function _getMemCredentials(profile) { - const credentials = memCredentials[profile] || memCredentials.default; - - const accessKeyId = credentials.accessKey; - const secretAccessKey = credentials.secretKey; - + const { accessKeyId, secretAccessKey } = getCredentials(profile); return new AWS.Credentials(accessKeyId, secretAccessKey); } diff --git a/tests/functional/aws-node-sdk/test/support/credentials.js b/tests/functional/aws-node-sdk/test/support/credentials.js new file mode 100644 index 0000000000..4aa6a176c0 --- /dev/null +++ b/tests/functional/aws-node-sdk/test/support/credentials.js @@ -0,0 +1,21 @@ +const memCredentials = require('../../lib/json/mem_credentials.json'); + +if (!memCredentials || Object.is(memCredentials, {})) { + throw new Error('Credential info is missing in mem_credentials.json'); +} + +function getCredentials(profile = 'default') { + const credentials = memCredentials[profile] || memCredentials.default; + + const accessKeyId = credentials.accessKey; + const secretAccessKey = credentials.secretKey; + + return { + accessKeyId, + secretAccessKey, + }; +} + +module.exports = { + getCredentials, +}; diff --git a/tests/functional/raw-node/test/routes/routeBackbeat.js b/tests/functional/raw-node/test/routes/routeBackbeat.js index d95e6c2069..a5dcb9ebd5 100644 --- a/tests/functional/raw-node/test/routes/routeBackbeat.js +++ b/tests/functional/raw-node/test/routes/routeBackbeat.js @@ -6,14 +6,17 @@ const versionIdUtils = versioning.VersionID; const { makeRequest, makeBackbeatRequest } = require('../../utils/makeRequest'); const BucketUtility = require('../../../aws-node-sdk/lib/utility/bucket-util'); +const { getCredentials } = require('../../../aws-node-sdk/test/support/credentials'); const ipAddress = process.env.IP ? process.env.IP : '127.0.0.1'; const describeSkipIfAWS = process.env.AWS_ON_AIR ? describe.skip : describe; const isNullVersionCompatMode = process.env.ENABLE_NULL_VERSION_COMPAT_MODE === 'true'; +const { accessKeyId, secretAccessKey } = getCredentials(); + const backbeatAuthCredentials = { - accessKey: 'accessKey1', - secretKey: 'verySecretKey1', + accessKey: accessKeyId, + secretKey: secretAccessKey, }; const TEST_BUCKET = 'backbeatbucket'; diff --git a/tests/functional/raw-node/test/routes/routeBackbeatForReplication.js b/tests/functional/raw-node/test/routes/routeBackbeatForReplication.js index ac2ca73401..f696a1e407 100644 --- a/tests/functional/raw-node/test/routes/routeBackbeatForReplication.js +++ b/tests/functional/raw-node/test/routes/routeBackbeatForReplication.js @@ -5,12 +5,15 @@ const { ObjectMD } = models; const { makeBackbeatRequest } = require('../../utils/makeRequest'); const BucketUtility = require('../../../aws-node-sdk/lib/utility/bucket-util'); +const { getCredentials } = require('../../../aws-node-sdk/test/support/credentials'); const describeSkipIfAWS = process.env.AWS_ON_AIR ? describe.skip : describe; +const { accessKeyId, secretAccessKey } = getCredentials(); + const backbeatAuthCredentials = { - accessKey: 'accessKey1', - secretKey: 'verySecretKey1', + accessKey: accessKeyId, + secretKey: secretAccessKey, }; const testData = 'testkey data'; diff --git a/tests/functional/raw-node/test/routes/routeMetadata.js b/tests/functional/raw-node/test/routes/routeMetadata.js index 7af418d36d..d453df95ea 100644 --- a/tests/functional/raw-node/test/routes/routeMetadata.js +++ b/tests/functional/raw-node/test/routes/routeMetadata.js @@ -3,13 +3,16 @@ const http = require('http'); const { makeRequest } = require('../../utils/makeRequest'); const MetadataMock = require('../../utils/MetadataMock'); +const { getCredentials } = require('../../../aws-node-sdk/test/support/credentials'); const ipAddress = process.env.IP ? process.env.IP : 'localhost'; const metadataMock = new MetadataMock(); +const { accessKeyId, secretAccessKey } = getCredentials(); + const metadataAuthCredentials = { - accessKey: 'accessKey1', - secretKey: 'verySecretKey1', + accessKey: accessKeyId, + secretKey: secretAccessKey, }; function makeMetadataRequest(params, callback) { From 88a7bee3d3ca8b341b3fd74ddd10e21b9647d421 Mon Sep 17 00:00:00 2001 From: Nicolas Humbert Date: Wed, 17 Jul 2024 15:48:46 +0200 Subject: [PATCH 2/4] CLDSRV-554 fix put Metadata with hardcoded owner-id --- .../functional/raw-node/test/routes/routeBackbeat.js | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/functional/raw-node/test/routes/routeBackbeat.js b/tests/functional/raw-node/test/routes/routeBackbeat.js index a5dcb9ebd5..7d7833d680 100644 --- a/tests/functional/raw-node/test/routes/routeBackbeat.js +++ b/tests/functional/raw-node/test/routes/routeBackbeat.js @@ -66,6 +66,11 @@ const testMd = { }, }; +// S3_TESTVAL_OWNERCANONICALID variable is used by Integration that runs E2E tests with real Vault account. +if (process.env.S3_TESTVAL_OWNERCANONICALID) { + testMd['owner-id'] = process.env.S3_TESTVAL_OWNERCANONICALID; +} + function checkObjectData(s3, objectKey, dataValue, done) { s3.getObject({ Bucket: TEST_BUCKET, @@ -1939,13 +1944,14 @@ describeSkipIfAWS('backbeat routes', () => { '403 Forbidden if the account does not match the ' + 'backbeat user', done => { - makeBackbeatRequest({ + const { accessKeyId: accessKeyLisa, secretAccessKey: secretAccessKeyLisa } = getCredentials('lisa'); + return makeBackbeatRequest({ method: test.method, bucket: TEST_BUCKET, objectKey: TEST_KEY, resourceType: test.resourceType, queryObj, authCredentials: { - accessKey: 'accessKey2', - secretKey: 'verySecretKey2', + accessKey: accessKeyLisa, + secretKey: secretAccessKeyLisa, }, }, err => { From 9b77587bf0a0bb504d85d7b3b37f09fe9bc8c31a Mon Sep 17 00:00:00 2001 From: Nicolas Humbert Date: Thu, 18 Jul 2024 14:56:10 +0200 Subject: [PATCH 3/4] CLDSRV-554 Skip testing the deletion of an non-existing sproxyd key --- tests/functional/raw-node/test/routes/routeBackbeat.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/functional/raw-node/test/routes/routeBackbeat.js b/tests/functional/raw-node/test/routes/routeBackbeat.js index 7d7833d680..98d0af3925 100644 --- a/tests/functional/raw-node/test/routes/routeBackbeat.js +++ b/tests/functional/raw-node/test/routes/routeBackbeat.js @@ -2121,7 +2121,9 @@ describeSkipIfAWS('backbeat routes', () => { done(); }); }); - it('should skip batch delete of a non-existent location', done => { + + // TODO: unskip test when S3C-9123 is fixed + it.skip('should skip batch delete of a non-existent location', done => { async.series([ done => { const options = { @@ -2132,7 +2134,7 @@ describeSkipIfAWS('backbeat routes', () => { path: '/_/backbeat/batchdelete', requestBody: '{"Locations":' + - '[{"key":"abcdef","dataStoreName":"us-east-1"}]}', + '[{"key":"abcdefghijklmnopqrstuvwxyabcddefghijklmn","dataStoreName":"us-east-1"}]}', jsonResponse: true, }; makeRequest(options, done); From 9fe2d318b0117c5266e73be16e2bd026fc3c7db0 Mon Sep 17 00:00:00 2001 From: Nicolas Humbert Date: Thu, 18 Jul 2024 14:57:31 +0200 Subject: [PATCH 4/4] CLDSRV-554 fix route metadata API tests --- .../raw-node/test/routes/routeMetadata.js | 57 ++++++++++++++----- .../functional/raw-node/utils/MetadataMock.js | 5 +- 2 files changed, 47 insertions(+), 15 deletions(-) diff --git a/tests/functional/raw-node/test/routes/routeMetadata.js b/tests/functional/raw-node/test/routes/routeMetadata.js index d453df95ea..9a916ada5a 100644 --- a/tests/functional/raw-node/test/routes/routeMetadata.js +++ b/tests/functional/raw-node/test/routes/routeMetadata.js @@ -4,9 +4,10 @@ const http = require('http'); const { makeRequest } = require('../../utils/makeRequest'); const MetadataMock = require('../../utils/MetadataMock'); const { getCredentials } = require('../../../aws-node-sdk/test/support/credentials'); +const BucketUtility = require('../../../aws-node-sdk/lib/utility/bucket-util'); +const metadataMock = new MetadataMock(); const ipAddress = process.env.IP ? process.env.IP : 'localhost'; -const metadataMock = new MetadataMock(); const { accessKeyId, secretAccessKey } = getCredentials(); @@ -32,15 +33,39 @@ function makeMetadataRequest(params, callback) { makeRequest(options, callback); } -describe('metadata routes with metadata mock backend', () => { - let httpServer; +describe('metadata routes with metadata', () => { + const bucketUtil = new BucketUtility( + 'default', { signatureVersion: 'v4' }); + const s3 = bucketUtil.s3; - before(done => { - httpServer = http.createServer( - (req, res) => metadataMock.onRequest(req, res)).listen(9000, done); - }); + const bucket1 = 'bucket1'; + const bucket2 = 'bucket2'; + const keyName = 'testobject1'; - after(() => httpServer.close()); + // E2E tests use S3C metadata, whereas functional tests use mocked metadata. + if (process.env.S3_END_TO_END) { + before(done => s3.createBucket({ Bucket: bucket1 }).promise() + .then(() => s3.putObject({ Bucket: bucket1, Key: keyName, Body: '' }).promise()) + .then(() => s3.createBucket({ Bucket: bucket2 }).promise()) + .then(() => done(), err => done(err)) + ); + + after(done => bucketUtil.empty(bucket1) + .then(() => s3.deleteBucket({ Bucket: bucket1 }).promise()) + .then(() => bucketUtil.empty(bucket2)) + .then(() => s3.deleteBucket({ Bucket: bucket2 }).promise()) + .then(() => done(), err => done(err)) + ); + } else { + let httpServer; + + before(done => { + httpServer = http.createServer( + (req, res) => metadataMock.onRequest(req, res)).listen(9000, done); + }); + + after(() => httpServer.close()); + } it('should retrieve list of buckets', done => { makeMetadataRequest({ @@ -51,7 +76,13 @@ describe('metadata routes with metadata mock backend', () => { assert.ifError(err); assert.strictEqual(res.statusCode, 200); assert(res.body); - assert.strictEqual(res.body, '["bucket1","bucket2"]'); + const expectedArray = [bucket1, 'users..bucket', bucket2]; + const responseArray = JSON.parse(res.body); + + expectedArray.sort(); + responseArray.sort(); + + assert.deepStrictEqual(responseArray, expectedArray); return done(); }); }); @@ -60,7 +91,7 @@ describe('metadata routes with metadata mock backend', () => { makeMetadataRequest({ method: 'GET', authCredentials: metadataAuthCredentials, - path: '/_/metadata/default/bucket/bucket1', + path: `/_/metadata/default/bucket/${bucket1}`, queryObj: { listingType: 'Delimiter' }, }, (err, res) => { assert.ifError(err); @@ -75,7 +106,7 @@ describe('metadata routes with metadata mock backend', () => { makeMetadataRequest({ method: 'GET', authCredentials: metadataAuthCredentials, - path: '/_/metadata/default/attributes/bucket1', + path: `/_/metadata/default/attributes/${bucket1}`, }, (err, res) => { assert.ifError(err); assert.strictEqual(res.statusCode, 200); @@ -88,13 +119,13 @@ describe('metadata routes with metadata mock backend', () => { makeMetadataRequest({ method: 'GET', authCredentials: metadataAuthCredentials, - path: '/_/metadata/default/bucket/bucket1/testobject1', + path: `/_/metadata/default/bucket/${bucket1}/${keyName}`, }, (err, res) => { assert.ifError(err); assert(res.body); assert.strictEqual(res.statusCode, 200); const body = JSON.parse(res.body); - assert.strictEqual(body.metadata, 'dogsAreGood'); + assert(body['owner-id']); return done(); }); }); diff --git a/tests/functional/raw-node/utils/MetadataMock.js b/tests/functional/raw-node/utils/MetadataMock.js index 639a22758f..008a284b41 100644 --- a/tests/functional/raw-node/utils/MetadataMock.js +++ b/tests/functional/raw-node/utils/MetadataMock.js @@ -208,7 +208,7 @@ class MetadataMock { })); } if (/\/_\/raft_sessions\/[1-8]\/bucket/.test(req.url)) { - const value = ['bucket1', 'bucket2']; + const value = ['bucket1', 'bucket2', 'users..bucket']; res.writeHead(200, { 'content-type': 'application/json' }); return res.end(JSON.stringify(value)); } else if (/\/default\/attributes\/[a-z0-9]/.test(req.url)) { @@ -228,7 +228,8 @@ class MetadataMock { return res.end(JSON.stringify(objectList)); } else if (/\/default\/bucket\/.*\/.*?/.test(req.url)) { return res.end(JSON.stringify({ - metadata: 'dogsAreGood', + 'owner-id': '123', + 'metadata': 'dogsAreGood', })); } else if (mockLogURLRegex.test(req.url)) { return res.end(JSON.stringify(mockLogs));