From 1e8c05c8876fa0aa3553bda02bb60833a30f5d58 Mon Sep 17 00:00:00 2001 From: Elijah Zupancic Date: Thu, 10 Mar 2022 14:17:05 -0800 Subject: [PATCH] Allow using NGINX keyval store for credential caching --- common/etc/nginx/include/s3gateway.js | 134 +++++++++++++++--- .../conf.d/gateway/server_variables.conf | 5 + .../conf.d/gateway/server_variables.conf | 5 + .../conf.d/instance_credential_cache.conf | 4 + test/unit/s3gateway_test.js | 62 +++++++- 5 files changed, 187 insertions(+), 23 deletions(-) create mode 100644 plus/etc/nginx/conf.d/instance_credential_cache.conf diff --git a/common/etc/nginx/include/s3gateway.js b/common/etc/nginx/include/s3gateway.js index 880f3faf..9f37b0e0 100644 --- a/common/etc/nginx/include/s3gateway.js +++ b/common/etc/nginx/include/s3gateway.js @@ -150,12 +150,60 @@ function _credentialsTempFile() { } /** - * Read the contents of the credentials file into memory. If it is not - * found, then return undefined. + * Write the instance profile credentials to a caching backend. * - * @returns {undefined|object} AWS instance profile credentials or undefined + * @param r {Request} HTTP request object (not used, but required for NGINX configuration) + * @param credentials {{accessKeyId: (string), secretAccessKey: (string), sessionToken: (string), expiration: (string)}} AWS instance profile credentials */ -function readCredentials() { +function writeCredentials(r, credentials) { + // Do not bother writing credentials if we are running in a mode where we + // do not need instance credentials. + if (process.env['S3_ACCESS_KEY_ID'] && process.env['S3_SECRET_KEY']) { + return; + } + + if (!credentials) { + throw `Cannot write invalid credentials: ${JSON.stringify(credentials)}`; + } + + if ("variables" in r && r.variables.cache_instance_credentials_enabled == 1) { + _writeCredentialsToKeyValStore(r, credentials); + } else { + _writeCredentialsToFile(credentials); + } +} + +/** + * Write the instance profile credentials to the NGINX Keyval store. + * + * @param r {Request} HTTP request object (not used, but required for NGINX configuration) + * @param credentials {{accessKeyId: (string), secretAccessKey: (string), sessionToken: (string), expiration: (string)}} AWS instance profile credentials + * @private + */ +function _writeCredentialsToKeyValStore(r, credentials) { + r.variables.instance_credential_json = JSON.stringify(credentials); +} + +/** + * Write the instance profile credentials to a file on the file system. This + * file will be quite small and should end up in the file cache relatively + * quickly if it is repeatedly read. + * + * @param r {Request} HTTP request object (not used, but required for NGINX configuration) + * @param credentials {{accessKeyId: (string), secretAccessKey: (string), sessionToken: (string), expiration: (string)}} AWS instance profile credentials + * @private + */ +function _writeCredentialsToFile(credentials) { + fs.writeFileSync(_credentialsTempFile(), JSON.stringify(credentials)); +} + +/** + * Get the instance profile credentials needed to authenticated against S3 from + * a backend cache. If the credentials cannot be found, then return undefined. + * @param r {Request} HTTP request object (not used, but required for NGINX configuration) + * @returns {undefined|{accessKeyId: (string), secretAccessKey: (string), sessionToken: (string), expiration: (string)}} AWS instance profile credentials or undefined + */ +function readCredentials(r) { if (process.env['S3_ACCESS_KEY_ID'] && process.env['S3_SECRET_KEY']) { return { accessKeyId: process.env['S3_ACCESS_KEY_ID'], @@ -165,6 +213,44 @@ function readCredentials() { }; } + if ("variables" in r && r.variables.cache_instance_credentials_enabled == 1) { + return _readCredentialsFromKeyValStore(r); + } else { + return _readCredentialsFromFile(); + } +} + +/** + * Read credentials from the NGINX Keyval store. If it is not found, then + * return undefined. + * + * @param r {Request} HTTP request object (not used, but required for NGINX configuration) + * @returns {undefined|{accessKeyId: (string), secretAccessKey: (string), sessionToken: (string), expiration: (string)}} AWS instance profile credentials or undefined + * @private + */ +function _readCredentialsFromKeyValStore(r) { + var cached = r.variables.instance_credential_json; + + if (!cached) { + return undefined; + } + + try { + return JSON.parse(cached); + } catch (e) { + _debug_log(r, `Error parsing JSON value from r.variables.instance_credential_json: ${e}`); + return undefined; + } +} + +/** + * Read the contents of the credentials file into memory. If it is not + * found, then return undefined. + * + * @returns {undefined|{accessKeyId: (string), secretAccessKey: (string), sessionToken: (string), expiration: (string)}} AWS instance profile credentials or undefined + * @private + */ +function _readCredentialsFromFile() { var credsFilePath = _credentialsTempFile(); try { @@ -201,7 +287,7 @@ function s3auth(r) { var signature; - var credentials = readCredentials(); + var credentials = readCredentials(r); if (sigver == '2') { signature = signatureV2(r, bucket, credentials); } else { @@ -211,8 +297,14 @@ function s3auth(r) { return signature; } -function s3SecurityToken() { - var credentials = readCredentials(); +/** + * Get the current session token from the instance profile credential cache. + * + * @param r {Request} HTTP request object (not used, but required for NGINX configuration) + * @returns {string} current session token or empty string + */ +function s3SecurityToken(r) { + var credentials = readCredentials(r); if (credentials.sessionToken) { return credentials.sessionToken; } @@ -782,7 +874,21 @@ var maxValidityOffsetMs = 4.5 * 60 * 100; * @returns {Promise} */ async function fetchCredentials(r) { - var current = readCredentials(); + // If we are not using an AWS instance profile to set our credentials we + // exit quickly and don't write a credentials file. + if (process.env['S3_ACCESS_KEY_ID'] && process.env['S3_SECRET_KEY']) { + r.return(200); + return; + } + + try { + var current = readCredentials(r); + } catch (e) { + _debug_log(r, `Could not read credentials: ${e}`); + r.return(500); + return; + } + if (current) { var exp = new Date(current.expiration).getTime() - maxValidityOffsetMs; if (now.getTime() < exp) { @@ -793,13 +899,6 @@ async function fetchCredentials(r) { var credentials; - // If we are not using an AWS instance profile to set our credentials we - // exit quickly and don't write a credentials file. - if (process.env['S3_ACCESS_KEY_ID'] && process.env['S3_SECRET_KEY']) { - r.return(200); - return; - } - _debug_log(r, 'Cached credentials are expired or not present, requesting new ones'); if (process.env['AWS_CONTAINER_CREDENTIALS_RELATIVE_URI']) { @@ -821,9 +920,9 @@ async function fetchCredentials(r) { } } try { - fs.writeFileSync(_credentialsTempFile(), JSON.stringify(credentials)); + writeCredentials(r, credentials); } catch (e) { - _debug_log(r, 'Could not write credentials file: ' + JSON.stringify(e)); + _debug_log(r, `Could not write credentials: ${e}`); r.return(500); return; } @@ -899,6 +998,7 @@ export default { awsHeaderDate, fetchCredentials, readCredentials, + writeCredentials, s3date, s3auth, s3SecurityToken, diff --git a/oss/etc/nginx/conf.d/gateway/server_variables.conf b/oss/etc/nginx/conf.d/gateway/server_variables.conf index 8f47f216..40b888df 100644 --- a/oss/etc/nginx/conf.d/gateway/server_variables.conf +++ b/oss/etc/nginx/conf.d/gateway/server_variables.conf @@ -2,3 +2,8 @@ # caching is turned off. This feature uses the keyval store, so it # is only enabled when using NGINX Plus. set $cache_signing_key_enabled 0; + +# Variable indicating to the s3gateway.js script that session token +# caching is turned on. This feature uses the keyval store, so it +# is only enabled when using NGINX Plus. +set $cache_instance_credentials_enabled 0; diff --git a/plus/etc/nginx/conf.d/gateway/server_variables.conf b/plus/etc/nginx/conf.d/gateway/server_variables.conf index 96337eb8..485d95e9 100644 --- a/plus/etc/nginx/conf.d/gateway/server_variables.conf +++ b/plus/etc/nginx/conf.d/gateway/server_variables.conf @@ -2,3 +2,8 @@ # caching is turned on. This feature uses the keyval store, so it # is only enabled when using NGINX Plus. set $cache_signing_key_enabled 1; + +# Variable indicating to the s3gateway.js script that session token +# caching is turned on. This feature uses the keyval store, so it +# is only enabled when using NGINX Plus. +set $cache_instance_credentials_enabled 1; diff --git a/plus/etc/nginx/conf.d/instance_credential_cache.conf b/plus/etc/nginx/conf.d/instance_credential_cache.conf new file mode 100644 index 00000000..b2e030e4 --- /dev/null +++ b/plus/etc/nginx/conf.d/instance_credential_cache.conf @@ -0,0 +1,4 @@ +# This key value zone allows us to cache a portion of the cryptographic +# signatures used by AWS v4 signatures. +keyval_zone zone=instance_credential_cache:32k type=string timeout=6h; +keyval 'instance_credential' $instance_credential_json zone=instance_credential_cache; diff --git a/test/unit/s3gateway_test.js b/test/unit/s3gateway_test.js index 5b1eb4cd..4f00c1f1 100755 --- a/test/unit/s3gateway_test.js +++ b/test/unit/s3gateway_test.js @@ -311,11 +311,12 @@ function testEscapeURIPathPreservesDoubleSlashes() { function testReadCredentialsWithAccessAndSecretKeySet() { printHeader('testReadCredentialsWithAccessAndSecretKeySet'); + let r = {}; process.env['S3_ACCESS_KEY_ID'] = 'SOME_ACCESS_KEY'; process.env['S3_SECRET_KEY'] = 'SOME_SECRET_KEY'; try { - var credentials = s3gateway.readCredentials(); + var credentials = s3gateway.readCredentials(r); if (credentials.accessKeyId !== process.env['S3_ACCESS_KEY_ID']) { throw 'static credentials do not match returned value [accessKeyId]'; } @@ -335,8 +336,14 @@ function testReadCredentialsWithAccessAndSecretKeySet() { } } -function testReadCredentials() { - printHeader('testReadCredentials'); +function testReadCredentialsFromFilePath() { + printHeader('testReadCredentialsFromFilePath'); + let r = { + variables: { + cache_instance_credentials_enabled: 0 + } + }; + var originalCredentialPath = process.env['S3_CREDENTIALS_TEMP_FILE']; var tempDir = (process.env['TMPDIR'] ? process.env['TMPDIR'] : '/tmp'); var uniqId = `${new Date().getTime()}-${Math.floor(Math.random()*101)}`; @@ -347,7 +354,7 @@ function testReadCredentials() { try { process.env['S3_CREDENTIALS_TEMP_FILE'] = tempFile; - var credentials = s3gateway.readCredentials(); + var credentials = s3gateway.readCredentials(r); var testDataAsJSON = JSON.parse(testData); if (credentials.accessKeyId !== testDataAsJSON.accessKeyId) { throw 'JSON test data does not match credentials [accessKeyId]'; @@ -373,6 +380,11 @@ function testReadCredentials() { function testReadCredentialsFromNonexistentPath() { printHeader('testReadCredentialsFromNonexistentPath'); + let r = { + variables: { + cache_instance_credentials_enabled: 0 + } + }; var originalCredentialPath = process.env['S3_CREDENTIALS_TEMP_FILE']; var tempDir = (process.env['TMPDIR'] ? process.env['TMPDIR'] : '/tmp'); var uniqId = `${new Date().getTime()}-${Math.floor(Math.random()*101)}`; @@ -380,7 +392,7 @@ function testReadCredentialsFromNonexistentPath() { try { process.env['S3_CREDENTIALS_TEMP_FILE'] = tempFile; - var credentials = s3gateway.readCredentials(); + var credentials = s3gateway.readCredentials(r); if (credentials !== undefined) { throw 'Credentials returned when no credentials file should be present'; } @@ -395,6 +407,43 @@ function testReadCredentialsFromNonexistentPath() { } } +function testReadAndWriteCredentialsFromKeyValStore() { + printHeader('testReadAndWriteCredentialsFromKeyValStore'); + + let accessKeyId = process.env['S3_ACCESS_KEY_ID']; + let secretKey = process.env['S3_SECRET_KEY']; + delete process.env.S3_ACCESS_KEY_ID; + delete process.env.S3_SECRET_KEY; + + try { + let r = { + variables: { + cache_instance_credentials_enabled: 1, + instance_credential_json: null + } + }; + let expectedCredentials = { + AccessKeyId: 'AN_ACCESS_KEY_ID', + Expiration: '2017-05-17T15:09:54Z', + RoleArn: 'TASK_ROLE_ARN', + SecretAccessKey: 'A_SECRET_ACCESS_KEY', + Token: 'A_SECURITY_TOKEN', + }; + + s3gateway.writeCredentials(r, expectedCredentials); + let credentials = JSON.stringify(s3gateway.readCredentials(r)); + let expectedJson = JSON.stringify(expectedCredentials); + + if (credentials !== expectedJson) { + console.log(`EXPECTED:\n${expectedJson}\nACTUAL:\n${credentials}`); + throw 'Credentials do not match expected value'; + } + } finally { + process.env['S3_ACCESS_KEY_ID'] = accessKeyId; + process.env['S3_SECRET_KEY'] = secretKey; + } +} + async function testEcsCredentialRetrieval() { printHeader('testEcsCredentialRetrieval'); process.env['S3_ACCESS_KEY_ID'] = undefined; @@ -531,8 +580,9 @@ async function test() { testEditAmzHeadersHeadDirectory(); testEscapeURIPathPreservesDoubleSlashes(); testReadCredentialsWithAccessAndSecretKeySet(); - testReadCredentials(); + testReadCredentialsFromFilePath(); testReadCredentialsFromNonexistentPath(); + testReadAndWriteCredentialsFromKeyValStore(); await testEcsCredentialRetrieval(); await testEc2CredentialRetrieval(); }