Skip to content

Commit

Permalink
Allow using NGINX keyval store for credential caching
Browse files Browse the repository at this point in the history
  • Loading branch information
@dekobon
dekobon committed May 17, 2022
1 parent 4e123f3 commit 1e8c05c
Show file tree
Hide file tree
Showing 5 changed files with 187 additions and 23 deletions.
134 changes: 117 additions & 17 deletions common/etc/nginx/include/s3gateway.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,12 +150,60 @@ function _credentialsTempFile() {
}

/**
* Read the contents of the credentials file into memory. If it is not
* found, then return undefined.
* Write the instance profile credentials to a caching backend.
*
* @returns {undefined|object} AWS instance profile credentials or undefined
* @param r {Request} HTTP request object (not used, but required for NGINX configuration)
* @param credentials {{accessKeyId: (string), secretAccessKey: (string), sessionToken: (string), expiration: (string)}} AWS instance profile credentials
*/
function readCredentials() {
function writeCredentials(r, credentials) {
// Do not bother writing credentials if we are running in a mode where we
// do not need instance credentials.
if (process.env['S3_ACCESS_KEY_ID'] && process.env['S3_SECRET_KEY']) {
return;
}

if (!credentials) {
throw `Cannot write invalid credentials: ${JSON.stringify(credentials)}`;
}

if ("variables" in r && r.variables.cache_instance_credentials_enabled == 1) {
_writeCredentialsToKeyValStore(r, credentials);
} else {
_writeCredentialsToFile(credentials);
}
}

/**
* Write the instance profile credentials to the NGINX Keyval store.
*
* @param r {Request} HTTP request object (not used, but required for NGINX configuration)
* @param credentials {{accessKeyId: (string), secretAccessKey: (string), sessionToken: (string), expiration: (string)}} AWS instance profile credentials
* @private
*/
function _writeCredentialsToKeyValStore(r, credentials) {
r.variables.instance_credential_json = JSON.stringify(credentials);
}

/**
* Write the instance profile credentials to a file on the file system. This
* file will be quite small and should end up in the file cache relatively
* quickly if it is repeatedly read.
*
* @param r {Request} HTTP request object (not used, but required for NGINX configuration)
* @param credentials {{accessKeyId: (string), secretAccessKey: (string), sessionToken: (string), expiration: (string)}} AWS instance profile credentials
* @private
*/
function _writeCredentialsToFile(credentials) {
fs.writeFileSync(_credentialsTempFile(), JSON.stringify(credentials));
}

/**
* Get the instance profile credentials needed to authenticated against S3 from
* a backend cache. If the credentials cannot be found, then return undefined.
* @param r {Request} HTTP request object (not used, but required for NGINX configuration)
* @returns {undefined|{accessKeyId: (string), secretAccessKey: (string), sessionToken: (string), expiration: (string)}} AWS instance profile credentials or undefined
*/
function readCredentials(r) {
if (process.env['S3_ACCESS_KEY_ID'] && process.env['S3_SECRET_KEY']) {
return {
accessKeyId: process.env['S3_ACCESS_KEY_ID'],
Expand All @@ -165,6 +213,44 @@ function readCredentials() {
};
}

if ("variables" in r && r.variables.cache_instance_credentials_enabled == 1) {
return _readCredentialsFromKeyValStore(r);
} else {
return _readCredentialsFromFile();
}
}

/**
* Read credentials from the NGINX Keyval store. If it is not found, then
* return undefined.
*
* @param r {Request} HTTP request object (not used, but required for NGINX configuration)
* @returns {undefined|{accessKeyId: (string), secretAccessKey: (string), sessionToken: (string), expiration: (string)}} AWS instance profile credentials or undefined
* @private
*/
function _readCredentialsFromKeyValStore(r) {
var cached = r.variables.instance_credential_json;

if (!cached) {
return undefined;
}

try {
return JSON.parse(cached);
} catch (e) {
_debug_log(r, `Error parsing JSON value from r.variables.instance_credential_json: ${e}`);
return undefined;
}
}

/**
* Read the contents of the credentials file into memory. If it is not
* found, then return undefined.
*
* @returns {undefined|{accessKeyId: (string), secretAccessKey: (string), sessionToken: (string), expiration: (string)}} AWS instance profile credentials or undefined
* @private
*/
function _readCredentialsFromFile() {
var credsFilePath = _credentialsTempFile();

try {
Expand Down Expand Up @@ -201,7 +287,7 @@ function s3auth(r) {

var signature;

var credentials = readCredentials();
var credentials = readCredentials(r);
if (sigver == '2') {
signature = signatureV2(r, bucket, credentials);
} else {
Expand All @@ -211,8 +297,14 @@ function s3auth(r) {
return signature;
}

function s3SecurityToken() {
var credentials = readCredentials();
/**
* Get the current session token from the instance profile credential cache.
*
* @param r {Request} HTTP request object (not used, but required for NGINX configuration)
* @returns {string} current session token or empty string
*/
function s3SecurityToken(r) {
var credentials = readCredentials(r);
if (credentials.sessionToken) {
return credentials.sessionToken;
}
Expand Down Expand Up @@ -782,7 +874,21 @@ var maxValidityOffsetMs = 4.5 * 60 * 100;
* @returns {Promise<void>}
*/
async function fetchCredentials(r) {
var current = readCredentials();
// If we are not using an AWS instance profile to set our credentials we
// exit quickly and don't write a credentials file.
if (process.env['S3_ACCESS_KEY_ID'] && process.env['S3_SECRET_KEY']) {
r.return(200);
return;
}

try {
var current = readCredentials(r);
} catch (e) {
_debug_log(r, `Could not read credentials: ${e}`);
r.return(500);
return;
}

if (current) {
var exp = new Date(current.expiration).getTime() - maxValidityOffsetMs;
if (now.getTime() < exp) {
Expand All @@ -793,13 +899,6 @@ async function fetchCredentials(r) {

var credentials;

// If we are not using an AWS instance profile to set our credentials we
// exit quickly and don't write a credentials file.
if (process.env['S3_ACCESS_KEY_ID'] && process.env['S3_SECRET_KEY']) {
r.return(200);
return;
}

_debug_log(r, 'Cached credentials are expired or not present, requesting new ones');

if (process.env['AWS_CONTAINER_CREDENTIALS_RELATIVE_URI']) {
Expand All @@ -821,9 +920,9 @@ async function fetchCredentials(r) {
}
}
try {
fs.writeFileSync(_credentialsTempFile(), JSON.stringify(credentials));
writeCredentials(r, credentials);
} catch (e) {
_debug_log(r, 'Could not write credentials file: ' + JSON.stringify(e));
_debug_log(r, `Could not write credentials: ${e}`);
r.return(500);
return;
}
Expand Down Expand Up @@ -899,6 +998,7 @@ export default {
awsHeaderDate,
fetchCredentials,
readCredentials,
writeCredentials,
s3date,
s3auth,
s3SecurityToken,
Expand Down
5 changes: 5 additions & 0 deletions oss/etc/nginx/conf.d/gateway/server_variables.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,8 @@
# caching is turned off. This feature uses the keyval store, so it
# is only enabled when using NGINX Plus.
set $cache_signing_key_enabled 0;

# Variable indicating to the s3gateway.js script that session token
# caching is turned on. This feature uses the keyval store, so it
# is only enabled when using NGINX Plus.
set $cache_instance_credentials_enabled 0;
5 changes: 5 additions & 0 deletions plus/etc/nginx/conf.d/gateway/server_variables.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,3 +2,8 @@
# caching is turned on. This feature uses the keyval store, so it
# is only enabled when using NGINX Plus.
set $cache_signing_key_enabled 1;

# Variable indicating to the s3gateway.js script that session token
# caching is turned on. This feature uses the keyval store, so it
# is only enabled when using NGINX Plus.
set $cache_instance_credentials_enabled 1;
4 changes: 4 additions & 0 deletions plus/etc/nginx/conf.d/instance_credential_cache.conf
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
# This key value zone allows us to cache a portion of the cryptographic
# signatures used by AWS v4 signatures.
keyval_zone zone=instance_credential_cache:32k type=string timeout=6h;
keyval 'instance_credential' $instance_credential_json zone=instance_credential_cache;
62 changes: 56 additions & 6 deletions test/unit/s3gateway_test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -311,11 +311,12 @@ function testEscapeURIPathPreservesDoubleSlashes() {

function testReadCredentialsWithAccessAndSecretKeySet() {
printHeader('testReadCredentialsWithAccessAndSecretKeySet');
let r = {};
process.env['S3_ACCESS_KEY_ID'] = 'SOME_ACCESS_KEY';
process.env['S3_SECRET_KEY'] = 'SOME_SECRET_KEY';

try {
var credentials = s3gateway.readCredentials();
var credentials = s3gateway.readCredentials(r);
if (credentials.accessKeyId !== process.env['S3_ACCESS_KEY_ID']) {
throw 'static credentials do not match returned value [accessKeyId]';
}
Expand All @@ -335,8 +336,14 @@ function testReadCredentialsWithAccessAndSecretKeySet() {
}
}

function testReadCredentials() {
printHeader('testReadCredentials');
function testReadCredentialsFromFilePath() {
printHeader('testReadCredentialsFromFilePath');
let r = {
variables: {
cache_instance_credentials_enabled: 0
}
};

var originalCredentialPath = process.env['S3_CREDENTIALS_TEMP_FILE'];
var tempDir = (process.env['TMPDIR'] ? process.env['TMPDIR'] : '/tmp');
var uniqId = `${new Date().getTime()}-${Math.floor(Math.random()*101)}`;
Expand All @@ -347,7 +354,7 @@ function testReadCredentials() {

try {
process.env['S3_CREDENTIALS_TEMP_FILE'] = tempFile;
var credentials = s3gateway.readCredentials();
var credentials = s3gateway.readCredentials(r);
var testDataAsJSON = JSON.parse(testData);
if (credentials.accessKeyId !== testDataAsJSON.accessKeyId) {
throw 'JSON test data does not match credentials [accessKeyId]';
Expand All @@ -373,14 +380,19 @@ function testReadCredentials() {

function testReadCredentialsFromNonexistentPath() {
printHeader('testReadCredentialsFromNonexistentPath');
let r = {
variables: {
cache_instance_credentials_enabled: 0
}
};
var originalCredentialPath = process.env['S3_CREDENTIALS_TEMP_FILE'];
var tempDir = (process.env['TMPDIR'] ? process.env['TMPDIR'] : '/tmp');
var uniqId = `${new Date().getTime()}-${Math.floor(Math.random()*101)}`;
var tempFile = `${tempDir}/credentials-unit-test-${uniqId}.json`;

try {
process.env['S3_CREDENTIALS_TEMP_FILE'] = tempFile;
var credentials = s3gateway.readCredentials();
var credentials = s3gateway.readCredentials(r);
if (credentials !== undefined) {
throw 'Credentials returned when no credentials file should be present';
}
Expand All @@ -395,6 +407,43 @@ function testReadCredentialsFromNonexistentPath() {
}
}

function testReadAndWriteCredentialsFromKeyValStore() {
printHeader('testReadAndWriteCredentialsFromKeyValStore');

let accessKeyId = process.env['S3_ACCESS_KEY_ID'];
let secretKey = process.env['S3_SECRET_KEY'];
delete process.env.S3_ACCESS_KEY_ID;
delete process.env.S3_SECRET_KEY;

try {
let r = {
variables: {
cache_instance_credentials_enabled: 1,
instance_credential_json: null
}
};
let expectedCredentials = {
AccessKeyId: 'AN_ACCESS_KEY_ID',
Expiration: '2017-05-17T15:09:54Z',
RoleArn: 'TASK_ROLE_ARN',
SecretAccessKey: 'A_SECRET_ACCESS_KEY',
Token: 'A_SECURITY_TOKEN',
};

s3gateway.writeCredentials(r, expectedCredentials);
let credentials = JSON.stringify(s3gateway.readCredentials(r));
let expectedJson = JSON.stringify(expectedCredentials);

if (credentials !== expectedJson) {
console.log(`EXPECTED:\n${expectedJson}\nACTUAL:\n${credentials}`);
throw 'Credentials do not match expected value';
}
} finally {
process.env['S3_ACCESS_KEY_ID'] = accessKeyId;
process.env['S3_SECRET_KEY'] = secretKey;
}
}

async function testEcsCredentialRetrieval() {
printHeader('testEcsCredentialRetrieval');
process.env['S3_ACCESS_KEY_ID'] = undefined;
Expand Down Expand Up @@ -531,8 +580,9 @@ async function test() {
testEditAmzHeadersHeadDirectory();
testEscapeURIPathPreservesDoubleSlashes();
testReadCredentialsWithAccessAndSecretKeySet();
testReadCredentials();
testReadCredentialsFromFilePath();
testReadCredentialsFromNonexistentPath();
testReadAndWriteCredentialsFromKeyValStore();
await testEcsCredentialRetrieval();
await testEc2CredentialRetrieval();
}
Expand Down

0 comments on commit 1e8c05c

Please sign in to comment.