From 2aae4bab428f157306c24195f8b7f47c07a55cc9 Mon Sep 17 00:00:00 2001 From: MansurTsutiev Date: Wed, 1 Sep 2021 17:10:59 -0400 Subject: [PATCH 01/20] feat: add token types and RequestBuilder - update karma config to include sub-directories Co-authored-by: Alex Guevara --- karma.conf.js | 4 +- src/token-builder/request-builder.js | 25 ++ src/token-builder/types.js | 192 +++++++++++ test/token-builder/request-builder-test.js | 84 +++++ test/token-builder/types-test.js | 351 +++++++++++++++++++++ 5 files changed, 654 insertions(+), 2 deletions(-) create mode 100644 src/token-builder/request-builder.js create mode 100644 src/token-builder/types.js create mode 100644 test/token-builder/request-builder-test.js create mode 100644 test/token-builder/types-test.js diff --git a/karma.conf.js b/karma.conf.js index 3423c4f..f34d7ba 100644 --- a/karma.conf.js +++ b/karma.conf.js @@ -9,7 +9,7 @@ module.exports = function karmaConfig(config) { config.set({ browsers: ['ChromeWithConfiguration'], frameworks: ['qunit'], - files: ['test/*.js'], + files: ['test/**/*.js'], crossOriginAttribute: false, customLaunchers: { ChromeWithConfiguration: { @@ -18,7 +18,7 @@ module.exports = function karmaConfig(config) { }, }, preprocessors: { - 'test/*.js': ['rollup'], + 'test/**/*.js': ['rollup'], }, rollupPreprocessor: { input: 'test/data-source-test.js', diff --git a/src/token-builder/request-builder.js b/src/token-builder/request-builder.js new file mode 100644 index 0000000..da6275d --- /dev/null +++ b/src/token-builder/request-builder.js @@ -0,0 +1,25 @@ +export class RequestBuilder { + constructor(tokens) { + this.tokens = tokens || []; + this.tokenApiVersion = 'V1'; + } + + toJSON() { + const payload = { tokenApiVersion: this.tokenApiVersion, tokens: [] }; + const errors = []; + + this.tokens.forEach((token, index) => { + if (token.errors.length) { + errors.push(`token ${index + 1}: ${token.errors.join(', ')}.`); + } else { + payload.tokens.push(token.toJSON()); + } + }); + + if (errors.length) { + throw new Error(`Errors found while parsing tokens:\n${errors.join('\n')}`); + } + + return payload; + } +} diff --git a/src/token-builder/types.js b/src/token-builder/types.js new file mode 100644 index 0000000..eb928f6 --- /dev/null +++ b/src/token-builder/types.js @@ -0,0 +1,192 @@ +const ALLOWED_ALGOS = new Set(['sha256', 'sha1', 'md5']); +const ALLOWED_ENCODINGS = new Set(['hex', 'base64', 'base64url', 'base64percent']); + +export const REPLACE_CHAR_LIMIT = 100; + +export class TokenBase { + constructor({ name, cacheOverride = null, skipCache = false }) { + this.name = name; + this.type = 'base'; + this.requiredProperties = ['name']; + this.errors = []; + this.cacheOverride = cacheOverride; + this.skipCache = skipCache; + } + + validateOptions() { + const missingProps = []; + this.requiredProperties.forEach((prop) => { + if (!this[prop]) { + missingProps.push(prop); + } + }); + + if (missingProps.length) { + this.errors.push(`Missing properties for ${this.type} token: "${missingProps.join(', ')}"`); + } + } +} + +export class ReplaceToken extends TokenBase { + constructor(params) { + super(params); + this.type = 'replace'; + this.requiredProperties = [...this.requiredProperties, 'value']; + this.value = params.value; + + this.validateOptions(); + } + + toJSON() { + return { + name: this.name, + type: this.type, + value: this.value, + cacheOverride: this.cacheOverride, + skipCache: this.skipCache, + }; + } + + validateOptions = () => { + super.validateOptions(); + + if (this.value && this.value.length > REPLACE_CHAR_LIMIT) { + this.errors.push(`Replace value exceeds ${REPLACE_CHAR_LIMIT} character limit`); + } + }; +} + +export class ReplaceLargeToken extends TokenBase { + constructor(params) { + super(params); + this.type = 'replaceLarge'; + this.requiredProperties = [...this.requiredProperties, 'value']; + this.value = params.value; + + this.validateOptions(); + } + + toJSON() { + return { + name: this.name, + type: this.type, + value: this.value, + cacheOverride: this.cacheOverride, + skipCache: this.skipCache, + }; + } + + validateOptions() { + super.validateOptions(); + + if (this.value && this.value.length <= REPLACE_CHAR_LIMIT) { + this.errors.push( + `ReplaceLarge token can only be used when value exceeds ${REPLACE_CHAR_LIMIT} character limit` + ); + } + } +} + +export class SecretToken extends TokenBase { + constructor(params) { + super(params); + this.type = 'secret'; + this.requiredProperties = [...this.requiredProperties, 'path']; + this.path = params.path; + this.validateOptions(); + } + + toJSON() { + return { + name: this.name, + type: this.type, + path: this.path, + cacheOverride: this.cacheOverride, + skipCache: this.skipCache, + }; + } +} + +export class HmacToken extends TokenBase { + constructor(params) { + super(params); + this.type = 'hmac'; + this.hmacOptions = params.options; + this.validateOptions(); + } + + toJSON() { + return { + name: this.name, + type: this.type, + cacheOverride: this.cacheOverride, + skipCache: this.skipCache, + options: this.hmacOptions, + }; + } + + validateOptions() { + super.validateOptions(); + + if (!ALLOWED_ALGOS.has(this.hmacOptions.algorithm)) { + this.errors.push('HMAC algorithm is invalid'); + } + + if (!this.hmacOptions.secretName) { + this.errors.push('HMAC secret name not provided'); + } + + if (!ALLOWED_ENCODINGS.has(this.hmacOptions.encoding)) { + this.errors.push('HMAC encoding is invalid'); + } + } +} + +export class Sha1Token extends TokenBase { + constructor(params) { + super(params); + this.type = 'sha1'; + this.options = params.options || {}; + this.validateOptions(); + } + + toJSON() { + return { + name: this.name, + type: this.type, + cacheOverride: this.cacheOverride, + skipCache: this.skipCache, + options: this.options, + }; + } + + hasInvalidSecrets() { + if (!Array.isArray(this.options.tokens) || !this.options.tokens.length) { + return false; + } + + for (let token of this.options.tokens) { + if (token.type !== 'secret' || !token.path) { + return true; + } + } + + return false; + } + + validateOptions() { + super.validateOptions(); + + if (!this.options.text) { + this.errors.push('Missing text to encrypt'); + } + + if (this.options.encoding !== 'hex' && this.options.encoding !== 'base64') { + this.errors.push('SHA1 encoding is invalid'); + } + + if (this.hasInvalidSecrets()) { + this.errors.push('Invalid secret token passed into SHA1 tokens array'); + } + } +} diff --git a/test/token-builder/request-builder-test.js b/test/token-builder/request-builder-test.js new file mode 100644 index 0000000..8ab7377 --- /dev/null +++ b/test/token-builder/request-builder-test.js @@ -0,0 +1,84 @@ +import { RequestBuilder } from '../../src/token-builder/request-builder'; +import { ReplaceToken, HmacToken } from '../../src/token-builder/types'; +const { test, module } = QUnit; + +module('RequestBuilder', function () { + test('builds post body payload', (assert) => { + const options = { + name: 'FavoriteBand', + cacheOverride: 'Movable Band', + value: 'Beatles', + }; + + const replaceToken = new ReplaceToken(options); + + const hmacOptions = { + name: 'hmac_sig', + cacheOverride: 'xyz', + options: { + stringToSign: 'mystring', + algorithm: 'sha1', + secretName: 'watson', + encoding: 'hex', + }, + }; + const hmacToken = new HmacToken(hmacOptions); + + const requestBuilder = new RequestBuilder([replaceToken, hmacToken]); + + const expectedPayload = { + tokenApiVersion: 'V1', + tokens: [ + { + name: 'FavoriteBand', + cacheOverride: 'Movable Band', + type: 'replace', + value: 'Beatles', + skipCache: false, + }, + { + name: 'hmac_sig', + type: 'hmac', + cacheOverride: 'xyz', + skipCache: false, + options: { + algorithm: 'sha1', + encoding: 'hex', + secretName: 'watson', + stringToSign: 'mystring', + }, + }, + ], + }; + assert.deepEqual(requestBuilder.toJSON(), expectedPayload); + }); + + test('raises an error with invalid tokens', (assert) => { + const options = { + cacheOverride: 'Movable Band', + }; + + const replaceToken = new ReplaceToken(options); + + const hmacOptions = { + cacheOverride: 'xyz', + options: { + stringToSign: 'mystring', + algorithm: 'ash1', + encoding: 'lex', + }, + }; + const hmacToken = new HmacToken(hmacOptions); + + const requestBuilder = new RequestBuilder([replaceToken, hmacToken]); + const expectedErrors = [ + 'Errors found while parsing tokens:', + 'token 1: Missing properties for replace token: "name, value".', + 'token 2: Missing properties for hmac token: "name", HMAC algorithm is invalid, HMAC secret name not provided, HMAC encoding is invalid.', + ]; + + assert.throws(function () { + requestBuilder.toJSON(); + }, new RegExp(expectedErrors.join('\n'))); + }); +}); diff --git a/test/token-builder/types-test.js b/test/token-builder/types-test.js new file mode 100644 index 0000000..06b9465 --- /dev/null +++ b/test/token-builder/types-test.js @@ -0,0 +1,351 @@ +import { + ReplaceToken, + ReplaceLargeToken, + SecretToken, + HmacToken, + Sha1Token, + REPLACE_CHAR_LIMIT, +} from '../../src/token-builder/types'; + +const { test, module } = QUnit; + +module('ReplaceToken', function () { + test('can be instantiated with all options', (assert) => { + const replaceValue = '*'.repeat(REPLACE_CHAR_LIMIT); + + const options = { + name: 'FavoriteBand', + cacheOverride: 'Movable Band', + value: replaceValue, + skipCache: true, + }; + + const tokenModel = new ReplaceToken(options); + + const expectedJson = { + name: 'FavoriteBand', + type: 'replace', + cacheOverride: 'Movable Band', + skipCache: true, + value: replaceValue, + }; + assert.deepEqual(tokenModel.toJSON(), expectedJson); + }); + + test('can be instantiated with default options', (assert) => { + const options = { + name: 'FavoriteBand', + value: 'Beatles', + }; + + const tokenModel = new ReplaceToken(options); + + const expectedJson = { + name: 'FavoriteBand', + type: 'replace', + cacheOverride: null, + skipCache: false, + value: 'Beatles', + }; + assert.deepEqual(tokenModel.toJSON(), expectedJson); + }); + + test('will include an error if instantiated with missing options', (assert) => { + const tokenModel = new ReplaceToken({}); + + assert.equal(tokenModel.errors.length, 1); + assert.equal(tokenModel.errors[0], 'Missing properties for replace token: "name, value"'); + }); + + test('will include an error if value is longer than replace character limit', (assert) => { + const value = '*'.repeat(REPLACE_CHAR_LIMIT + 1); + const tokenModel = new ReplaceToken({ name: 'my token', value }); + + assert.equal(tokenModel.errors.length, 1); + assert.equal( + tokenModel.errors[0], + `Replace value exceeds ${REPLACE_CHAR_LIMIT} character limit` + ); + }); +}); + +module('ReplaceLargeToken', function () { + test('can be instantiated with all options', (assert) => { + const replaceValue = '*'.repeat(REPLACE_CHAR_LIMIT + 1); + + const options = { + name: 'FavoriteBand', + cacheOverride: 'Movable Band', + value: replaceValue, + skipCache: true, + }; + + const tokenModel = new ReplaceLargeToken(options); + + const expectedJson = { + name: 'FavoriteBand', + type: 'replaceLarge', + cacheOverride: 'Movable Band', + skipCache: true, + value: replaceValue, + }; + assert.deepEqual(tokenModel.toJSON(), expectedJson); + }); + + test('can be instantiated with default options', (assert) => { + const replaceValue = '*'.repeat(REPLACE_CHAR_LIMIT + 1); + const options = { + name: 'FavoriteBand', + value: replaceValue, + }; + + const tokenModel = new ReplaceLargeToken(options); + + const expectedJson = { + name: 'FavoriteBand', + type: 'replaceLarge', + cacheOverride: null, + skipCache: false, + value: replaceValue, + }; + assert.deepEqual(tokenModel.toJSON(), expectedJson); + }); + + test('will include an error if instantiated with missing options', (assert) => { + const tokenModel = new ReplaceLargeToken({}); + + assert.equal(tokenModel.errors.length, 1); + assert.equal(tokenModel.errors[0], 'Missing properties for replaceLarge token: "name, value"'); + }); + + test('will include an error if value is shorter than replace character limit', (assert) => { + const tokenModel = new ReplaceLargeToken({ name: 'my token', value: 'Beatles' }); + + assert.equal(tokenModel.errors.length, 1); + assert.equal( + tokenModel.errors[0], + `ReplaceLarge token can only be used when value exceeds ${REPLACE_CHAR_LIMIT} character limit` + ); + }); +}); + +module('SecretToken', function () { + test('can be instantiated with all options', (assert) => { + const options = { name: 'myApiKey', path: 'watson', skipCache: true, cacheOverride: 'xyz' }; + + const expectedJson = { + name: 'myApiKey', + type: 'secret', + path: 'watson', + skipCache: true, + cacheOverride: 'xyz', + }; + + const tokenModel = new SecretToken(options); + assert.deepEqual(tokenModel.toJSON(), expectedJson); + }); + + test('can be instantiated with default options', (assert) => { + const options = { name: 'myApiKey', path: 'watson' }; + + const expectedJson = { + name: 'myApiKey', + type: 'secret', + path: 'watson', + skipCache: false, + cacheOverride: null, + }; + + const tokenModel = new SecretToken(options); + assert.deepEqual(tokenModel.toJSON(), expectedJson); + }); + + test('will include an error if instantiated with missing options', (assert) => { + const tokenModel = new SecretToken({}); + assert.deepEqual(tokenModel.errors[0], 'Missing properties for secret token: "name, path"'); + }); +}); + +module('HmacToken', function () { + test('can be instantiated with all options', (assert) => { + const hmacOptions = { + name: 'hmac_sig', + cacheOverride: 'xyz', + skipCache: true, + options: { + tokenName: 'hmac_sig', + stringToSign: 'application/json\nGET\n', + algorithm: 'sha1', + secretName: 'watson', + encoding: 'hex', + }, + }; + + const tokenModel = new HmacToken(hmacOptions); + + const expectedJson = { + name: 'hmac_sig', + type: 'hmac', + cacheOverride: 'xyz', + skipCache: true, + options: { + tokenName: 'hmac_sig', + stringToSign: 'application/json\nGET\n', + algorithm: 'sha1', + secretName: 'watson', + encoding: 'hex', + }, + }; + + assert.deepEqual(tokenModel.toJSON(), expectedJson); + }); + + test('gets instantiated with default options', (assert) => { + const hmacOptions = { + name: 'hmac_sig', + options: { + tokenName: 'hmac_sig', + stringToSign: 'application/json\nGET\n', + algorithm: 'sha1', + secretName: 'watson', + encoding: 'hex', + }, + }; + + const tokenModel = new HmacToken(hmacOptions); + + const expectedJson = { + name: 'hmac_sig', + type: 'hmac', + cacheOverride: null, + skipCache: false, + options: { + tokenName: 'hmac_sig', + stringToSign: 'application/json\nGET\n', + algorithm: 'sha1', + secretName: 'watson', + encoding: 'hex', + }, + }; + + assert.deepEqual(tokenModel.toJSON(), expectedJson); + }); + + test('will include an error if instantiated with missing options', (assert) => { + const hmacOptions = { + name: 'hmac_sig', + options: { + tokenName: 'hmac_sig', + stringToSign: 'application/json\nGET\n', + algorithm: 'invalid', + encoding: 'neo', + }, + }; + + const tokenModel = new HmacToken(hmacOptions); + + const expectedErrors = [ + 'HMAC algorithm is invalid', + 'HMAC secret name not provided', + 'HMAC encoding is invalid', + ]; + assert.deepEqual(tokenModel.errors, expectedErrors); + }); +}); + +module('Sha1Token', function () { + test('can be instantiated with all options', (assert) => { + const tokens = [{ name: 'secureValue', type: 'secret', path: 'mySecretPath' }]; + const sha1Options = { + name: 'FavoriteBand', + cacheOverride: 'Movable Band', + options: { + text: 'my text', + encoding: 'hex', + tokens, + }, + skipCache: true, + }; + + const tokenModel = new Sha1Token(sha1Options); + + const expectedJson = { + name: 'FavoriteBand', + type: 'sha1', + cacheOverride: 'Movable Band', + skipCache: true, + options: { + text: 'my text', + encoding: 'hex', + tokens, + }, + }; + assert.deepEqual(tokenModel.toJSON(), expectedJson); + }); + + test('can be instantiated with default options', (assert) => { + const tokens = [{ name: 'secureValue', type: 'secret', path: 'mySecretPath' }]; + const sha1Options = { + name: 'FavoriteBand', + options: { + text: 'my text', + encoding: 'hex', + tokens, + }, + }; + + const tokenModel = new Sha1Token(sha1Options); + + const expectedJson = { + name: 'FavoriteBand', + type: 'sha1', + cacheOverride: null, + skipCache: false, + options: { + text: 'my text', + encoding: 'hex', + tokens, + }, + }; + assert.deepEqual(tokenModel.toJSON(), expectedJson); + }); + + test('will include an error if instantiated with missing options', (assert) => { + const tokenModel = new Sha1Token({}); + + assert.equal(tokenModel.errors.length, 3); + assert.equal(tokenModel.errors[0], 'Missing properties for sha1 token: "name"'); + assert.equal(tokenModel.errors[1], 'Missing text to encrypt'); + assert.equal(tokenModel.errors[2], 'SHA1 encoding is invalid'); + }); + + test('will include an error if a non-secret-type token is passed into tokens array', (assert) => { + const tokens = [{ name: 'favorite band', type: 'replaceToken', value: 'beatles' }]; + const tokenModel = new Sha1Token({ + name: 'my_token', + options: { + text: 'myfavoriteband', + encoding: 'base64', + tokens, + }, + }); + + assert.equal(tokenModel.errors.length, 1); + assert.equal(tokenModel.errors[0], 'Invalid secret token passed into SHA1 tokens array'); + }); + + test('will include an error if a secret token with a missing path is passed into tokens array', (assert) => { + const tokens = [{ name: 'mysecret', type: 'secret', path: '' }]; + const tokenModel = new Sha1Token({ + name: 'my_token', + options: { + text: 'myfavoriteband', + encoding: 'base64', + tokens, + }, + }); + + assert.equal(tokenModel.errors.length, 1); + assert.equal(tokenModel.errors[0], 'Invalid secret token passed into SHA1 tokens array'); + }); +}); From 894cf734deb7d6226f5cfbbe8e1045fddf1747cc Mon Sep 17 00:00:00 2001 From: MansurTsutiev Date: Thu, 2 Sep 2021 11:47:08 -0400 Subject: [PATCH 02/20] feat: update RequestBuilder tests with new token types - get rid of the `.` period when building errors Co-authored-by: Alex Guevara --- src/token-builder/request-builder.js | 2 +- test/token-builder/request-builder-test.js | 113 ++++++++++++++++++--- 2 files changed, 100 insertions(+), 15 deletions(-) diff --git a/src/token-builder/request-builder.js b/src/token-builder/request-builder.js index da6275d..e519496 100644 --- a/src/token-builder/request-builder.js +++ b/src/token-builder/request-builder.js @@ -10,7 +10,7 @@ export class RequestBuilder { this.tokens.forEach((token, index) => { if (token.errors.length) { - errors.push(`token ${index + 1}: ${token.errors.join(', ')}.`); + errors.push(`token ${index + 1}: ${token.errors.join(', ')}`); } else { payload.tokens.push(token.toJSON()); } diff --git a/test/token-builder/request-builder-test.js b/test/token-builder/request-builder-test.js index 8ab7377..b05c13b 100644 --- a/test/token-builder/request-builder-test.js +++ b/test/token-builder/request-builder-test.js @@ -1,16 +1,34 @@ import { RequestBuilder } from '../../src/token-builder/request-builder'; -import { ReplaceToken, HmacToken } from '../../src/token-builder/types'; +import { REPLACE_CHAR_LIMIT } from '../../src/token-builder/types'; +import { + ReplaceToken, + ReplaceLargeToken, + SecretToken, + HmacToken, + Sha1Token, +} from '../../src/token-builder/types'; const { test, module } = QUnit; module('RequestBuilder', function () { test('builds post body payload', (assert) => { - const options = { + const replaceToken = new ReplaceToken({ name: 'FavoriteBand', cacheOverride: 'Movable Band', value: 'Beatles', - }; + }); + + const replaceLargeToken = new ReplaceLargeToken({ + name: 'band', + cacheOverride: 'flooding', + value: '*'.repeat(REPLACE_CHAR_LIMIT + 1), + skipCache: true, + }); - const replaceToken = new ReplaceToken(options); + const secretToken = new SecretToken({ + name: 'myApiKey', + path: 'watson', + cacheOverride: 'xyz', + }); const hmacOptions = { name: 'hmac_sig', @@ -24,18 +42,47 @@ module('RequestBuilder', function () { }; const hmacToken = new HmacToken(hmacOptions); - const requestBuilder = new RequestBuilder([replaceToken, hmacToken]); + const sha1Token = new Sha1Token({ + name: 'sha1_sig', + options: { + text: 'my text', + encoding: 'hex', + tokens: [{ name: 'secureValue', type: 'secret', path: 'mySecretPath' }], + }, + }); + + const requestBuilder = new RequestBuilder([ + replaceToken, + replaceLargeToken, + secretToken, + hmacToken, + sha1Token, + ]); const expectedPayload = { tokenApiVersion: 'V1', tokens: [ { name: 'FavoriteBand', - cacheOverride: 'Movable Band', type: 'replace', + cacheOverride: 'Movable Band', value: 'Beatles', skipCache: false, }, + { + name: 'band', + type: 'replaceLarge', + cacheOverride: 'flooding', + value: '*'.repeat(REPLACE_CHAR_LIMIT + 1), + skipCache: true, + }, + { + name: 'myApiKey', + type: 'secret', + path: 'watson', + skipCache: false, + cacheOverride: 'xyz', + }, { name: 'hmac_sig', type: 'hmac', @@ -48,17 +95,36 @@ module('RequestBuilder', function () { stringToSign: 'mystring', }, }, + { + name: 'sha1_sig', + type: 'sha1', + cacheOverride: null, + skipCache: false, + options: { + text: 'my text', + encoding: 'hex', + tokens: [{ name: 'secureValue', type: 'secret', path: 'mySecretPath' }], + }, + }, ], }; assert.deepEqual(requestBuilder.toJSON(), expectedPayload); }); test('raises an error with invalid tokens', (assert) => { - const options = { - cacheOverride: 'Movable Band', - }; + const replaceToken = new ReplaceToken({ cacheOverride: 'Movable Band' }); + + const replaceLargeToken = new ReplaceLargeToken({ + name: 'band', + cacheOverride: 'flooding', + value: 'short string', + skipCache: true, + }); - const replaceToken = new ReplaceToken(options); + const secretToken = new SecretToken({ + name: 'myApiKey', + cacheOverride: 'xyz', + }); const hmacOptions = { cacheOverride: 'xyz', @@ -70,15 +136,34 @@ module('RequestBuilder', function () { }; const hmacToken = new HmacToken(hmacOptions); - const requestBuilder = new RequestBuilder([replaceToken, hmacToken]); + const sha1Token = new Sha1Token({ + name: 'sha1_sig', + options: { + text: 'my text', + encoding: 'flex', + tokens: [{ name: 'secureValue', type: 'secret', path: '' }], + }, + }); + + const requestBuilder = new RequestBuilder([ + replaceToken, + replaceLargeToken, + secretToken, + hmacToken, + sha1Token, + ]); + const expectedErrors = [ 'Errors found while parsing tokens:', - 'token 1: Missing properties for replace token: "name, value".', - 'token 2: Missing properties for hmac token: "name", HMAC algorithm is invalid, HMAC secret name not provided, HMAC encoding is invalid.', + 'token 1: Missing properties for replace token: "name, value"', + `token 2: ReplaceLarge token can only be used when value exceeds ${REPLACE_CHAR_LIMIT} character limit`, + 'token 3: Missing properties for secret token: "path"', + 'token 4: Missing properties for hmac token: "name", HMAC algorithm is invalid, HMAC secret name not provided, HMAC encoding is invalid', + 'token 5: SHA1 encoding is invalid, Invalid secret token passed into SHA1 tokens array', ]; assert.throws(function () { requestBuilder.toJSON(); - }, new RegExp(expectedErrors.join('\n'))); + }, new Error(expectedErrors.join('\n'))); }); }); From 5a3d9ffa5cd6e63195e0cfba708dcf9c140abc27 Mon Sep 17 00:00:00 2001 From: Alex Guevara Date: Wed, 8 Sep 2021 13:48:47 -0400 Subject: [PATCH 03/20] default hmacOptions to empty object --- src/token-builder/types.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/token-builder/types.js b/src/token-builder/types.js index eb928f6..3fe4d8b 100644 --- a/src/token-builder/types.js +++ b/src/token-builder/types.js @@ -111,7 +111,7 @@ export class HmacToken extends TokenBase { constructor(params) { super(params); this.type = 'hmac'; - this.hmacOptions = params.options; + this.hmacOptions = params.options || {}; this.validateOptions(); } From 95c73f123b23395b819726e9ddffe8537c44a40b Mon Sep 17 00:00:00 2001 From: Alex Guevara Date: Wed, 8 Sep 2021 14:07:09 -0400 Subject: [PATCH 04/20] add toJSON method to TokenBase --- src/token-builder/types.js | 59 +++++++++++++------------------- test/token-builder/types-test.js | 40 ++++++++++++++++++++++ 2 files changed, 64 insertions(+), 35 deletions(-) diff --git a/src/token-builder/types.js b/src/token-builder/types.js index 3fe4d8b..eed6d46 100644 --- a/src/token-builder/types.js +++ b/src/token-builder/types.js @@ -13,6 +13,15 @@ export class TokenBase { this.skipCache = skipCache; } + toJSON() { + return { + name: this.name, + type: this.type, + cacheOverride: this.cacheOverride, + skipCache: this.skipCache, + }; + } + validateOptions() { const missingProps = []; this.requiredProperties.forEach((prop) => { @@ -38,13 +47,9 @@ export class ReplaceToken extends TokenBase { } toJSON() { - return { - name: this.name, - type: this.type, - value: this.value, - cacheOverride: this.cacheOverride, - skipCache: this.skipCache, - }; + const json = super.toJSON(); + + return { ...json, value: this.value }; } validateOptions = () => { @@ -67,13 +72,9 @@ export class ReplaceLargeToken extends TokenBase { } toJSON() { - return { - name: this.name, - type: this.type, - value: this.value, - cacheOverride: this.cacheOverride, - skipCache: this.skipCache, - }; + const json = super.toJSON(); + + return { ...json, value: this.value }; } validateOptions() { @@ -97,13 +98,9 @@ export class SecretToken extends TokenBase { } toJSON() { - return { - name: this.name, - type: this.type, - path: this.path, - cacheOverride: this.cacheOverride, - skipCache: this.skipCache, - }; + const json = super.toJSON(); + + return { ...json, path: this.path }; } } @@ -116,13 +113,9 @@ export class HmacToken extends TokenBase { } toJSON() { - return { - name: this.name, - type: this.type, - cacheOverride: this.cacheOverride, - skipCache: this.skipCache, - options: this.hmacOptions, - }; + const json = super.toJSON(); + + return { ...json, options: this.hmacOptions }; } validateOptions() { @@ -151,13 +144,9 @@ export class Sha1Token extends TokenBase { } toJSON() { - return { - name: this.name, - type: this.type, - cacheOverride: this.cacheOverride, - skipCache: this.skipCache, - options: this.options, - }; + const json = super.toJSON(); + + return { ...json, options: this.options }; } hasInvalidSecrets() { diff --git a/test/token-builder/types-test.js b/test/token-builder/types-test.js index 06b9465..8139f66 100644 --- a/test/token-builder/types-test.js +++ b/test/token-builder/types-test.js @@ -1,4 +1,5 @@ import { + TokenBase, ReplaceToken, ReplaceLargeToken, SecretToken, @@ -8,6 +9,45 @@ import { } from '../../src/token-builder/types'; const { test, module } = QUnit; +module('BaseToken', function () { + test('can be instantiated with all options', (assert) => { + const options = { + name: 'FavoriteBand', + cacheOverride: 'Movable Band', + skipCache: true, + }; + + const tokenModel = new TokenBase(options); + + const expectedJson = { + name: 'FavoriteBand', + type: 'base', + cacheOverride: 'Movable Band', + skipCache: true, + }; + assert.deepEqual(tokenModel.toJSON(), expectedJson); + }); + + test('can be instantiated with default options', (assert) => { + const tokenModel = new TokenBase({ name: 'FavoriteBand' }); + + const expectedJson = { + name: 'FavoriteBand', + type: 'base', + cacheOverride: null, + skipCache: false, + }; + assert.deepEqual(tokenModel.toJSON(), expectedJson); + }); + + test('will include an error if instantiated with missing options', (assert) => { + const tokenModel = new TokenBase({}); + tokenModel.validateOptions(); + + assert.equal(tokenModel.errors.length, 1); + assert.equal(tokenModel.errors[0], 'Missing properties for base token: "name"'); + }); +}); module('ReplaceToken', function () { test('can be instantiated with all options', (assert) => { From 17fa970590ba91c97cb52ac7b60ff4e02260aaf4 Mon Sep 17 00:00:00 2001 From: Alex Guevara Date: Wed, 8 Sep 2021 14:10:23 -0400 Subject: [PATCH 05/20] update error message in RequestBuilder --- src/token-builder/request-builder.js | 6 +++++- test/token-builder/request-builder-test.js | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/token-builder/request-builder.js b/src/token-builder/request-builder.js index e519496..78413b9 100644 --- a/src/token-builder/request-builder.js +++ b/src/token-builder/request-builder.js @@ -17,7 +17,11 @@ export class RequestBuilder { }); if (errors.length) { - throw new Error(`Errors found while parsing tokens:\n${errors.join('\n')}`); + throw new Error( + `Request was not made due to invalid tokens. See validation errors below:\n${errors.join( + '\n' + )}` + ); } return payload; diff --git a/test/token-builder/request-builder-test.js b/test/token-builder/request-builder-test.js index b05c13b..8e94fa2 100644 --- a/test/token-builder/request-builder-test.js +++ b/test/token-builder/request-builder-test.js @@ -154,7 +154,7 @@ module('RequestBuilder', function () { ]); const expectedErrors = [ - 'Errors found while parsing tokens:', + 'Request was not made due to invalid tokens. See validation errors below:', 'token 1: Missing properties for replace token: "name, value"', `token 2: ReplaceLarge token can only be used when value exceeds ${REPLACE_CHAR_LIMIT} character limit`, 'token 3: Missing properties for secret token: "path"', From 1a31e0c99750dd0799cf9b15929a2a807173d7ea Mon Sep 17 00:00:00 2001 From: Alex Guevara Date: Wed, 8 Sep 2021 16:26:42 -0400 Subject: [PATCH 06/20] update how Replace and ReplaceLarge 'value' property is validated --- src/token-builder/types.js | 12 ++++++------ test/token-builder/request-builder-test.js | 2 +- test/token-builder/types-test.js | 22 ++++++++++++++++++---- 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/src/token-builder/types.js b/src/token-builder/types.js index eed6d46..14813e7 100644 --- a/src/token-builder/types.js +++ b/src/token-builder/types.js @@ -40,9 +40,7 @@ export class ReplaceToken extends TokenBase { constructor(params) { super(params); this.type = 'replace'; - this.requiredProperties = [...this.requiredProperties, 'value']; this.value = params.value; - this.validateOptions(); } @@ -55,7 +53,9 @@ export class ReplaceToken extends TokenBase { validateOptions = () => { super.validateOptions(); - if (this.value && this.value.length > REPLACE_CHAR_LIMIT) { + if (this.value === null || this.value === undefined) { + this.errors.push('Token was not instantiated with a replace value'); + } else if (this.value && this.value.length > REPLACE_CHAR_LIMIT) { this.errors.push(`Replace value exceeds ${REPLACE_CHAR_LIMIT} character limit`); } }; @@ -65,9 +65,7 @@ export class ReplaceLargeToken extends TokenBase { constructor(params) { super(params); this.type = 'replaceLarge'; - this.requiredProperties = [...this.requiredProperties, 'value']; this.value = params.value; - this.validateOptions(); } @@ -80,7 +78,9 @@ export class ReplaceLargeToken extends TokenBase { validateOptions() { super.validateOptions(); - if (this.value && this.value.length <= REPLACE_CHAR_LIMIT) { + if (this.value === null || this.value === undefined) { + this.errors.push('Token was not instantiated with a replace value'); + } else if (this.value && this.value.length <= REPLACE_CHAR_LIMIT) { this.errors.push( `ReplaceLarge token can only be used when value exceeds ${REPLACE_CHAR_LIMIT} character limit` ); diff --git a/test/token-builder/request-builder-test.js b/test/token-builder/request-builder-test.js index 8e94fa2..5a693d5 100644 --- a/test/token-builder/request-builder-test.js +++ b/test/token-builder/request-builder-test.js @@ -155,7 +155,7 @@ module('RequestBuilder', function () { const expectedErrors = [ 'Request was not made due to invalid tokens. See validation errors below:', - 'token 1: Missing properties for replace token: "name, value"', + 'token 1: Missing properties for replace token: "name", Token was not instantiated with a replace value', `token 2: ReplaceLarge token can only be used when value exceeds ${REPLACE_CHAR_LIMIT} character limit`, 'token 3: Missing properties for secret token: "path"', 'token 4: Missing properties for hmac token: "name", HMAC algorithm is invalid, HMAC secret name not provided, HMAC encoding is invalid', diff --git a/test/token-builder/types-test.js b/test/token-builder/types-test.js index 8139f66..922c110 100644 --- a/test/token-builder/types-test.js +++ b/test/token-builder/types-test.js @@ -46,6 +46,12 @@ module('BaseToken', function () { assert.equal(tokenModel.errors.length, 1); assert.equal(tokenModel.errors[0], 'Missing properties for base token: "name"'); + + const tokenModelWithEmptyName = new TokenBase({ name: '' }); + tokenModelWithEmptyName.validateOptions(); + + assert.equal(tokenModelWithEmptyName.errors.length, 1); + assert.equal(tokenModelWithEmptyName.errors[0], 'Missing properties for base token: "name"'); }); }); @@ -90,11 +96,18 @@ module('ReplaceToken', function () { assert.deepEqual(tokenModel.toJSON(), expectedJson); }); + test('an empty string is a valid replace value', (assert) => { + const tokenModel = new ReplaceToken({ name: 'MyToken', value: '' }); + + assert.equal(tokenModel.errors.length, 0); + }); + test('will include an error if instantiated with missing options', (assert) => { const tokenModel = new ReplaceToken({}); - assert.equal(tokenModel.errors.length, 1); - assert.equal(tokenModel.errors[0], 'Missing properties for replace token: "name, value"'); + assert.equal(tokenModel.errors.length, 2); + assert.equal(tokenModel.errors[0], 'Missing properties for replace token: "name"'); + assert.equal(tokenModel.errors[1], 'Token was not instantiated with a replace value'); }); test('will include an error if value is longer than replace character limit', (assert) => { @@ -154,8 +167,9 @@ module('ReplaceLargeToken', function () { test('will include an error if instantiated with missing options', (assert) => { const tokenModel = new ReplaceLargeToken({}); - assert.equal(tokenModel.errors.length, 1); - assert.equal(tokenModel.errors[0], 'Missing properties for replaceLarge token: "name, value"'); + assert.equal(tokenModel.errors.length, 2); + assert.equal(tokenModel.errors[0], 'Missing properties for replaceLarge token: "name"'); + assert.equal(tokenModel.errors[1], 'Token was not instantiated with a replace value'); }); test('will include an error if value is shorter than replace character limit', (assert) => { From 5c5e8833c56cf6bb7aaee77b57b1ce2f18f5ac12 Mon Sep 17 00:00:00 2001 From: Alex Guevara Date: Wed, 8 Sep 2021 16:29:18 -0400 Subject: [PATCH 07/20] update version in package.jsonn --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 1e40171..a8dbe63 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@movable-internal/data-source.js", - "version": "3.0.0", + "version": "3.1.0", "main": "./dist/index.js", "module": "./dist/index.es.js", "license": "MIT", From 52792ab101fc885bb661073f0598122551839605 Mon Sep 17 00:00:00 2001 From: Alex Guevara Date: Fri, 10 Sep 2021 12:06:29 -0400 Subject: [PATCH 08/20] update readme --- README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/README.md b/README.md index f74b89c..3c09b8e 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,15 @@ Which will replace the tokens in the call and effectively the call will be `https://product-api.com/product/123/abc` + +### Passing in tokens to the request body +The `RequestBuilder` and `Token` utility classes allow users of `data-source.js` to easily pass in tokens to the body of the POST request when calling `getRawData` for an API data source. + +These tokens can be included in the `options` object as an alternative to passing them in as `targetingKeys`. The computed values of each token will replace that token when included in the data source's url, post body, or headers. + +Examples and additional details on the Token Builder API can be found in the [Wiki](https://github.com/movableink/data-source.js/wiki/Token-Builder-API +). + ### Multiple target retrieval for CSV Data Sources To fetch multiple targets from a CSV DataSource you can use the `getMultipleTargets` method, which will return you an array of objects based on the number of rows that match. @@ -330,6 +339,10 @@ $ npm publish ## Changelog +### 3.1.0 + +- Creates `RequestBuilder` and "Token" type utility classes + ### 3.0.0 - Remove `getSingleTarget` method From 84c37213a9e2de7ba527fae3914d5fa5202b03ec Mon Sep 17 00:00:00 2001 From: Alex Guevara Date: Fri, 10 Sep 2021 15:56:59 -0400 Subject: [PATCH 09/20] remove references to tokenName for hmac tokens --- test/token-builder/types-test.js | 5 ----- 1 file changed, 5 deletions(-) diff --git a/test/token-builder/types-test.js b/test/token-builder/types-test.js index 922c110..2f0cc18 100644 --- a/test/token-builder/types-test.js +++ b/test/token-builder/types-test.js @@ -227,7 +227,6 @@ module('HmacToken', function () { cacheOverride: 'xyz', skipCache: true, options: { - tokenName: 'hmac_sig', stringToSign: 'application/json\nGET\n', algorithm: 'sha1', secretName: 'watson', @@ -243,7 +242,6 @@ module('HmacToken', function () { cacheOverride: 'xyz', skipCache: true, options: { - tokenName: 'hmac_sig', stringToSign: 'application/json\nGET\n', algorithm: 'sha1', secretName: 'watson', @@ -258,7 +256,6 @@ module('HmacToken', function () { const hmacOptions = { name: 'hmac_sig', options: { - tokenName: 'hmac_sig', stringToSign: 'application/json\nGET\n', algorithm: 'sha1', secretName: 'watson', @@ -274,7 +271,6 @@ module('HmacToken', function () { cacheOverride: null, skipCache: false, options: { - tokenName: 'hmac_sig', stringToSign: 'application/json\nGET\n', algorithm: 'sha1', secretName: 'watson', @@ -289,7 +285,6 @@ module('HmacToken', function () { const hmacOptions = { name: 'hmac_sig', options: { - tokenName: 'hmac_sig', stringToSign: 'application/json\nGET\n', algorithm: 'invalid', encoding: 'neo', From 438d343ab0aa1adcb99c80294f72ab5eecc7041b Mon Sep 17 00:00:00 2001 From: MansurTsutiev Date: Mon, 13 Sep 2021 12:13:08 -0400 Subject: [PATCH 10/20] refactor: perform a single check for nullish values --- src/token-builder/types.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/token-builder/types.js b/src/token-builder/types.js index 14813e7..c880803 100644 --- a/src/token-builder/types.js +++ b/src/token-builder/types.js @@ -53,7 +53,7 @@ export class ReplaceToken extends TokenBase { validateOptions = () => { super.validateOptions(); - if (this.value === null || this.value === undefined) { + if (this.value == undefined) { this.errors.push('Token was not instantiated with a replace value'); } else if (this.value && this.value.length > REPLACE_CHAR_LIMIT) { this.errors.push(`Replace value exceeds ${REPLACE_CHAR_LIMIT} character limit`); From e7a7109f4578e94a4f58946827808b891f12784e Mon Sep 17 00:00:00 2001 From: Alex Guevara Date: Tue, 14 Sep 2021 12:55:40 -0400 Subject: [PATCH 11/20] add documentation for token-builder --- README.md | 2 +- docs/token-builder.md | 288 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 289 insertions(+), 1 deletion(-) create mode 100644 docs/token-builder.md diff --git a/README.md b/README.md index 3c09b8e..cf5c101 100644 --- a/README.md +++ b/README.md @@ -108,7 +108,7 @@ The `RequestBuilder` and `Token` utility classes allow users of `data-source.js` These tokens can be included in the `options` object as an alternative to passing them in as `targetingKeys`. The computed values of each token will replace that token when included in the data source's url, post body, or headers. -Examples and additional details on the Token Builder API can be found in the [Wiki](https://github.com/movableink/data-source.js/wiki/Token-Builder-API +[See separate docs for examples and additional details on the Token Builder API](./docs/token-builder.md) ). ### Multiple target retrieval for CSV Data Sources diff --git a/docs/token-builder.md b/docs/token-builder.md new file mode 100644 index 0000000..ab2988b --- /dev/null +++ b/docs/token-builder.md @@ -0,0 +1,288 @@ +# Token Builder API + +## Overview + +The Token Builder is comprised of two parts: the `RequestBuilder` class and a variety of `Token` type classes. + +Each of the token classes has its own set of validations which are run when the token is instantiated. Any errors are added to an `errors` property on that instance of the token class. + +In `sorcerer`, the Token Parser will extract the tokens from request body and replace any occurrence of the token in an API data source's `url`, `body`, or `headers` with the token's computed value. + +## Token Types + +The Token Builder API includes several token utility classes, each serving a unique purpose. + +Currently supported tokens are: + +- ReplaceToken +- ReplaceLargeToken +- SecretToken +- HmacToken +- Sha1Token + +### ReplaceToken + +Replaces token with the `value` included in the token options. The length of this value must be less than 100 characters. + +Example: + +```jsx +const options = { + name: 'FavoriteBand', + cacheOverride: 'Movable Band', + value: 'Beatles', + skipCache: true, +}; + +const tokenModel = new ReplaceToken(options); + +tokenModel.toJSON() // returns: +// { +// name: 'FavoriteBand', +// type: 'replace', +// cacheOverride: 'Movable Band', +// skipCache: true, +// value: 'Beatles', +// }; +``` + +### ReplaceLargeToken + +Functionally, this behaves the same as the `ReplaceToken`, with the difference being that this token should only be used if the length of the replace value is greater than a specified character limit + +**Example:** + +```jsx +const replaceValue = "some really long string" + +const options = { + name: 'FavoriteBand', + cacheOverride: 'Movable Band', + value: replaceValue, + skipCache: true, +}; + +const tokenModel = new ReplaceLargeToken(options); + +tokenModel.toJSON() // returns: +// { +// name: 'FavoriteBand', +// type: 'replace', +// cacheOverride: 'Movable Band', +// skipCache: true, +// value: 'some really long string', +// }; +``` + +### SecretToken + +Replaces token with the decrypted value of the a secret, where `path` is the name of a secret that was created for a data source in the data source wizard. + +**Example:** + +```jsx +const options = { name: 'myApiKey', path: 'watson', skipCache: true, cacheOverride: 'xyz' }; +const tokenModel = new SecretToken(options); + +// tokenModel.toJSON() returns: +// { +// name: 'myApiKey', +// type: 'secret', +// path: 'watson', +// skipCache: true, +// cacheOverride: 'xyz', +// }; + +``` + +### HmacToken + +Replaces HMAC token with an HMAC signature generated from the token options + +O**ptions:** + +- **stringToSign** + + String that represents the request & will be used when generating HMAC signature + +- **algorithm** + + The hashing algorithm: `sha1` , `sha256`, `md5` + + (for now we only support these 3, but can easily add more later if needed) + +- **secretName** + + Name of the data source secret (ex: `watson`) + +- **encoding** + + Once the signature is generated it needs to be encoded + + The following encodings are supported:`hex`, `base64`,`base64url` ,`base64percent` + + - `base64url` produces the same result as `base64` but in addition also replaces + + `+` with `-` , `/` with `_` , and removes the trailing padding character `=` + + - `base64percent` encodes the signature as `base64` and then also URI percent encodes it + +**Example:** + +```jsx +const hmacOptions = { + name: 'hmac_sig', + cacheOverride: 'xyz', + skipCache: true, + options: { + stringToSign: 'some_message', + algorithm: 'sha1', + secretName: 'watson', + encoding: 'hex', + }, +}; + +const tokenModel = new HmacToken(hmacOptions); + +tokenModel.toJSON() // returns: +// { +// name: 'hmac_sig', +// type: 'hmac', +// cacheOverride: 'xyz', +// skipCache: true, +// options: { +// stringToSign: 'some_message', +// algorithm: 'sha1', +// secretName: 'watson', +// encoding: 'hex', +// }, +// }; +``` + +### Sha1Token + +Replaces token with a SHA-1 signature generated from the token options. + +**Options:** + +- text + + The data that will be hashed to generate the signature + +- encoding + + Used to encode the result of hash function. + + Accepted values: `hex` or `base64` + +- tokens + + An array of [Secret]() tokens that could be included in the `text`. When included, these tokens will be interpolated into the `text` + +**Example:** + +```jsx +const tokens = [{ name: 'secureValue', type: 'secret', path: 'mySecretPath' }]; +const sha1Options = { + name: 'FavoriteBand', + cacheOverride: 'Movable Band', + options: { + text: 'my text', + encoding: 'hex', + tokens, + }, + skipCache: true, +}; + +const tokenModel = new Sha1Token(sha1Options); + +tokenModel.toJSON() // returns: +// { +// name: 'FavoriteBand', +// type: 'sha1', +// cacheOverride: 'Movable Band', +// skipCache: true, +// options: { +// text: 'my text', +// encoding: 'hex', +// tokens, +// }, +// }; +``` + +## **Caching** + +Each token has a couple of properties for handling how the token is included when generating a cache key: + +- **skipCache** - if set to `true`, then the token will not be considered when caching requests +- **cacheOverride** - if given a value, this will be used in place of the value of the token when generating the cache key + +By default, a `Replace` token's `value` will be included as part of the cache key. `ReplaceLarge`, `Secret`, `Hmac` and `Sha1` tokens will not be included when generating a cache key unless `cacheOverride` is supplied. + +## RequestBuilder + +This class is instantiated with an array of tokens and has a `toJSON` method which either returns a payload that can be included in the body of the request to sorcerer *or* throws an error listing validation errors from any invalid tokens. + +### Versioning + +The request builder has a `tokenApiVersion` property which will automatically be included in the request payload when calling `toJSON()`. The current namespace version is `V1`. + +### Generating a request payload + +Example + +```jsx +const replaceToken = new ReplaceToken({ + name: 'FavoriteBand', + cacheOverride: 'Movable Band', + value: 'Beatles', +}); + +const secretToken = new SecretToken({ + name: 'myApiKey', + path: 'watson', + cacheOverride: 'xyz', +}); + +const tokens = [ + replaceToken, + secretToken, +] +const requestBuilder = new RequestBuilder(); + +requestBuilder.toJSON() // returns +// { +// tokenApiVersion: 'V1', +// tokens: [ +// { +// name: 'FavoriteBand', +// type: 'replace', +// cacheOverride: 'Movable Band', +// value: 'Beatles', +// skipCache: false, +// }, +// { +// name: 'myApiKey', +// type: 'secret', +// path: 'watson', +// skipCache: false, +// cacheOverride: 'xyz', +// } +// ] +// }; +``` + +### Error Handling + +If any invalid tokens are passed into `RequestBuilder` and `toJSON()` is called, an error will be thrown which will state the validation error(s) for each token, separated by the tokens' indexes + +Example message: + +```jsx +"Error: Request was not made due to invalid tokens. See validation errors below: +token 1: Missing properties for replace token: \"name\", Token was not instantiated with a replace value +token 2: ReplaceLarge token can only be used when value exceeds 100 character limit +token 3: Missing properties for secret token: \"path\" +token 4: Missing properties for hmac token: \"name\", HMAC algorithm is invalid, HMAC secret name not provided, HMAC encoding is invalid +token 5: SHA1 encoding is invalid, Invalid secret token passed into SHA1 tokens array" +``` From da6fbdbd56866d4ef848b9bb9e639a609217d86a Mon Sep 17 00:00:00 2001 From: Alex Guevara Date: Tue, 14 Sep 2021 15:23:48 -0400 Subject: [PATCH 12/20] fix: typos, inconsistent styling --- docs/token-builder.md | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/docs/token-builder.md b/docs/token-builder.md index ab2988b..6cfb512 100644 --- a/docs/token-builder.md +++ b/docs/token-builder.md @@ -6,7 +6,7 @@ The Token Builder is comprised of two parts: the `RequestBuilder` class and a va Each of the token classes has its own set of validations which are run when the token is instantiated. Any errors are added to an `errors` property on that instance of the token class. -In `sorcerer`, the Token Parser will extract the tokens from request body and replace any occurrence of the token in an API data source's `url`, `body`, or `headers` with the token's computed value. +In `sorcerer`, the Token Parser will extract the tokens from the request body and replace any occurrence of the token in an API data source's `url`, `body`, or `headers` with the token's computed value. ## Token Types @@ -14,17 +14,17 @@ The Token Builder API includes several token utility classes, each serving a uni Currently supported tokens are: -- ReplaceToken -- ReplaceLargeToken -- SecretToken -- HmacToken -- Sha1Token +- [ReplaceToken](#replacetoken) +- [ReplaceLargeToken](#replacelargetoken) +- [SecretToken](#secrettoken) +- [HmacToken](#hmactoken) +- [Sha1Token](#sha1token) ### ReplaceToken -Replaces token with the `value` included in the token options. The length of this value must be less than 100 characters. +Replaces token with the `value` included in the token options. The length of this value must be no greater than 100 characters. -Example: +**Example:** ```jsx const options = { @@ -48,7 +48,7 @@ tokenModel.toJSON() // returns: ### ReplaceLargeToken -Functionally, this behaves the same as the `ReplaceToken`, with the difference being that this token should only be used if the length of the replace value is greater than a specified character limit +Functionally, this behaves the same as the `ReplaceToken`, with the difference being that this token should only be used if the length of the replace value is greater than 100 characters. **Example:** @@ -67,7 +67,7 @@ const tokenModel = new ReplaceLargeToken(options); tokenModel.toJSON() // returns: // { // name: 'FavoriteBand', -// type: 'replace', +// type: 'replaceLarge', // cacheOverride: 'Movable Band', // skipCache: true, // value: 'some really long string', @@ -97,9 +97,9 @@ const tokenModel = new SecretToken(options); ### HmacToken -Replaces HMAC token with an HMAC signature generated from the token options +Replaces HMAC token with an HMAC signature generated from the token options. -O**ptions:** +**Options:** - **stringToSign** @@ -119,7 +119,7 @@ O**ptions:** Once the signature is generated it needs to be encoded - The following encodings are supported:`hex`, `base64`,`base64url` ,`base64percent` + The following encodings are supported: `hex`, `base64`, `base64url`, `base64percent` - `base64url` produces the same result as `base64` but in addition also replaces @@ -177,7 +177,7 @@ Replaces token with a SHA-1 signature generated from the token options. - tokens - An array of [Secret]() tokens that could be included in the `text`. When included, these tokens will be interpolated into the `text` + An array of [Secret](#secrettoken) tokens that could be included in the `text`. When included, these tokens will be interpolated into the `text` **Example:** @@ -229,7 +229,7 @@ The request builder has a `tokenApiVersion` property which will automatically be ### Generating a request payload -Example +**Example:** ```jsx const replaceToken = new ReplaceToken({ @@ -276,7 +276,7 @@ requestBuilder.toJSON() // returns If any invalid tokens are passed into `RequestBuilder` and `toJSON()` is called, an error will be thrown which will state the validation error(s) for each token, separated by the tokens' indexes -Example message: +**Example message:** ```jsx "Error: Request was not made due to invalid tokens. See validation errors below: From bd4724d285df0905c691853545333d6402350240 Mon Sep 17 00:00:00 2001 From: Alex Guevara Date: Tue, 14 Sep 2021 15:44:56 -0400 Subject: [PATCH 13/20] chore: update token examples in docs, add examples to caching section --- docs/token-builder.md | 76 +++++++++++++++++++++++++++++++------------ 1 file changed, 56 insertions(+), 20 deletions(-) diff --git a/docs/token-builder.md b/docs/token-builder.md index 6cfb512..bcee212 100644 --- a/docs/token-builder.md +++ b/docs/token-builder.md @@ -29,9 +29,7 @@ Replaces token with the `value` included in the token options. The length of thi ```jsx const options = { name: 'FavoriteBand', - cacheOverride: 'Movable Band', value: 'Beatles', - skipCache: true, }; const tokenModel = new ReplaceToken(options); @@ -40,8 +38,8 @@ tokenModel.toJSON() // returns: // { // name: 'FavoriteBand', // type: 'replace', -// cacheOverride: 'Movable Band', -// skipCache: true, +// cacheOverride: null, +// skipCache: false, // value: 'Beatles', // }; ``` @@ -57,9 +55,7 @@ const replaceValue = "some really long string" const options = { name: 'FavoriteBand', - cacheOverride: 'Movable Band', value: replaceValue, - skipCache: true, }; const tokenModel = new ReplaceLargeToken(options); @@ -68,8 +64,8 @@ tokenModel.toJSON() // returns: // { // name: 'FavoriteBand', // type: 'replaceLarge', -// cacheOverride: 'Movable Band', -// skipCache: true, +// cacheOverride: null, +// skipCache: false, // value: 'some really long string', // }; ``` @@ -81,7 +77,7 @@ Replaces token with the decrypted value of the a secret, where `path` is the nam **Example:** ```jsx -const options = { name: 'myApiKey', path: 'watson', skipCache: true, cacheOverride: 'xyz' }; +const options = { name: 'myApiKey', path: 'watson' }; const tokenModel = new SecretToken(options); // tokenModel.toJSON() returns: @@ -89,8 +85,8 @@ const tokenModel = new SecretToken(options); // name: 'myApiKey', // type: 'secret', // path: 'watson', -// skipCache: true, -// cacheOverride: 'xyz', +// cacheOverride: null, +// skipCache: false, // }; ``` @@ -132,8 +128,6 @@ Replaces HMAC token with an HMAC signature generated from the token options. ```jsx const hmacOptions = { name: 'hmac_sig', - cacheOverride: 'xyz', - skipCache: true, options: { stringToSign: 'some_message', algorithm: 'sha1', @@ -148,8 +142,8 @@ tokenModel.toJSON() // returns: // { // name: 'hmac_sig', // type: 'hmac', -// cacheOverride: 'xyz', -// skipCache: true, +// cacheOverride: null, +// skipCache: false, // options: { // stringToSign: 'some_message', // algorithm: 'sha1', @@ -185,13 +179,11 @@ Replaces token with a SHA-1 signature generated from the token options. const tokens = [{ name: 'secureValue', type: 'secret', path: 'mySecretPath' }]; const sha1Options = { name: 'FavoriteBand', - cacheOverride: 'Movable Band', options: { text: 'my text', encoding: 'hex', tokens, }, - skipCache: true, }; const tokenModel = new Sha1Token(sha1Options); @@ -200,8 +192,8 @@ tokenModel.toJSON() // returns: // { // name: 'FavoriteBand', // type: 'sha1', -// cacheOverride: 'Movable Band', -// skipCache: true, +// cacheOverride: null, +// skipCache: false, // options: { // text: 'my text', // encoding: 'hex', @@ -217,7 +209,51 @@ Each token has a couple of properties for handling how the token is included whe - **skipCache** - if set to `true`, then the token will not be considered when caching requests - **cacheOverride** - if given a value, this will be used in place of the value of the token when generating the cache key -By default, a `Replace` token's `value` will be included as part of the cache key. `ReplaceLarge`, `Secret`, `Hmac` and `Sha1` tokens will not be included when generating a cache key unless `cacheOverride` is supplied. +**Examples:** + +Setting `skipCache` to `true` for a ReplaceToken +```jsx +const options = { + name: 'FavoriteBand', + value: 'Beatles', + skipCache: true +}; + +const tokenModel = new ReplaceToken(options); + +tokenModel.toJSON() // returns: +// { +// name: 'FavoriteBand', +// type: 'replace', +// cacheOverride: null, +// skipCache: true, +// value: 'Beatles', +// }; +``` + +Setting a `cacheOverride` for a ReplaceLarge token +```jsx +const replaceValue = "some really long string" + +const options = { + name: 'SongLyrics', + value: replaceValue, + cacheOverride: 'name of song' +}; + +const tokenModel = new ReplaceLargeToken(options); + +tokenModel.toJSON() // returns: +// { +// name: 'SongLyrics', +// type: 'replaceLarge', +// cacheOverride: 'name of song', +// skipCache: false, +// value: 'some really long string', +// }; +``` + +Note: By default, a `Replace` token's `value` will be included as part of the cache key. `ReplaceLarge`, `Secret`, `Hmac` and `Sha1` tokens will not be included when generating a cache key unless `cacheOverride` is supplied. ## RequestBuilder From 7529a6f19ded0b24174c1496c6e70c53fc16c509 Mon Sep 17 00:00:00 2001 From: Alex Guevara Date: Tue, 14 Sep 2021 17:38:26 -0400 Subject: [PATCH 14/20] chore: add params section to documentation --- docs/token-builder.md | 202 ++++++++++++------------------------------ 1 file changed, 56 insertions(+), 146 deletions(-) diff --git a/docs/token-builder.md b/docs/token-builder.md index bcee212..afc8172 100644 --- a/docs/token-builder.md +++ b/docs/token-builder.md @@ -2,16 +2,23 @@ ## Overview -The Token Builder is comprised of two parts: the `RequestBuilder` class and a variety of `Token` type classes. +The Token Builder is comprised of two parts: the `RequestBuilder` class and a variety of `Token` classes. Each of the token classes has its own set of validations which are run when the token is instantiated. Any errors are added to an `errors` property on that instance of the token class. In `sorcerer`, the Token Parser will extract the tokens from the request body and replace any occurrence of the token in an API data source's `url`, `body`, or `headers` with the token's computed value. -## Token Types +## Token Classes The Token Builder API includes several token utility classes, each serving a unique purpose. +Each token is instantiated with the following params: +- **name** (required) - the name of the token +- **skipCache** (optional) - if set to `true`, then the token will not be considered when caching requests +- **cacheOverride** (optional) - when provided, this value will be used when caching the requests + +In addition, each type of token will have its own unique set of properties it should be instantiated with, detailed below. + Currently supported tokens are: - [ReplaceToken](#replacetoken) @@ -20,113 +27,72 @@ Currently supported tokens are: - [HmacToken](#hmactoken) - [Sha1Token](#sha1token) + ### ReplaceToken +Replaces token with the provided value. -Replaces token with the `value` included in the token options. The length of this value must be no greater than 100 characters. +**Params** +- **value** (required) - the replace value of the token. The length of this value must be no greater than 100 characters. **Example:** ```jsx -const options = { +const params = { name: 'FavoriteBand', value: 'Beatles', }; -const tokenModel = new ReplaceToken(options); - -tokenModel.toJSON() // returns: -// { -// name: 'FavoriteBand', -// type: 'replace', -// cacheOverride: null, -// skipCache: false, -// value: 'Beatles', -// }; +const tokenModel = new ReplaceToken(params); ``` ### ReplaceLargeToken +Replaces token with the provided value. -Functionally, this behaves the same as the `ReplaceToken`, with the difference being that this token should only be used if the length of the replace value is greater than 100 characters. +**Params** +- **value** (required)- the replace value of the token. The length of this value must be greater than 100 characters. **Example:** ```jsx const replaceValue = "some really long string" -const options = { +const params = { name: 'FavoriteBand', value: replaceValue, }; -const tokenModel = new ReplaceLargeToken(options); - -tokenModel.toJSON() // returns: -// { -// name: 'FavoriteBand', -// type: 'replaceLarge', -// cacheOverride: null, -// skipCache: false, -// value: 'some really long string', -// }; +const tokenModel = new ReplaceLargeToken(params); ``` ### SecretToken +Replaces token with the decrypted value of a secret. -Replaces token with the decrypted value of the a secret, where `path` is the name of a secret that was created for a data source in the data source wizard. +**Params** +- **path** (required) - the name of a secret that was created for a data source in the data source wizard. **Example:** ```jsx -const options = { name: 'myApiKey', path: 'watson' }; -const tokenModel = new SecretToken(options); - -// tokenModel.toJSON() returns: -// { -// name: 'myApiKey', -// type: 'secret', -// path: 'watson', -// cacheOverride: null, -// skipCache: false, -// }; - +const params = { name: 'myApiKey', path: 'watson' }; +const tokenModel = new SecretToken(params); ``` ### HmacToken +Replaces token with an HMAC signature. -Replaces HMAC token with an HMAC signature generated from the token options. - -**Options:** - -- **stringToSign** - - String that represents the request & will be used when generating HMAC signature - -- **algorithm** - - The hashing algorithm: `sha1` , `sha256`, `md5` - - (for now we only support these 3, but can easily add more later if needed) - -- **secretName** - - Name of the data source secret (ex: `watson`) - -- **encoding** - - Once the signature is generated it needs to be encoded - - The following encodings are supported: `hex`, `base64`, `base64url`, `base64percent` - - - `base64url` produces the same result as `base64` but in addition also replaces - - `+` with `-` , `/` with `_` , and removes the trailing padding character `=` - - - `base64percent` encodes the signature as `base64` and then also URI percent encodes it +**Params** +- **options** (required) + - **stringToSign** (optional) - any string that will be used when generating HMAC signature + - **algorithm** (required)- the hashing algorithm: `sha1` , `sha256`, `md5` + - **secretName** (required) - name of the data source secret (ex: `watson`) + - **encoding** (required) - option to encode the signature once it is generated: `hex`, `base64`, `base64url`, `base64percent` + - `base64url` produces the same result as `base64` but in addition also replaces `+` with `-` , `/` with `_` , and removes the trailing padding character `=` + - `base64percent` encodes the signature as `base64` and then also URI percent encodes it **Example:** ```jsx -const hmacOptions = { +const params = { name: 'hmac_sig', options: { stringToSign: 'some_message', @@ -136,48 +102,24 @@ const hmacOptions = { }, }; -const tokenModel = new HmacToken(hmacOptions); - -tokenModel.toJSON() // returns: -// { -// name: 'hmac_sig', -// type: 'hmac', -// cacheOverride: null, -// skipCache: false, -// options: { -// stringToSign: 'some_message', -// algorithm: 'sha1', -// secretName: 'watson', -// encoding: 'hex', -// }, -// }; +const tokenModel = new HmacToken(params); ``` ### Sha1Token -Replaces token with a SHA-1 signature generated from the token options. - -**Options:** - -- text +Replaces token with a SHA-1 signature. - The data that will be hashed to generate the signature - -- encoding - - Used to encode the result of hash function. - - Accepted values: `hex` or `base64` - -- tokens - - An array of [Secret](#secrettoken) tokens that could be included in the `text`. When included, these tokens will be interpolated into the `text` +**Params** +- **options** (required) + - **text** (required) - the data that will be hashed to generate the signature + - **encoding** (required) - option used to encode the result of hash function: `hex`, `base64` + - **tokens** (optional) - an array of [Secret](#secrettoken) token params that could be included in the `text`. When included, these tokens will be interpolated into the `text` **Example:** ```jsx const tokens = [{ name: 'secureValue', type: 'secret', path: 'mySecretPath' }]; -const sha1Options = { +const params = { name: 'FavoriteBand', options: { text: 'my text', @@ -186,75 +128,43 @@ const sha1Options = { }, }; -const tokenModel = new Sha1Token(sha1Options); - -tokenModel.toJSON() // returns: -// { -// name: 'FavoriteBand', -// type: 'sha1', -// cacheOverride: null, -// skipCache: false, -// options: { -// text: 'my text', -// encoding: 'hex', -// tokens, -// }, -// }; +const tokenModel = new Sha1Token(params); ``` ## **Caching** +By default, a `Replace` token's `value` will be used to cache requests. `ReplaceLarge`, `Secret`, `Hmac` and `Sha1` tokens will not be included when caching requests unless `cacheOverride` is supplied. -Each token has a couple of properties for handling how the token is included when generating a cache key: - -- **skipCache** - if set to `true`, then the token will not be considered when caching requests -- **cacheOverride** - if given a value, this will be used in place of the value of the token when generating the cache key +The `skipCache` and `cacheOverride` properties can be used to customize how the token is handled when caching requests: **Examples:** -Setting `skipCache` to `true` for a ReplaceToken +Setting `skipCache` to `true` for a token will ensure that the token will not be used to cache requests, even if a `cacheOverride` is set. + ```jsx -const options = { +const params = { name: 'FavoriteBand', value: 'Beatles', - skipCache: true + cacheOverride: 'My Favorite Band' + skipCache: true, }; -const tokenModel = new ReplaceToken(options); - -tokenModel.toJSON() // returns: -// { -// name: 'FavoriteBand', -// type: 'replace', -// cacheOverride: null, -// skipCache: true, -// value: 'Beatles', -// }; +const tokenModel = new ReplaceToken(params); ``` -Setting a `cacheOverride` for a ReplaceLarge token +Setting a `cacheOverride` for any token will ignore default behavior and will use the value of `cacheOverride` when caching requests. + ```jsx const replaceValue = "some really long string" -const options = { +const params = { name: 'SongLyrics', value: replaceValue, cacheOverride: 'name of song' }; -const tokenModel = new ReplaceLargeToken(options); - -tokenModel.toJSON() // returns: -// { -// name: 'SongLyrics', -// type: 'replaceLarge', -// cacheOverride: 'name of song', -// skipCache: false, -// value: 'some really long string', -// }; +const tokenModel = new ReplaceLargeToken(params); ``` -Note: By default, a `Replace` token's `value` will be included as part of the cache key. `ReplaceLarge`, `Secret`, `Hmac` and `Sha1` tokens will not be included when generating a cache key unless `cacheOverride` is supplied. - ## RequestBuilder This class is instantiated with an array of tokens and has a `toJSON` method which either returns a payload that can be included in the body of the request to sorcerer *or* throws an error listing validation errors from any invalid tokens. From 2d037f6e705153b4f09368a350cf58a0f22700b0 Mon Sep 17 00:00:00 2001 From: Alex Guevara Date: Wed, 15 Sep 2021 10:43:54 -0400 Subject: [PATCH 15/20] chore: add char limit validation for cacheOverride --- src/token-builder/types.js | 14 ++++--- test/token-builder/request-builder-test.js | 8 ++-- test/token-builder/types-test.js | 46 +++++++++++++++++----- 3 files changed, 49 insertions(+), 19 deletions(-) diff --git a/src/token-builder/types.js b/src/token-builder/types.js index c880803..a4cb31e 100644 --- a/src/token-builder/types.js +++ b/src/token-builder/types.js @@ -1,7 +1,7 @@ const ALLOWED_ALGOS = new Set(['sha256', 'sha1', 'md5']); const ALLOWED_ENCODINGS = new Set(['hex', 'base64', 'base64url', 'base64percent']); -export const REPLACE_CHAR_LIMIT = 100; +export const CHAR_LIMIT = 100; export class TokenBase { constructor({ name, cacheOverride = null, skipCache = false }) { @@ -33,6 +33,10 @@ export class TokenBase { if (missingProps.length) { this.errors.push(`Missing properties for ${this.type} token: "${missingProps.join(', ')}"`); } + + if (typeof this.cacheOverride === 'string' && this.cacheOverride.length > CHAR_LIMIT) { + this.errors.push(`cacheOverride cannot be over ${CHAR_LIMIT} characters`); + } } } @@ -55,8 +59,8 @@ export class ReplaceToken extends TokenBase { if (this.value == undefined) { this.errors.push('Token was not instantiated with a replace value'); - } else if (this.value && this.value.length > REPLACE_CHAR_LIMIT) { - this.errors.push(`Replace value exceeds ${REPLACE_CHAR_LIMIT} character limit`); + } else if (this.value && this.value.length > CHAR_LIMIT) { + this.errors.push(`Replace value exceeds ${CHAR_LIMIT} character limit`); } }; } @@ -80,9 +84,9 @@ export class ReplaceLargeToken extends TokenBase { if (this.value === null || this.value === undefined) { this.errors.push('Token was not instantiated with a replace value'); - } else if (this.value && this.value.length <= REPLACE_CHAR_LIMIT) { + } else if (this.value && this.value.length <= CHAR_LIMIT) { this.errors.push( - `ReplaceLarge token can only be used when value exceeds ${REPLACE_CHAR_LIMIT} character limit` + `ReplaceLarge token can only be used when value exceeds ${CHAR_LIMIT} character limit` ); } } diff --git a/test/token-builder/request-builder-test.js b/test/token-builder/request-builder-test.js index 5a693d5..ee03118 100644 --- a/test/token-builder/request-builder-test.js +++ b/test/token-builder/request-builder-test.js @@ -1,5 +1,5 @@ import { RequestBuilder } from '../../src/token-builder/request-builder'; -import { REPLACE_CHAR_LIMIT } from '../../src/token-builder/types'; +import { CHAR_LIMIT } from '../../src/token-builder/types'; import { ReplaceToken, ReplaceLargeToken, @@ -20,7 +20,7 @@ module('RequestBuilder', function () { const replaceLargeToken = new ReplaceLargeToken({ name: 'band', cacheOverride: 'flooding', - value: '*'.repeat(REPLACE_CHAR_LIMIT + 1), + value: '*'.repeat(CHAR_LIMIT + 1), skipCache: true, }); @@ -73,7 +73,7 @@ module('RequestBuilder', function () { name: 'band', type: 'replaceLarge', cacheOverride: 'flooding', - value: '*'.repeat(REPLACE_CHAR_LIMIT + 1), + value: '*'.repeat(CHAR_LIMIT + 1), skipCache: true, }, { @@ -156,7 +156,7 @@ module('RequestBuilder', function () { const expectedErrors = [ 'Request was not made due to invalid tokens. See validation errors below:', 'token 1: Missing properties for replace token: "name", Token was not instantiated with a replace value', - `token 2: ReplaceLarge token can only be used when value exceeds ${REPLACE_CHAR_LIMIT} character limit`, + `token 2: ReplaceLarge token can only be used when value exceeds ${CHAR_LIMIT} character limit`, 'token 3: Missing properties for secret token: "path"', 'token 4: Missing properties for hmac token: "name", HMAC algorithm is invalid, HMAC secret name not provided, HMAC encoding is invalid', 'token 5: SHA1 encoding is invalid, Invalid secret token passed into SHA1 tokens array', diff --git a/test/token-builder/types-test.js b/test/token-builder/types-test.js index 2f0cc18..4deba1b 100644 --- a/test/token-builder/types-test.js +++ b/test/token-builder/types-test.js @@ -5,7 +5,7 @@ import { SecretToken, HmacToken, Sha1Token, - REPLACE_CHAR_LIMIT, + CHAR_LIMIT, } from '../../src/token-builder/types'; const { test, module } = QUnit; @@ -53,11 +53,40 @@ module('BaseToken', function () { assert.equal(tokenModelWithEmptyName.errors.length, 1); assert.equal(tokenModelWithEmptyName.errors[0], 'Missing properties for base token: "name"'); }); + + test('will include an error if cacheOverride is over the character limit', (assert) => { + { + const options = { + name: 'FavoriteBand', + type: 'base', + cacheOverride: '*'.repeat(CHAR_LIMIT), + }; + + const token1 = new TokenBase(options); + token1.validateOptions(); + + assert.equal(token1.errors.length, 0); + } + + { + const options = { + name: 'FavoriteBand', + type: 'base', + cacheOverride: '*'.repeat(CHAR_LIMIT + 1), + }; + + const token1 = new TokenBase(options); + token1.validateOptions(); + + assert.equal(token1.errors.length, 1); + assert.equal(token1.errors[0], `cacheOverride cannot be over ${CHAR_LIMIT} characters`); + } + }); }); module('ReplaceToken', function () { test('can be instantiated with all options', (assert) => { - const replaceValue = '*'.repeat(REPLACE_CHAR_LIMIT); + const replaceValue = '*'.repeat(CHAR_LIMIT); const options = { name: 'FavoriteBand', @@ -111,20 +140,17 @@ module('ReplaceToken', function () { }); test('will include an error if value is longer than replace character limit', (assert) => { - const value = '*'.repeat(REPLACE_CHAR_LIMIT + 1); + const value = '*'.repeat(CHAR_LIMIT + 1); const tokenModel = new ReplaceToken({ name: 'my token', value }); assert.equal(tokenModel.errors.length, 1); - assert.equal( - tokenModel.errors[0], - `Replace value exceeds ${REPLACE_CHAR_LIMIT} character limit` - ); + assert.equal(tokenModel.errors[0], `Replace value exceeds ${CHAR_LIMIT} character limit`); }); }); module('ReplaceLargeToken', function () { test('can be instantiated with all options', (assert) => { - const replaceValue = '*'.repeat(REPLACE_CHAR_LIMIT + 1); + const replaceValue = '*'.repeat(CHAR_LIMIT + 1); const options = { name: 'FavoriteBand', @@ -146,7 +172,7 @@ module('ReplaceLargeToken', function () { }); test('can be instantiated with default options', (assert) => { - const replaceValue = '*'.repeat(REPLACE_CHAR_LIMIT + 1); + const replaceValue = '*'.repeat(CHAR_LIMIT + 1); const options = { name: 'FavoriteBand', value: replaceValue, @@ -178,7 +204,7 @@ module('ReplaceLargeToken', function () { assert.equal(tokenModel.errors.length, 1); assert.equal( tokenModel.errors[0], - `ReplaceLarge token can only be used when value exceeds ${REPLACE_CHAR_LIMIT} character limit` + `ReplaceLarge token can only be used when value exceeds ${CHAR_LIMIT} character limit` ); }); }); From 731c75676f18d0bf6268c52ba2e6b82f7c9336a5 Mon Sep 17 00:00:00 2001 From: Alex Guevara Date: Thu, 16 Sep 2021 10:38:27 -0400 Subject: [PATCH 16/20] update check for valid cacheOverride --- src/token-builder/types.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/token-builder/types.js b/src/token-builder/types.js index a4cb31e..0743109 100644 --- a/src/token-builder/types.js +++ b/src/token-builder/types.js @@ -34,7 +34,7 @@ export class TokenBase { this.errors.push(`Missing properties for ${this.type} token: "${missingProps.join(', ')}"`); } - if (typeof this.cacheOverride === 'string' && this.cacheOverride.length > CHAR_LIMIT) { + if (this.cacheOverride && this.cacheOverride.length > CHAR_LIMIT) { this.errors.push(`cacheOverride cannot be over ${CHAR_LIMIT} characters`); } } From 09ec1c59adf78631e7928736923e29f1194dc726 Mon Sep 17 00:00:00 2001 From: Alex Guevara Date: Fri, 17 Sep 2021 15:45:46 -0400 Subject: [PATCH 17/20] update documentation - update README table of contents changelog to link to new version - update Token Builder docs with toc, new examples - Co-authored-by: Mansur Tsutiev --- README.md | 6 +- docs/token-builder.md | 179 +++++++++++++++++++++++++++++++----------- 2 files changed, 134 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index cf5c101..effb7bb 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ Data Source is a JS library meant to help developers access Movable Ink Data Sou - [Details on how Sorcerer determines priority](#details-on-how-sorcerer-determines-priority) - [Publishing package:](#publishing-package) - [Changelog](#changelog) + - [3.1.0](#310) - [3.0.0](#300) - [2.0.0](#200) - [1.0.0](#100) @@ -79,7 +80,7 @@ const { data } = await source.getRawData(targetingKeys, options); The `key` above is supposed to be a unique identifier that refers to the data source that you are trying to receive raw data from. You can find this key in the Movable Ink platform. -#### Example: +#### Example If the API DS is set up with the following URL @@ -108,8 +109,7 @@ The `RequestBuilder` and `Token` utility classes allow users of `data-source.js` These tokens can be included in the `options` object as an alternative to passing them in as `targetingKeys`. The computed values of each token will replace that token when included in the data source's url, post body, or headers. -[See separate docs for examples and additional details on the Token Builder API](./docs/token-builder.md) -). +[See separate docs for examples and additional details on the Token Builder API](./docs/token-builder.md). ### Multiple target retrieval for CSV Data Sources diff --git a/docs/token-builder.md b/docs/token-builder.md index afc8172..f063b00 100644 --- a/docs/token-builder.md +++ b/docs/token-builder.md @@ -1,21 +1,33 @@ # Token Builder API - ## Overview +The Token Builder API provides a means of substituting tokens within an API data source's url, request headers, and post body with dynamic values. -The Token Builder is comprised of two parts: the `RequestBuilder` class and a variety of `Token` classes. - -Each of the token classes has its own set of validations which are run when the token is instantiated. Any errors are added to an `errors` property on that instance of the token class. +The Token Builder is comprised of two parts: the `RequestBuilder` class and a variety of `Token` classes. Each of the token classes has its own set of validations which are run when the token is instantiated. Any errors are added to an `errors` property on that instance of the token class. In `sorcerer`, the Token Parser will extract the tokens from the request body and replace any occurrence of the token in an API data source's `url`, `body`, or `headers` with the token's computed value. +## Table of Contents +- [Overview](#overview) +- [Token Classes](#token-classes) + - [ReplaceToken](#replacetoken) + - [ReplaceLargeToken](#replacelargetoken) + - [SecretToken](#secrettoken) + - [HmacToken](#hmactoken) + - [Sha1Token](#sha1token) +- [RequestBuilder](#requestbuilder) + - [Generating a request payload](#generating-a-request-payload) + - [Error handling](#error-handling) +- [Making a request](#making-a-request) +- [Caching](#caching) + ## Token Classes The Token Builder API includes several token utility classes, each serving a unique purpose. -Each token is instantiated with the following params: +Tokens are instantiated with the following params: - **name** (required) - the name of the token - **skipCache** (optional) - if set to `true`, then the token will not be considered when caching requests -- **cacheOverride** (optional) - when provided, this value will be used when caching the requests +- **cacheOverride** (optional) - when provided, this value will be used when caching requests In addition, each type of token will have its own unique set of properties it should be instantiated with, detailed below. @@ -54,10 +66,10 @@ Replaces token with the provided value. **Example:** ```jsx -const replaceValue = "some really long string" +const replaceValue = "some really long string"; const params = { - name: 'FavoriteBand', + name: 'SongLyrics', value: replaceValue, }; @@ -84,7 +96,7 @@ Replaces token with an HMAC signature. - **options** (required) - **stringToSign** (optional) - any string that will be used when generating HMAC signature - **algorithm** (required)- the hashing algorithm: `sha1` , `sha256`, `md5` - - **secretName** (required) - name of the data source secret (ex: `watson`) + - **secretName** (required) - name of the data source secret (e.g. `watson`) - **encoding** (required) - option to encode the signature once it is generated: `hex`, `base64`, `base64url`, `base64percent` - `base64url` produces the same result as `base64` but in addition also replaces `+` with `-` , `/` with `_` , and removes the trailing padding character `=` - `base64percent` encodes the signature as `base64` and then also URI percent encodes it @@ -113,16 +125,18 @@ Replaces token with a SHA-1 signature. - **options** (required) - **text** (required) - the data that will be hashed to generate the signature - **encoding** (required) - option used to encode the result of hash function: `hex`, `base64` - - **tokens** (optional) - an array of [Secret](#secrettoken) token params that could be included in the `text`. When included, these tokens will be interpolated into the `text` + - **tokens** (optional) - an array of [Secret](#secrettoken) token params that could be included in the `text`. When included, these tokens will be interpolated into the `text`. **Example:** +If the `text` includes tokens, then then `tokens` array must include secret params corresponding to those tokens. + ```jsx const tokens = [{ name: 'secureValue', type: 'secret', path: 'mySecretPath' }]; const params = { - name: 'FavoriteBand', + name: 'sha1_sig', options: { - text: 'my text', + text: 'the_secret_is_[secureValue]', encoding: 'hex', tokens, }, @@ -131,49 +145,28 @@ const params = { const tokenModel = new Sha1Token(params); ``` -## **Caching** -By default, a `Replace` token's `value` will be used to cache requests. `ReplaceLarge`, `Secret`, `Hmac` and `Sha1` tokens will not be included when caching requests unless `cacheOverride` is supplied. - -The `skipCache` and `cacheOverride` properties can be used to customize how the token is handled when caching requests: - -**Examples:** +If the text does not include a token then the `tokens` array can be left out of the params object when instantiating the Sha1Token. -Setting `skipCache` to `true` for a token will ensure that the token will not be used to cache requests, even if a `cacheOverride` is set. - -```jsx -const params = { - name: 'FavoriteBand', - value: 'Beatles', - cacheOverride: 'My Favorite Band' - skipCache: true, -}; - -const tokenModel = new ReplaceToken(params); ``` - -Setting a `cacheOverride` for any token will ignore default behavior and will use the value of `cacheOverride` when caching requests. - -```jsx -const replaceValue = "some really long string" - const params = { - name: 'SongLyrics', - value: replaceValue, - cacheOverride: 'name of song' + name: 'sha1_sig', + options: { + text: 'mystring', + encoding: 'base64' + }, }; -const tokenModel = new ReplaceLargeToken(params); +const tokenModel = new Sha1Token(params); ``` ## RequestBuilder This class is instantiated with an array of tokens and has a `toJSON` method which either returns a payload that can be included in the body of the request to sorcerer *or* throws an error listing validation errors from any invalid tokens. -### Versioning - -The request builder has a `tokenApiVersion` property which will automatically be included in the request payload when calling `toJSON()`. The current namespace version is `V1`. - ### Generating a request payload +Calling `toJSON` will return an object with two properties: `tokenApiVersion` and `tokens`. +- `tokenApiVersion` is an internal property that is automatically set. The current namespace version is `V1`. +- `tokens` is an array of POJOS representing the valid tokens. **Example:** @@ -194,7 +187,8 @@ const tokens = [ replaceToken, secretToken, ] -const requestBuilder = new RequestBuilder(); + +const requestBuilder = new RequestBuilder(tokens); requestBuilder.toJSON() // returns // { @@ -203,16 +197,16 @@ requestBuilder.toJSON() // returns // { // name: 'FavoriteBand', // type: 'replace', -// cacheOverride: 'Movable Band', // value: 'Beatles', // skipCache: false, +// cacheOverride: 'Movable Band' // }, // { // name: 'myApiKey', // type: 'secret', // path: 'watson', // skipCache: false, -// cacheOverride: 'xyz', +// cacheOverride: 'xyz' // } // ] // }; @@ -220,9 +214,9 @@ requestBuilder.toJSON() // returns ### Error Handling -If any invalid tokens are passed into `RequestBuilder` and `toJSON()` is called, an error will be thrown which will state the validation error(s) for each token, separated by the tokens' indexes +If any invalid tokens are passed into `RequestBuilder` and `toJSON` is called, an error will be thrown which will state the validation error(s) for each token, separated by the tokens' indexes. -**Example message:** +**Example:** ```jsx "Error: Request was not made due to invalid tokens. See validation errors below: @@ -232,3 +226,92 @@ token 3: Missing properties for secret token: \"path\" token 4: Missing properties for hmac token: \"name\", HMAC algorithm is invalid, HMAC secret name not provided, HMAC encoding is invalid token 5: SHA1 encoding is invalid, Invalid secret token passed into SHA1 tokens array" ``` + +## Making a request +To use the Token Builder API in a custom app, `RequestBuilder` and the relevant token classes can be imported from`data-source.js` + +``` +import DataSource, { + RequestBuilder, + ReplaceToken, + SecretToken, + HmacToken +} from '@movable-internal/data-source.js'; +``` + +When setting a property, a `RequestBuilder` can be instantiated with tokens and then used to generate the body for the POST request to Sorcerer using `getRawData`. + +``` +app.setProperty('apiData', async () => { + const ds = new DataSource(''); + + const replaceToken = new ReplaceToken({ name: 'breed', value: 'chow' }); + const secretToken = new SecretToken({ name: 'credentials', path: 'api_key' }); + + const hmacParams = { + name: 'hmac_sig', + options: { + stringToSign: 'mystring', + algorithm: 'sha1', + secretName: 'api_key', + encoding: 'base64', + }, + }; + + const hmacToken = new HmacToken(hmacParams); + + const tokens = [replaceToken, secretToken, hmacToken]; + const requestBuilder = new RequestBuilder(tokens); + const postBody = JSON.stringify(requestBuilder.toJSON()); + + const options = { + method: 'POST', // method has to be POST + body: postBody, + }; + + let { data } = await ds.getRawData({}, options); + ... +} + +``` + +**Note on error handling** + +Even if a valid request is made to Sorcerer using the `RequestBuilder`, Sorcerer has additional validations which will throw an error in the back end. In this situation, calling `getRawData` will return a non-200 `status` code and the value of `data` will be the error message from Sorcerer. + +## **Caching** +By default, a `Replace` token's `value` will be used to cache requests. `ReplaceLarge`, `Secret`, `Hmac` and `Sha1` tokens will not be included when caching requests unless `cacheOverride` is supplied. + +The `skipCache` and `cacheOverride` properties can be used to customize how the token is handled when caching requests. + +**Note**: the value of `cacheOverride` must be less than 100 characters. + +**Examples:** + +Setting a `cacheOverride` for any token will ignore default behavior and will use the value of `cacheOverride` when caching requests. + +```jsx +const replaceValue = "some really long string"; + +const params = { + name: 'SongLyrics', + value: replaceValue, + cacheOverride: 'name of song' +}; + +const tokenModel = new ReplaceLargeToken(params); +``` + +Setting `skipCache` to `true` for a token will ensure that the token will not be used to cache requests, even if a `cacheOverride` is set. + + +```jsx +const params = { + name: 'FavoriteBand', + value: 'Beatles', + cacheOverride: 'My Favorite Band' + skipCache: true, +}; + +const tokenModel = new ReplaceToken(params); +``` From 2990bf492687d9b70b3ec4096448012201df52a0 Mon Sep 17 00:00:00 2001 From: Alex Guevara Date: Fri, 17 Sep 2021 15:48:11 -0400 Subject: [PATCH 18/20] export TokenBuilder classes from index.js Co-authored-by: Mansur Tsutiev --- src/index.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/index.ts b/src/index.ts index 6ea06c3..90388ce 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,5 +1,13 @@ import CD from 'cropduster'; import { CDResponse } from '../types/cropduster'; +export { RequestBuilder } from './token-builder/request-builder'; +export { + ReplaceToken, + ReplaceLargeToken, + SecretToken, + HmacToken, + Sha1Token, +} from './token-builder/types'; export interface TargetingParams { [key: string]: string | number | Object[] | Object; From 0abd844aa2bbff8eb6191d18e6ee33a5c6809fd1 Mon Sep 17 00:00:00 2001 From: Alex Guevara Date: Fri, 17 Sep 2021 15:49:29 -0400 Subject: [PATCH 19/20] refactor: perform a single check for nullish values for ReplaceLarge token --- src/token-builder/types.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/token-builder/types.js b/src/token-builder/types.js index 0743109..234dcfa 100644 --- a/src/token-builder/types.js +++ b/src/token-builder/types.js @@ -82,7 +82,7 @@ export class ReplaceLargeToken extends TokenBase { validateOptions() { super.validateOptions(); - if (this.value === null || this.value === undefined) { + if (this.value == undefined) { this.errors.push('Token was not instantiated with a replace value'); } else if (this.value && this.value.length <= CHAR_LIMIT) { this.errors.push( From ed84ec1131d337627e3766c385c95a1a9d6eb023 Mon Sep 17 00:00:00 2001 From: MansurTsutiev Date: Mon, 20 Sep 2021 18:18:03 -0400 Subject: [PATCH 20/20] refactor: use null instead of undefined it's more common to use null instead of undefined and it's less characters :) --- src/token-builder/types.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/token-builder/types.js b/src/token-builder/types.js index 234dcfa..99d2663 100644 --- a/src/token-builder/types.js +++ b/src/token-builder/types.js @@ -57,7 +57,7 @@ export class ReplaceToken extends TokenBase { validateOptions = () => { super.validateOptions(); - if (this.value == undefined) { + if (this.value == null) { this.errors.push('Token was not instantiated with a replace value'); } else if (this.value && this.value.length > CHAR_LIMIT) { this.errors.push(`Replace value exceeds ${CHAR_LIMIT} character limit`); @@ -82,7 +82,7 @@ export class ReplaceLargeToken extends TokenBase { validateOptions() { super.validateOptions(); - if (this.value == undefined) { + if (this.value == null) { this.errors.push('Token was not instantiated with a replace value'); } else if (this.value && this.value.length <= CHAR_LIMIT) { this.errors.push(