From c8c40504cf2036bd4aff46161ed3d1c3b0534e3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Berkefeld?= Date: Tue, 27 Jun 2023 23:29:13 +0200 Subject: [PATCH 1/8] #987: add initial --like support to retrieve method --- docs/dist/documentation.md | 54 +++++++++++++++++++++++++++++++ lib/cli.js | 6 ++++ lib/index.js | 5 +-- lib/metadataTypes/MetadataType.js | 4 +++ lib/util/util.js | 44 +++++++++++++++++++++++++ 5 files changed, 111 insertions(+), 2 deletions(-) diff --git a/docs/dist/documentation.md b/docs/dist/documentation.md index c937ec619..264fee542 100644 --- a/docs/dist/documentation.md +++ b/docs/dist/documentation.md @@ -5880,6 +5880,8 @@ CLI entry for SFMC DevTools * [.getKeysString(keyArr, [isId])](#Util.getKeysString) ⇒ string * [.sleep(ms)](#Util.sleep) ⇒ Promise.<void> * [.getSsjs(code)](#Util.getSsjs) ⇒ string + * [.stringLike(testString, search)](#Util.stringLike) ⇒ boolean + * [.fieldsLike(metadata)](#Util.fieldsLike) ⇒ boolean @@ -6236,6 +6238,31 @@ the following is invalid: // 3 ``` + + +### Util.stringLike(testString, search) ⇒ boolean +allows us to filter just like with SQL's LIKE operator + +**Kind**: static method of [Util](#Util) +**Returns**: boolean - true if testString matches search + +| Param | Type | Description | +| --- | --- | --- | +| testString | string | field value to test | +| search | string | search string in SQL LIKE format | + + + +### Util.fieldsLike(metadata) ⇒ boolean +returns true if no LIKE filter is defined or if all filters match + +**Kind**: static method of [Util](#Util) +**Returns**: boolean - true if no LIKE filter is defined or if all filters match + +| Param | Type | Description | +| --- | --- | --- | +| metadata | TYPE.MetadataTypeItem | a single metadata item | + ## MetadataTypeDefinitions @@ -7770,6 +7797,8 @@ Util that contains logger and simple util methods * [.getKeysString(keyArr, [isId])](#Util.getKeysString) ⇒ string * [.sleep(ms)](#Util.sleep) ⇒ Promise.<void> * [.getSsjs(code)](#Util.getSsjs) ⇒ string + * [.stringLike(testString, search)](#Util.stringLike) ⇒ boolean + * [.fieldsLike(metadata)](#Util.fieldsLike) ⇒ boolean @@ -8126,6 +8155,31 @@ the following is invalid: // 3 ``` + + +### Util.stringLike(testString, search) ⇒ boolean +allows us to filter just like with SQL's LIKE operator + +**Kind**: static method of [Util](#Util) +**Returns**: boolean - true if testString matches search + +| Param | Type | Description | +| --- | --- | --- | +| testString | string | field value to test | +| search | string | search string in SQL LIKE format | + + + +### Util.fieldsLike(metadata) ⇒ boolean +returns true if no LIKE filter is defined or if all filters match + +**Kind**: static method of [Util](#Util) +**Returns**: boolean - true if no LIKE filter is defined or if all filters match + +| Param | Type | Description | +| --- | --- | --- | +| metadata | TYPE.MetadataTypeItem | a single metadata item | + ## csvToArray(csv) ⇒ Array.<string> diff --git a/lib/cli.js b/lib/cli.js index 4bd979400..81aac2d06 100644 --- a/lib/cli.js +++ b/lib/cli.js @@ -31,6 +31,12 @@ yargs .positional('KEY', { type: 'string', describe: 'metadata keys that shall be exclusively downloaded', + }) + .option('like', { + type: 'string', + group: 'Options for retrieve:', + describe: + 'filter metadata components (can include % as wildcard or _ for a single character, comma separated)', }); }, handler: (argv) => { diff --git a/lib/index.js b/lib/index.js index 3589f02b6..bf3ada5e1 100644 --- a/lib/index.js +++ b/lib/index.js @@ -51,15 +51,16 @@ class Mcdev { static setOptions(argv) { const knownOptions = [ 'api', - 'commitHistory', 'changeKeyField', 'changeKeyValue', + 'commitHistory', 'filter', 'fromRetrieve', 'json', + 'like', + 'noLogFile', 'refresh', 'skipInteraction', - 'noLogFile', ]; for (const option of knownOptions) { if (argv[option] !== undefined) { diff --git a/lib/metadataTypes/MetadataType.js b/lib/metadataTypes/MetadataType.js index 584b7fa69..97a776854 100644 --- a/lib/metadataTypes/MetadataType.js +++ b/lib/metadataTypes/MetadataType.js @@ -1570,6 +1570,10 @@ class MetadataType { } } + if (Util.OPTIONS.like && !Util.fieldsLike(results[originalKey])) { + Util.logger.debug(`Filtered ${originalKey} because of --like option`); + continue; + } // we dont store Id on local disk, but we need it for caching logic, // so its in retrieve but not in save. Here we put into the clone so that the original // object used for caching doesnt have the Id removed. diff --git a/lib/util/util.js b/lib/util/util.js index 173e88cbd..7f447b988 100644 --- a/lib/util/util.js +++ b/lib/util/util.js @@ -780,6 +780,50 @@ const Util = { // no script found return null; }, + /** + * allows us to filter just like with SQL's LIKE operator + * + * @param {string} testString field value to test + * @param {string} search search string in SQL LIKE format + * @returns {boolean} true if testString matches search + */ + stringLike(testString, search) { + if (typeof search !== 'string' || this === null) { + return false; + } + // Remove special chars + search = search.replaceAll( + new RegExp('([\\.\\\\\\+\\*\\?\\[\\^\\]\\$\\(\\)\\{\\}\\=\\!\\<\\>\\|\\:\\-])', 'g'), + '\\$1' + ); + // Replace % and _ with equivalent regex + search = search.replaceAll('%', '.*').replaceAll('_', '.'); + // Check matches + return new RegExp('^' + search + '$', 'gi').test(testString); + }, + /** + * returns true if no LIKE filter is defined or if all filters match + * + * @param {TYPE.MetadataTypeItem} metadata a single metadata item + * @returns {boolean} true if no LIKE filter is defined or if all filters match + */ + fieldsLike(metadata) { + const filters = Util.OPTIONS.like; + if (!filters) { + return true; + } + const fields = Object.keys(filters); + return fields.every((field) => { + const filter = filters[field]; + // TODO apply recursive logic for nested objects + if (typeof filter === 'string') { + return Util.stringLike(metadata[field], filter); + } else if (Array.isArray(filter)) { + return filter.some((f) => Util.stringLike(metadata[field], f)); + } + return false; + }); + }, }; Util.startLogger(false, true); From 9e2f013b53c4ce006b56a83faea04168696efe2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Berkefeld?= Date: Tue, 27 Jun 2023 23:57:14 +0200 Subject: [PATCH 2/8] #987: handle complex field paths in objects and arrays for --like --- docs/dist/documentation.md | 10 ++++++---- lib/util/util.js | 20 +++++++++++++------- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/docs/dist/documentation.md b/docs/dist/documentation.md index 264fee542..7bf37edbd 100644 --- a/docs/dist/documentation.md +++ b/docs/dist/documentation.md @@ -5881,7 +5881,7 @@ CLI entry for SFMC DevTools * [.sleep(ms)](#Util.sleep) ⇒ Promise.<void> * [.getSsjs(code)](#Util.getSsjs) ⇒ string * [.stringLike(testString, search)](#Util.stringLike) ⇒ boolean - * [.fieldsLike(metadata)](#Util.fieldsLike) ⇒ boolean + * [.fieldsLike(metadata, [filters])](#Util.fieldsLike) ⇒ boolean @@ -6253,7 +6253,7 @@ allows us to filter just like with SQL's LIKE operator -### Util.fieldsLike(metadata) ⇒ boolean +### Util.fieldsLike(metadata, [filters]) ⇒ boolean returns true if no LIKE filter is defined or if all filters match **Kind**: static method of [Util](#Util) @@ -6262,6 +6262,7 @@ returns true if no LIKE filter is defined or if all filters match | Param | Type | Description | | --- | --- | --- | | metadata | TYPE.MetadataTypeItem | a single metadata item | +| [filters] | object | only used in recursive calls | @@ -7798,7 +7799,7 @@ Util that contains logger and simple util methods * [.sleep(ms)](#Util.sleep) ⇒ Promise.<void> * [.getSsjs(code)](#Util.getSsjs) ⇒ string * [.stringLike(testString, search)](#Util.stringLike) ⇒ boolean - * [.fieldsLike(metadata)](#Util.fieldsLike) ⇒ boolean + * [.fieldsLike(metadata, [filters])](#Util.fieldsLike) ⇒ boolean @@ -8170,7 +8171,7 @@ allows us to filter just like with SQL's LIKE operator -### Util.fieldsLike(metadata) ⇒ boolean +### Util.fieldsLike(metadata, [filters]) ⇒ boolean returns true if no LIKE filter is defined or if all filters match **Kind**: static method of [Util](#Util) @@ -8179,6 +8180,7 @@ returns true if no LIKE filter is defined or if all filters match | Param | Type | Description | | --- | --- | --- | | metadata | TYPE.MetadataTypeItem | a single metadata item | +| [filters] | object | only used in recursive calls | diff --git a/lib/util/util.js b/lib/util/util.js index 7f447b988..9f5146113 100644 --- a/lib/util/util.js +++ b/lib/util/util.js @@ -805,21 +805,27 @@ const Util = { * returns true if no LIKE filter is defined or if all filters match * * @param {TYPE.MetadataTypeItem} metadata a single metadata item + * @param {object} [filters] only used in recursive calls * @returns {boolean} true if no LIKE filter is defined or if all filters match */ - fieldsLike(metadata) { - const filters = Util.OPTIONS.like; + fieldsLike(metadata, filters) { + filters ||= Util.OPTIONS.like; if (!filters) { return true; } const fields = Object.keys(filters); return fields.every((field) => { const filter = filters[field]; - // TODO apply recursive logic for nested objects - if (typeof filter === 'string') { - return Util.stringLike(metadata[field], filter); - } else if (Array.isArray(filter)) { - return filter.some((f) => Util.stringLike(metadata[field], f)); + if (Array.isArray(metadata[field])) { + return metadata[field].some((f) => Util.fieldsLike(f, filter)); + } else { + if (typeof filter === 'string') { + return Util.stringLike(metadata[field], filter); + } else if (Array.isArray(filter)) { + return filter.some((f) => Util.stringLike(metadata[field], f)); + } else if (typeof filter === 'object') { + return Util.fieldsLike(metadata[field], filter); + } } return false; }); From 09fad8c402afab9d3ff83aa1df7d6d9fb042c7ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Berkefeld?= Date: Tue, 27 Jun 2023 23:58:55 +0200 Subject: [PATCH 3/8] #987: refactoring --- lib/cli.js | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/cli.js b/lib/cli.js index 81aac2d06..3dce5868f 100644 --- a/lib/cli.js +++ b/lib/cli.js @@ -63,26 +63,26 @@ yargs type: 'string', describe: 'metadata key that shall be exclusively uploaded', }) - .group( - ['changeKeyField', 'changeKeyValue', 'fromRetrieve', 'refresh'], - 'Options for deploy:' - ) .option('changeKeyField', { type: 'string', + group: 'Options for deploy:', describe: 'enables updating the key of the deployed metadata with the value in provided field (e.g. c__newKey). Can be used to sync name and key fields.', }) .option('changeKeyValue', { type: 'string', + group: 'Options for deploy:', describe: 'allows updating the key of the metadata to the provided value. Only available if a single type and key is deployed', }) .option('fromRetrieve', { type: 'boolean', + group: 'Options for deploy:', describe: 'optionally deploy from retrieve folder', }) .option('refresh', { type: 'boolean', + group: 'Options for deploy:', describe: 'optional for asset-message: runs refresh command for related triggeredSends after deploy', }); @@ -315,8 +315,9 @@ yargs aliases: ['et'], desc: 'explains metadata types that can be retrieved', builder: (yargs) => { - yargs.group(['json'], 'Options for explainTypes:').option('json', { + yargs.option('json', { type: 'boolean', + group: 'Options for explainTypes:', describe: 'optionaly return info in json format', }); }, @@ -335,14 +336,15 @@ yargs type: 'string', describe: 'Pull Request target branch or git commit range', }) - .group(['filter', 'commitHistory'], 'Options for createDeltaPkg:') .option('filter', { type: 'string', + group: 'Options for createDeltaPkg:', describe: 'Disable templating & instead filter by the specified BU path (comma separated), can include subtype, will be prefixed with "retrieve/"', }) .option('commitHistory', { type: 'number', + group: 'Options for createDeltaPkg:', describe: 'Number of commits to look back for changes (supersedes config)', }); }, From e8da1b2fe796679eb958058664b81784e904eb42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Berkefeld?= Date: Wed, 28 Jun 2023 16:50:38 +0200 Subject: [PATCH 4/8] #987: add --like option support to execute command --- docs/dist/documentation.md | 30 +++++-- lib/cli.js | 15 +++- lib/index.js | 167 +++++++++++++++++++++++++++---------- lib/util/util.js | 4 + 4 files changed, 160 insertions(+), 56 deletions(-) diff --git a/docs/dist/documentation.md b/docs/dist/documentation.md index 7bf37edbd..103c133e4 100644 --- a/docs/dist/documentation.md +++ b/docs/dist/documentation.md @@ -495,8 +495,9 @@ main class * [.buildDefinition(businessUnit, selectedType, name, market)](#Mcdev.buildDefinition) ⇒ Promise.<void> * [.buildDefinitionBulk(listName, type, name)](#Mcdev.buildDefinitionBulk) ⇒ Promise.<void> * [.getFilesToCommit(businessUnit, selectedType, keyArr)](#Mcdev.getFilesToCommit) ⇒ Promise.<Array.<string>> - * [.execute(businessUnit, [selectedTypesArr], keys)](#Mcdev.execute) ⇒ Promise.<boolean> - * [._executeBU(cred, bu, [selectedTypesArr], keyArr)](#Mcdev._executeBU) ⇒ Promise.<boolean> + * [.execute(businessUnit, [selectedType], [keys])](#Mcdev.execute) ⇒ Promise.<boolean> + * [._executeBU(cred, bu, [type], keyArr)](#Mcdev._executeBU) ⇒ Promise.<boolean> + * [._getKeysWithLike(selectedType, buObject)](#Mcdev._getKeysWithLike) ⇒ Array.<string> @@ -751,7 +752,7 @@ Build a specific metadata file based on a template using a list of bu-market com -### Mcdev.execute(businessUnit, [selectedTypesArr], keys) ⇒ Promise.<boolean> +### Mcdev.execute(businessUnit, [selectedType], [keys]) ⇒ Promise.<boolean> Start an item (query) **Kind**: static method of [Mcdev](#Mcdev) @@ -760,13 +761,13 @@ Start an item (query) | Param | Type | Description | | --- | --- | --- | | businessUnit | string | name of BU | -| [selectedTypesArr] | Array.<TYPE.SupportedMetadataTypes> | limit to given metadata types | -| keys | Array.<string> | customerkey of the metadata | +| [selectedType] | TYPE.SupportedMetadataTypes | limit to given metadata types | +| [keys] | Array.<string> | customerkey of the metadata | -### Mcdev.\_executeBU(cred, bu, [selectedTypesArr], keyArr) ⇒ Promise.<boolean> -helper for [execute](execute) +### Mcdev.\_executeBU(cred, bu, [type], keyArr) ⇒ Promise.<boolean> +helper for [execute](#Mcdev.execute) **Kind**: static method of [Mcdev](#Mcdev) **Returns**: Promise.<boolean> - true if all items were executed, false otherwise @@ -775,9 +776,22 @@ helper for [execute](execute) | --- | --- | --- | | cred | string | name of Credential | | bu | string | name of BU | -| [selectedTypesArr] | Array.<TYPE.SupportedMetadataTypes> | limit execution to given metadata type | +| [type] | TYPE.SupportedMetadataTypes | limit execution to given metadata type | | keyArr | Array.<string> | customerkey of the metadata | + + +### Mcdev.\_getKeysWithLike(selectedType, buObject) ⇒ Array.<string> +helper for [_executeBU](#Mcdev._executeBU) + +**Kind**: static method of [Mcdev](#Mcdev) +**Returns**: Array.<string> - keyArr + +| Param | Type | Description | +| --- | --- | --- | +| selectedType | TYPE.SupportedMetadataTypes | limit execution to given metadata type | +| buObject | TYPE.BuObject | properties for auth | + ## Asset ⇐ [MetadataType](#MetadataType) diff --git a/lib/cli.js b/lib/cli.js index 3dce5868f..3d3f60ed4 100644 --- a/lib/cli.js +++ b/lib/cli.js @@ -36,7 +36,7 @@ yargs type: 'string', group: 'Options for retrieve:', describe: - 'filter metadata components (can include % as wildcard or _ for a single character, comma separated)', + 'filter metadata components (can include % as wildcard or _ for a single character)', }); }, handler: (argv) => { @@ -403,8 +403,8 @@ yargs }, }) .command({ - command: 'execute ', - aliases: ['exec'], + command: 'execute [KEY]', + aliases: ['exec', 'start'], desc: 'executes the entity (query/journey/automation etc.)', builder: (yargs) => { yargs @@ -419,11 +419,18 @@ yargs .positional('KEY', { type: 'string', describe: 'key(s) of the metadata component(s)', + }) + .option('like', { + type: 'string', + group: 'Options for execute:', + describe: + 'filter metadata components (can include % as wildcard or _ for a single character)', }); }, handler: (argv) => { Mcdev.setOptions(argv); - Mcdev.execute(argv.BU, csvToArray(argv.TYPE), csvToArray(argv.KEY)); + // ! do not allow multiple types to be passed in here via csvToArray + Mcdev.execute(argv.BU, argv.TYPE, csvToArray(argv.KEY)); }, }) .command({ diff --git a/lib/index.js b/lib/index.js index bf3ada5e1..be18b67b1 100644 --- a/lib/index.js +++ b/lib/index.js @@ -236,7 +236,7 @@ class Mcdev { } } /** - * helper for {@link retrieve} + * helper for {@link Mcdev.retrieve} * * @private * @param {string} cred name of Credential @@ -705,11 +705,11 @@ class Mcdev { * Start an item (query) * * @param {string} businessUnit name of BU - * @param {TYPE.SupportedMetadataTypes[]} [selectedTypesArr] limit to given metadata types - * @param {string[]} keys customerkey of the metadata + * @param {TYPE.SupportedMetadataTypes} [selectedType] limit to given metadata types + * @param {string[]} [keys] customerkey of the metadata * @returns {Promise.} true if all started successfully, false if not */ - static async execute(businessUnit, selectedTypesArr, keys) { + static async execute(businessUnit, selectedType, keys) { Util.startLogger(); Util.logger.info('mcdev:: Execute'); const properties = await config.getProperties(); @@ -719,15 +719,29 @@ class Mcdev { // return null here to avoid seeing 2 error messages for the same issue return null; } - if (Array.isArray(selectedTypesArr)) { - // types and keys can be provided but for each type all provided keys are applied as filter - for (const selectedType of Array.isArray(selectedTypesArr) - ? selectedTypesArr - : Object.keys(selectedTypesArr)) { - if (!Util._isValidType(selectedType)) { - return; - } - } + if (!Util._isValidType(selectedType)) { + return; + } + if (!Object.prototype.hasOwnProperty.call(MetadataTypeInfo[selectedType], 'execute')) { + Util.logger.error( + ` ☇ skipping ${selectedType}: execute is not supported yet for ${selectedType}` + ); + return; + } + if ( + (!Array.isArray(keys) || !keys.length) && + (!Util.OPTIONS.like || !Object.keys(Util.OPTIONS.like).length) + ) { + Util.logger.error('At least one key or a --like filter is required.'); + return; + } else if ( + Array.isArray(keys) && + keys.length && + Util.OPTIONS.like && + Object.keys(Util.OPTIONS.like).length + ) { + Util.logger.error('You can either specify keys OR a --like filter.'); + return; } if (businessUnit === '*') { Util.logger.info( @@ -738,7 +752,7 @@ class Mcdev { Util.logger.info(`\n :: Executing the entity on all BUs for ${cred}`); for (const bu in properties.credentials[cred].businessUnits) { - if (await this._executeBU(cred, bu, selectedTypesArr, keys)) { + if (await this._executeBU(cred, bu, selectedType, keys)) { counter_credBu++; } else { counter_failed++; @@ -778,7 +792,7 @@ class Mcdev { Util.logger.info(`\n :: Executing the entity on all BUs for ${cred}`); let counter_credBu = 0; for (const bu in properties.credentials[cred].businessUnits) { - if (await this._executeBU(cred, bu, selectedTypesArr, keys)) { + if (await this._executeBU(cred, bu, selectedType, keys)) { counter_credBu++; } else { counter_failed++; @@ -790,7 +804,7 @@ class Mcdev { ); } else { // execute the entity on one BU only - if (await this._executeBU(cred, bu, selectedTypesArr, keys)) { + if (await this._executeBU(cred, bu, selectedType, keys)) { counter_credBu++; } else { counter_failed++; @@ -804,15 +818,15 @@ class Mcdev { return counter_failed === 0 ? true : false; } /** - * helper for {@link execute} + * helper for {@link Mcdev.execute} * * @param {string} cred name of Credential * @param {string} bu name of BU - * @param {TYPE.SupportedMetadataTypes[]} [selectedTypesArr] limit execution to given metadata type + * @param {TYPE.SupportedMetadataTypes} [type] limit execution to given metadata type * @param {string[]} keyArr customerkey of the metadata * @returns {Promise.} true if all items were executed, false otherwise */ - static async _executeBU(cred, bu, selectedTypesArr, keyArr) { + static async _executeBU(cred, bu, type, keyArr) { const properties = await config.getProperties(); let counter_failed = 0; const buObject = await Cli.getCredentialObject( @@ -821,38 +835,103 @@ class Mcdev { null, true ); - if (!keyArr || (Array.isArray(keyArr) && !keyArr.length)) { - throw new Error('No keys were provided'); + try { + if (!type) { + throw new Error('No type was provided'); + } + if (buObject !== null) { + cache.initCache(buObject); + cred = buObject.credential; + bu = buObject.businessUnit; + } + Util.logger.info(`\n :: Executing ${type} on ${cred}/${bu}\n`); + try { + MetadataTypeInfo[type].client = auth.getSDK(buObject); + } catch (ex) { + Util.logger.error(ex.message); + return; + } + if (Util.OPTIONS.like && Object.keys(Util.OPTIONS.like).length) { + keyArr = await this._getKeysWithLike(type, buObject); + } + if (!keyArr || (Array.isArray(keyArr) && !keyArr.length)) { + throw new Error('No keys were provided'); + } + + // result will be undefined (false) if execute is not supported for the type + if (!(await MetadataTypeInfo[type].execute(keyArr))) { + counter_failed++; + } + } catch (ex) { + Util.logger.errorStack(ex, 'mcdev.execute failed'); } - if (!selectedTypesArr || (Array.isArray(selectedTypesArr) && !selectedTypesArr.length)) { - throw new Error('No type was provided'); + + return counter_failed === 0 ? true : false; + } + + /** + * helper for {@link Mcdev._executeBU} + * + * @param {TYPE.SupportedMetadataTypes} selectedType limit execution to given metadata type + * @param {TYPE.BuObject} buObject properties for auth + * @returns {string[]} keyArr + */ + static async _getKeysWithLike(selectedType, buObject) { + const properties = await config.getProperties(); + + // cache depenencies + const deployOrder = Util.getMetadataHierachy([selectedType]); + for (const type in deployOrder) { + const subTypeArr = deployOrder[type]; + MetadataTypeInfo[type].client = auth.getSDK(buObject); + MetadataTypeInfo[type].properties = properties; + MetadataTypeInfo[type].buObject = buObject; + Util.logger.info(`Caching dependent Metadata: ${type}`); + Util.logSubtypes(subTypeArr); + const result = await MetadataTypeInfo[type].retrieveForCache(null, subTypeArr); + if (result) { + if (Array.isArray(result)) { + for (const result_i of result) { + if (result_i?.metadata && Object.keys(result_i.metadata).length) { + cache.mergeMetadata(type, result_i.metadata); + } + } + } else { + cache.setMetadata(type, result.metadata); + } + } } - if (buObject !== null) { - cache.initCache(buObject); - cred = buObject.credential; - bu = buObject.businessUnit; + + // find all keys in chosen type that match the like-filter + const keyArr = []; + const metadataMap = cache.getCache()[selectedType]; + if (!metadataMap) { + throw new Error(`Selected type ${selectedType} could not be cached`); } Util.logger.info( - `\n :: Executing ${selectedTypesArr.join(', ')} on ${cred}/${bu}\n` + Util.getGrayMsg(`Found ${Object.keys(metadataMap).length} ${selectedType}s`) ); - try { - // more than one type was provided, iterate types and execute items - for (const type of selectedTypesArr) { - try { - MetadataTypeInfo[type].client = auth.getSDK(buObject); - } catch (ex) { - Util.logger.error(ex.message); - return; - } - // result will be undefined (false) if execute is not supported for the type - if (!(await MetadataTypeInfo[type].execute(keyArr))) { - counter_failed++; - } + for (const originalKey in metadataMap) { + // hide postRetrieveOutput + Util.setLoggingLevel({ silent: true }); + metadataMap[originalKey] = MetadataTypeInfo[selectedType].postRetrieveTasks( + metadataMap[originalKey] + ); + // reactivate logging + Util.setLoggingLevel({}); + if (Util.fieldsLike(metadataMap[originalKey])) { + keyArr.push(originalKey); } - } catch (ex) { - Util.logger.errorStack(ex, 'mcdev.execute failed'); } - return counter_failed === 0 ? true : false; + Util.logger.info( + Util.getGrayMsg( + `Identified ${keyArr.length} ${selectedType}${ + keyArr.length === 1 ? '' : 's' + } that match${selectedType}${keyArr.length === 1 ? 'es' : ''} the like-filter` + ) + ); + + return keyArr; } } diff --git a/lib/util/util.js b/lib/util/util.js index 9f5146113..e552fe382 100644 --- a/lib/util/util.js +++ b/lib/util/util.js @@ -809,6 +809,10 @@ const Util = { * @returns {boolean} true if no LIKE filter is defined or if all filters match */ fieldsLike(metadata, filters) { + if (metadata.json && metadata.codeArr) { + // Compensate for CodeExtractItem format + metadata = metadata.json; + } filters ||= Util.OPTIONS.like; if (!filters) { return true; From dbc5bcd29f9408b175a07565b446dc061989f0d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Berkefeld?= Date: Wed, 28 Jun 2023 17:09:07 +0200 Subject: [PATCH 5/8] #987: fix execute query test to work with new --like logic --- lib/index.js | 21 ++++++++------------- test/type.query.test.js | 26 ++++++++++++++++++++------ 2 files changed, 28 insertions(+), 19 deletions(-) diff --git a/lib/index.js b/lib/index.js index be18b67b1..26d19ed66 100644 --- a/lib/index.js +++ b/lib/index.js @@ -711,29 +711,29 @@ class Mcdev { */ static async execute(businessUnit, selectedType, keys) { Util.startLogger(); - Util.logger.info('mcdev:: Execute'); + Util.logger.info('mcdev:: Executing ' + selectedType); const properties = await config.getProperties(); let counter_credBu = 0; let counter_failed = 0; if (!(await config.checkProperties(properties))) { // return null here to avoid seeing 2 error messages for the same issue - return null; + return false; } if (!Util._isValidType(selectedType)) { - return; + return false; } if (!Object.prototype.hasOwnProperty.call(MetadataTypeInfo[selectedType], 'execute')) { Util.logger.error( ` ☇ skipping ${selectedType}: execute is not supported yet for ${selectedType}` ); - return; + return false; } if ( (!Array.isArray(keys) || !keys.length) && (!Util.OPTIONS.like || !Object.keys(Util.OPTIONS.like).length) ) { Util.logger.error('At least one key or a --like filter is required.'); - return; + return false; } else if ( Array.isArray(keys) && keys.length && @@ -741,7 +741,7 @@ class Mcdev { Object.keys(Util.OPTIONS.like).length ) { Util.logger.error('You can either specify keys OR a --like filter.'); - return; + return false; } if (businessUnit === '*') { Util.logger.info( @@ -782,7 +782,7 @@ class Mcdev { true ); if (buObject === null) { - return; + return false; } else { cred = buObject.credential; bu = buObject.businessUnit; @@ -845,12 +845,7 @@ class Mcdev { bu = buObject.businessUnit; } Util.logger.info(`\n :: Executing ${type} on ${cred}/${bu}\n`); - try { - MetadataTypeInfo[type].client = auth.getSDK(buObject); - } catch (ex) { - Util.logger.error(ex.message); - return; - } + MetadataTypeInfo[type].client = auth.getSDK(buObject); if (Util.OPTIONS.like && Object.keys(Util.OPTIONS.like).length) { keyArr = await this._getKeysWithLike(type, buObject); } diff --git a/test/type.query.test.js b/test/type.query.test.js index edaf72161..489e51898 100644 --- a/test/type.query.test.js +++ b/test/type.query.test.js @@ -279,15 +279,29 @@ describe('type: query', () => { }); }); describe('Execute ================', () => { - it('Should start executing a query', async () => { - const execute = await handler.execute( - 'testInstance/testBU', - ['query'], - ['testExisting_query'] - ); + it('Should start a query by key', async () => { + const execute = await handler.execute('testInstance/testBU', 'query', [ + 'testExisting_query', + ]); + assert.equal(process.exitCode, false, 'execute should not have thrown an error'); + assert.equal(execute, true, 'query was supposed to be executed'); + return; + }); + it('Should start a query selected via --like', async () => { + handler.setOptions({ like: { key: 'testExisting%' } }); + const execute = await handler.execute('testInstance/testBU', 'query'); assert.equal(process.exitCode, false, 'execute should not have thrown an error'); assert.equal(execute, true, 'query was supposed to be executed'); return; }); + it('Should not start executing a query because key and --like was specified', async () => { + handler.setOptions({ like: { key: 'testExisting%' } }); + const execute = await handler.execute('testInstance/testBU', 'query', [ + 'testExisting_query', + ]); + assert.equal(process.exitCode, true, 'execute should not have thrown an error'); + assert.equal(execute, false, 'query was not supposed to be executed'); + return; + }); }); }); From 9784c730a4fc21c290dc7f1a422b0ef33bf27b51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Berkefeld?= Date: Wed, 28 Jun 2023 17:26:01 +0200 Subject: [PATCH 6/8] #987: add retrieve --like tests --- test/type.query.test.js | 57 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/test/type.query.test.js b/test/type.query.test.js index 489e51898..0b04d878f 100644 --- a/test/type.query.test.js +++ b/test/type.query.test.js @@ -53,7 +53,7 @@ describe('type: query', () => { ); return; }); - it('Should retrieve one specific query', async () => { + it('Should retrieve one specific query by key', async () => { // WHEN await handler.retrieve('testInstance/testBU', ['query'], ['testExisting_query']); // THEN @@ -80,6 +80,61 @@ describe('type: query', () => { ); return; }); + it('Should retrieve one specific query via --like', async () => { + // WHEN + handler.setOptions({ like: { key: '%Existing_query' } }); + await handler.retrieve('testInstance/testBU', ['query']); + // THEN + assert.equal(process.exitCode, false, 'retrieve should not have thrown an error'); + // get results from cache + const result = cache.getCache(); + assert.equal( + result.query ? Object.keys(result.query).length : 0, + 2, + 'two queries in cache expected' + ); + assert.deepEqual( + await testUtils.getActualJson('testExisting_query', 'query'), + await testUtils.getExpectedJson('9999999', 'query', 'get'), + 'returned metadata was not equal expected' + ); + expect(file(testUtils.getActualFile('testExisting_query', 'query', 'sql'))).to.equal( + file(testUtils.getExpectedFile('9999999', 'query', 'get', 'sql')) + ); + expect(file(testUtils.getActualFile('testExisting_query2', 'query', 'sql'))).to.not + .exist; + assert.equal( + testUtils.getAPIHistoryLength(), + 6, + 'Unexpected number of requests made. Run testUtils.logAPIHistoryDebug() to see the requests' + ); + return; + }); + it('Should not retrieve any query via --like and key due to a mismatching filter', async () => { + // WHEN + handler.setOptions({ like: { key: 'NotExisting_query' } }); + await handler.retrieve('testInstance/testBU', ['query']); + // THEN + assert.equal(process.exitCode, false, 'retrieve should not have thrown an error'); + // get results from cache + const result = cache.getCache(); + assert.equal( + result.query ? Object.keys(result.query).length : 0, + 2, + 'two queries in cache expected' + ); + + expect(file(testUtils.getActualFile('testExisting_query', 'query', 'sql'))).to.not + .exist; + expect(file(testUtils.getActualFile('testExisting_query2', 'query', 'sql'))).to.not + .exist; + assert.equal( + testUtils.getAPIHistoryLength(), + 6, + 'Unexpected number of requests made. Run testUtils.logAPIHistoryDebug() to see the requests' + ); + return; + }); }); describe('Deploy ================', () => { beforeEach(() => { From da912a975ff410bad2358d8281e0969f32d86598 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Berkefeld?= Date: Thu, 29 Jun 2023 14:34:41 +0200 Subject: [PATCH 7/8] #987: fix test to only find ONE query --- test/type.query.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/type.query.test.js b/test/type.query.test.js index 0de1e0780..b82df7cc4 100644 --- a/test/type.query.test.js +++ b/test/type.query.test.js @@ -370,7 +370,7 @@ describe('type: query', () => { return; }); it('Should start a query selected via --like', async () => { - handler.setOptions({ like: { key: 'testExisting%' } }); + handler.setOptions({ like: { key: 'testExist%query' } }); const execute = await handler.execute('testInstance/testBU', 'query'); assert.equal(process.exitCode, false, 'execute should not have thrown an error'); assert.equal(execute, true, 'query was supposed to be executed'); From 1d3cf0f42be1c8cc7323b0ae525e2743f24f5059 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Berkefeld?= Date: Thu, 29 Jun 2023 14:38:25 +0200 Subject: [PATCH 8/8] #987: rename method to describe better what it does --- docs/dist/documentation.md | 6 +++--- lib/index.js | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/dist/documentation.md b/docs/dist/documentation.md index 2e8bd94f4..82eedc210 100644 --- a/docs/dist/documentation.md +++ b/docs/dist/documentation.md @@ -497,7 +497,7 @@ main class * [.getFilesToCommit(businessUnit, selectedType, keyArr)](#Mcdev.getFilesToCommit) ⇒ Promise.<Array.<string>> * [.execute(businessUnit, [selectedType], [keys])](#Mcdev.execute) ⇒ Promise.<boolean> * [._executeBU(cred, bu, [type], keyArr)](#Mcdev._executeBU) ⇒ Promise.<boolean> - * [._getKeysWithLike(selectedType, buObject)](#Mcdev._getKeysWithLike) ⇒ Array.<string> + * [._retrieveKeysWithLike(selectedType, buObject)](#Mcdev._retrieveKeysWithLike) ⇒ Array.<string> @@ -779,9 +779,9 @@ helper for [execute](#Mcdev.execute) | [type] | TYPE.SupportedMetadataTypes | limit execution to given metadata type | | keyArr | Array.<string> | customerkey of the metadata | - + -### Mcdev.\_getKeysWithLike(selectedType, buObject) ⇒ Array.<string> +### Mcdev.\_retrieveKeysWithLike(selectedType, buObject) ⇒ Array.<string> helper for [_executeBU](#Mcdev._executeBU) **Kind**: static method of [Mcdev](#Mcdev) diff --git a/lib/index.js b/lib/index.js index d381ca7c4..5928a854e 100644 --- a/lib/index.js +++ b/lib/index.js @@ -842,7 +842,7 @@ class Mcdev { Util.logger.info(`\n :: Executing ${type} on ${cred}/${bu}\n`); MetadataTypeInfo[type].client = auth.getSDK(buObject); if (Util.OPTIONS.like && Object.keys(Util.OPTIONS.like).length) { - keyArr = await this._getKeysWithLike(type, buObject); + keyArr = await this._retrieveKeysWithLike(type, buObject); } if (!keyArr || (Array.isArray(keyArr) && !keyArr.length)) { throw new Error('No keys were provided'); @@ -866,7 +866,7 @@ class Mcdev { * @param {TYPE.BuObject} buObject properties for auth * @returns {string[]} keyArr */ - static async _getKeysWithLike(selectedType, buObject) { + static async _retrieveKeysWithLike(selectedType, buObject) { const properties = await config.getProperties(); // cache depenencies