diff --git a/docs/dist/documentation.md b/docs/dist/documentation.md index 666ecb716..f01ef5cf2 100644 --- a/docs/dist/documentation.md +++ b/docs/dist/documentation.md @@ -1325,7 +1325,7 @@ AttributeSet MetadataType * [AttributeSet](#AttributeSet) ⇐ [MetadataType](#MetadataType) * [.retrieve(retrieveDir, [_], [__], [key])](#AttributeSet.retrieve) ⇒ Promise.<TYPE.MetadataTypeMapObj> * [.retrieveForCache()](#AttributeSet.retrieveForCache) ⇒ Promise.<TYPE.MetadataTypeMapObj> - * [.retrieveForSharedDEs(sharedDataExtensionIds)](#AttributeSet.retrieveForSharedDEs) ⇒ Promise.<Array.<string>> + * [.fixShared_retrieve(sharedDataExtensionMap, fixShared_fieldChange)](#AttributeSet.fixShared_retrieve) ⇒ Promise.<Array.<string>> * [.parseResponseBody(body, [singleRetrieve])](#AttributeSet.parseResponseBody) ⇒ TYPE.MetadataTypeMap * [.postRetrieveTasks(metadata)](#AttributeSet.postRetrieveTasks) ⇒ TYPE.MetadataTypeItem * [._getSystemValueDefinitions()](#AttributeSet._getSystemValueDefinitions) ⇒ Array.<object> @@ -1352,9 +1352,9 @@ Retrieves Metadata of schema set definitions for caching. **Kind**: static method of [AttributeSet](#AttributeSet) **Returns**: Promise.<TYPE.MetadataTypeMapObj> - Promise - + -### AttributeSet.retrieveForSharedDEs(sharedDataExtensionIds) ⇒ Promise.<Array.<string>> +### AttributeSet.fixShared\_retrieve(sharedDataExtensionMap, fixShared_fieldChange) ⇒ Promise.<Array.<string>> Retrieves Metadata of schema set definitions for caching. **Kind**: static method of [AttributeSet](#AttributeSet) @@ -1362,7 +1362,8 @@ Retrieves Metadata of schema set definitions for caching. | Param | Type | Description | | --- | --- | --- | -| sharedDataExtensionIds | Array.<string> | ID array for shared data extensions | +| sharedDataExtensionMap | Object.<string, string> | ID-Key relationship of shared data extensions | +| fixShared_fieldChange | object | DataExtensionField.fixShared_fieldChange | @@ -1918,7 +1919,7 @@ Retrieves dataExtension metadata. Afterwards starts retrieval of dataExtensionCo ### DataExtension.retrieveSharedForCache([additionalFields]) ⇒ Promise.<TYPE.DataExtensionMap> get shared dataExtensions from parent BU and merge them into the cache -helper for [retrieve](#DataExtension.retrieve) and for AttributeSet.retrieveForSharedDEs +helper for [retrieve](#DataExtension.retrieve) and for AttributeSet.fixShared_retrieve **Kind**: static method of [DataExtension](#DataExtension) **Returns**: Promise.<TYPE.DataExtensionMap> - keyField => metadata map diff --git a/lib/metadataTypes/AttributeSet.js b/lib/metadataTypes/AttributeSet.js index a558587df..a076d514d 100644 --- a/lib/metadataTypes/AttributeSet.js +++ b/lib/metadataTypes/AttributeSet.js @@ -50,28 +50,114 @@ class AttributeSet extends MetadataType { /** * Retrieves Metadata of schema set definitions for caching. * - * @param {string[]} sharedDataExtensionIds ID array for shared data extensions + * @param {Object.} sharedDataExtensionMap ID-Key relationship of shared data extensions + * @param {object} fixShared_fieldChange DataExtensionField.fixShared_fieldChange * @returns {Promise.} Promise of list of shared dataExtension IDs */ - static async retrieveForSharedDEs(sharedDataExtensionIds) { - if (!sharedDataExtensionIds.length) { + static async fixShared_retrieve(sharedDataExtensionMap, fixShared_fieldChange) { + if (!Object.keys(sharedDataExtensionMap).length) { return []; } const result = await super.retrieveREST(null, '/hub/v1/contacts/schema/setDefinitions'); const metadataMap = result?.metadata; if (metadataMap && Object.keys(metadataMap).length) { - const sharedDEs = Object.keys(metadataMap) + const sharedDeIds = Object.keys(metadataMap) .filter( - (key) => - metadataMap[key].storageLogicalType === 'ExactTargetSchema' || - metadataMap[key].storageLogicalType === 'DataExtension' - ) - .filter((key) => - sharedDataExtensionIds.includes(metadataMap[key].storageReferenceID.value) + (asKey) => + metadataMap[asKey].storageLogicalType === 'ExactTargetSchema' || + metadataMap[asKey].storageLogicalType === 'DataExtension' ) + .filter((asKey) => { + // check if dataExtension ID is found on any attributeSet of this BU + if (sharedDataExtensionMap[metadataMap[asKey].storageReferenceID.value]) { + Util.logger.debug( + ` shared dataExtension ID ${metadataMap[asKey].storageReferenceID.value} found in attributeGroup ${asKey}` + ); + return true; + } else { + return false; + } + }) + .filter((asKey) => { + // check if any of the updated dataExtension fields dont exist on the attributeSet or are out of date + const deKey = + sharedDataExtensionMap[metadataMap[asKey].storageReferenceID.value]; + const asFields = metadataMap[asKey].valueDefinitions; + if (!fixShared_fieldChange[deKey]) { + Util.logger.debug( + ` - No changed fields found. Assuming re-deploy with --fixShared flag` + ); + return true; + } + const deFields = Object.values(fixShared_fieldChange[deKey]); + return deFields.some((deField) => { + const search = asFields.filter((asf) => asf.name === deField.Name); + if (!search.length) { + Util.logger.debug( + Util.getGrayMsg( + ` - Field ${deField.Name} not found in attributeSet` + ) + ); + return true; + } + const asField = search[0]; + if (asField.dataType !== deField.FieldType) { + Util.logger.debug( + Util.getGrayMsg( + ` - Field ${deField.Name} FieldType changed (old: ${asField.dataType}; new: ${deField.FieldType})` + ) + ); + return true; + } + if ( + (asField.defaultValue && deField.DefaultValue !== '') || + (deField.FieldType === 'Boolean' && + deField.DefaultValue !== '' && + deField.DefaultValue + ? 'True' + : 'False' !== asField.defaultValue) || + (deField.FieldType !== 'Boolean' && + deField.DefaultValue !== asField.defaultValue) + ) { + Util.logger.debug( + Util.getGrayMsg( + ` - Field ${deField.Name} DefaultValue changed (old: ${asField.defaultValue}; new: ${deField.DefaultValue})` + ) + ); + return true; + } + // some field types don't carry the length property. reset to 0 to ease comparison + asField.length ||= 0; + if (asField.length !== deField.MaxLength) { + Util.logger.debug( + Util.getGrayMsg( + ` - Field ${deField.Name} MaxLength changed (old: ${asField.length}; new: ${deField.MaxLength})` + ) + ); + return true; + } + if (asField.isNullable !== deField.IsRequired) { + Util.logger.debug( + Util.getGrayMsg( + ` - Field ${deField.Name} IsRequired changed (old: ${asField.isNullable}; new: ${deField.IsRequired})` + ) + ); + return true; + } + if (asField.isPrimaryKey !== deField.IsPrimaryKey) { + Util.logger.debug( + Util.getGrayMsg( + ` - Field ${deField.Name} IsPrimaryKey changed (old: ${asField.isPrimaryKey}; new: ${deField.IsPrimaryKey})` + ) + ); + return true; + } + return false; + }); + }) .map((key) => metadataMap[key].storageReferenceID.value) .filter(Boolean); - return sharedDEs; + return sharedDeIds; } else { // nothing to do - return empty array return []; diff --git a/lib/metadataTypes/DataExtension.js b/lib/metadataTypes/DataExtension.js index acf737fe2..9215ff409 100644 --- a/lib/metadataTypes/DataExtension.js +++ b/lib/metadataTypes/DataExtension.js @@ -389,16 +389,19 @@ class DataExtension extends MetadataType { return; } - const sharedDataExtensionsKeys = this.deployedSharedKeys.filter((key) => - DataExtensionField.fixShared_fieldChange.includes(key) - ); + const sharedDataExtensionsKeys = this.deployedSharedKeys; this.deployedSharedKeys = null; - if (!sharedDataExtensionsKeys.length) { - // only if updates were made could the issue in https://issues.salesforce.com/#q=W-11031095 affect data designer + if ( + !sharedDataExtensionsKeys.filter( + (deKey) => + DataExtensionField.fixShared_fieldChange[deKey] && + Object.keys(DataExtensionField.fixShared_fieldChange[deKey])?.length + ) + ) { Util.logger.debug( - `Skipping fixShared logic because no fields were changed on updated Shared Data Extensions` + ` - No changed fields found. Assuming re-deploy with --fixShared flag` ); - return; + return true; } if (Util.OPTIONS.fixShared) { @@ -532,8 +535,9 @@ class DataExtension extends MetadataType { AttributeSet.properties = this.properties; AttributeSet.buObject = buObjectChildBu; AttributeSet.client = clientChildBu; - const sharedDeIdsUsedOnBU = await AttributeSet.retrieveForSharedDEs( - Object.keys(sharedDataExtensionMap) + const sharedDeIdsUsedOnBU = await AttributeSet.fixShared_retrieve( + sharedDataExtensionMap, + DataExtensionField.fixShared_fieldChange ); if (sharedDeIdsUsedOnBU.length) { let sharedDataExtensionsKeys = sharedDeIdsUsedOnBU.map( @@ -791,7 +795,7 @@ class DataExtension extends MetadataType { /** * get shared dataExtensions from parent BU and merge them into the cache - * helper for {@link DataExtension.retrieve} and for AttributeSet.retrieveForSharedDEs + * helper for {@link DataExtension.retrieve} and for AttributeSet.fixShared_retrieve * * @param {string[]} [additionalFields] Returns specified fields even if their retrieve definition is not set to true * @returns {Promise.} keyField => metadata map diff --git a/lib/metadataTypes/DataExtensionField.js b/lib/metadataTypes/DataExtensionField.js index c993c15f9..4d81e45f1 100644 --- a/lib/metadataTypes/DataExtensionField.js +++ b/lib/metadataTypes/DataExtensionField.js @@ -99,6 +99,10 @@ class DataExtensionField extends MetadataType { * @returns {Promise.>} existing fields by their original name to allow re-adding FieldType after update */ static async prepareDeployColumnsOnUpdate(deployColumns, deKey) { + // create list of DE keys that had changes to their fields to be able to use it as a filter in the --fixShared logic + this.fixShared_fieldChange ||= {}; + this.fixShared_fieldChange[deKey] ||= {}; + // get row count to know which field restrictions apply let hasData = false; try { @@ -189,6 +193,9 @@ class DataExtensionField extends MetadataType { Util.logger.verbose(`no change - removed field [${deKey}].[${item.Name}]`); continue; } + // track name of changed field + this.fixShared_fieldChange[deKey][item.Name] = JSON.parse(JSON.stringify(item)); + this.fixShared_fieldChange[deKey][item.Name].FieldType = itemOld.FieldType; // set the ObjectId for clear identification during update item.ObjectID = itemOld.ObjectID; @@ -217,6 +224,8 @@ class DataExtensionField extends MetadataType { } // Field doesn't exist in target, therefore Remove ObjectID if present delete item.ObjectID; + + this.fixShared_fieldChange[deKey][item.Name] = JSON.parse(JSON.stringify(item)); } if (Util.isTrue(item.IsPrimaryKey) && Util.isFalse(item.IsRequired)) { // applicable: with or without data @@ -232,12 +241,14 @@ class DataExtensionField extends MetadataType { `- Invalid value for 'IsRequired' of [${deKey}].[${item.Name}]. Found '${item.IsRequired}' instead of 'true'/'false'. Removing field from deploy!` ); deployColumns.splice(i, 1); + delete this.fixShared_fieldChange[deKey][item.Name]; } if (!Util.isTrue(item.IsPrimaryKey) && !Util.isFalse(item.IsPrimaryKey)) { Util.logger.error( `- Invalid value for 'IsPrimaryKey' of [${deKey}].[${item.Name}]. Found '${item.IsPrimaryKey}' instead of 'true'/'false'. Removing field from deploy!` ); deployColumns.splice(i, 1); + delete this.fixShared_fieldChange[deKey][item.Name]; } } @@ -249,9 +260,9 @@ class DataExtensionField extends MetadataType { ) ); // create list of DE keys that had changes to their fields to be able to use it as a filter in the --fixShared logic - this.fixShared_fieldChange ||= []; - if (deployColumns.length) { - this.fixShared_fieldChange.push(deKey); + if (!deployColumns.length || !Object.keys(this.fixShared_fieldChange[deKey]).length) { + // no changed fields found. remove entry from list for easier processing + delete this.fixShared_fieldChange[deKey]; } return existingFieldByName; }