Skip to content

Commit

Permalink
#940: check if attributeSet fields actually need fixing
Browse files Browse the repository at this point in the history
  • Loading branch information
@JoernBerkefeld
JoernBerkefeld committed Aug 17, 2023
1 parent 1076b70 commit 7947ce3
Show file tree
Hide file tree
Showing 4 changed files with 131 additions and 29 deletions.
11 changes: 6 additions & 5 deletions docs/dist/documentation.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1325,7 +1325,7 @@ AttributeSet MetadataType
* [AttributeSet](#AttributeSet) ⇐ [<code>MetadataType</code>](#MetadataType)
* [.retrieve(retrieveDir, [_], [__], [key])](#AttributeSet.retrieve) ⇒ <code>Promise.&lt;TYPE.MetadataTypeMapObj&gt;</code>
* [.retrieveForCache()](#AttributeSet.retrieveForCache) ⇒ <code>Promise.&lt;TYPE.MetadataTypeMapObj&gt;</code>
* [.retrieveForSharedDEs(sharedDataExtensionIds)](#AttributeSet.retrieveForSharedDEs) ⇒ <code>Promise.&lt;Array.&lt;string&gt;&gt;</code>
* [.fixShared_retrieve(sharedDataExtensionMap, fixShared_fieldChange)](#AttributeSet.fixShared_retrieve) ⇒ <code>Promise.&lt;Array.&lt;string&gt;&gt;</code>
* [.parseResponseBody(body, [singleRetrieve])](#AttributeSet.parseResponseBody) ⇒ <code>TYPE.MetadataTypeMap</code>
* [.postRetrieveTasks(metadata)](#AttributeSet.postRetrieveTasks) ⇒ <code>TYPE.MetadataTypeItem</code>
* [._getSystemValueDefinitions()](#AttributeSet._getSystemValueDefinitions) ⇒ <code>Array.&lt;object&gt;</code>
Expand All @@ -1352,17 +1352,18 @@ Retrieves Metadata of schema set definitions for caching.

**Kind**: static method of [<code>AttributeSet</code>](#AttributeSet)
**Returns**: <code>Promise.&lt;TYPE.MetadataTypeMapObj&gt;</code> - Promise
<a name="AttributeSet.retrieveForSharedDEs"></a>
<a name="AttributeSet.fixShared_retrieve"></a>

### AttributeSet.retrieveForSharedDEs(sharedDataExtensionIds) ⇒ <code>Promise.&lt;Array.&lt;string&gt;&gt;</code>
### AttributeSet.fixShared\_retrieve(sharedDataExtensionMap, fixShared_fieldChange) ⇒ <code>Promise.&lt;Array.&lt;string&gt;&gt;</code>
Retrieves Metadata of schema set definitions for caching.

**Kind**: static method of [<code>AttributeSet</code>](#AttributeSet)
**Returns**: <code>Promise.&lt;Array.&lt;string&gt;&gt;</code> - Promise of list of shared dataExtension IDs

| Param | Type | Description |
| --- | --- | --- |
| sharedDataExtensionIds | <code>Array.&lt;string&gt;</code> | ID array for shared data extensions |
| sharedDataExtensionMap | <code>Object.&lt;string, string&gt;</code> | ID-Key relationship of shared data extensions |
| fixShared_fieldChange | <code>object</code> | DataExtensionField.fixShared_fieldChange |

<a name="AttributeSet.parseResponseBody"></a>

Expand Down Expand Up @@ -1918,7 +1919,7 @@ Retrieves dataExtension metadata. Afterwards starts retrieval of dataExtensionCo

### DataExtension.retrieveSharedForCache([additionalFields]) ⇒ <code>Promise.&lt;TYPE.DataExtensionMap&gt;</code>
get shared dataExtensions from parent BU and merge them into the cache
helper for [retrieve](#DataExtension.retrieve) and for AttributeSet.retrieveForSharedDEs
helper for [retrieve](#DataExtension.retrieve) and for AttributeSet.fixShared_retrieve

**Kind**: static method of [<code>DataExtension</code>](#DataExtension)
**Returns**: <code>Promise.&lt;TYPE.DataExtensionMap&gt;</code> - keyField => metadata map
Expand Down
108 changes: 97 additions & 11 deletions lib/metadataTypes/AttributeSet.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,28 +50,114 @@ class AttributeSet extends MetadataType {
/**
* Retrieves Metadata of schema set definitions for caching.
*
* @param {string[]} sharedDataExtensionIds ID array for shared data extensions
* @param {Object.<string, string>} sharedDataExtensionMap ID-Key relationship of shared data extensions
* @param {object} fixShared_fieldChange DataExtensionField.fixShared_fieldChange
* @returns {Promise.<string[]>} Promise of list of shared dataExtension IDs
*/
static async retrieveForSharedDEs(sharedDataExtensionIds) {
if (!sharedDataExtensionIds.length) {
static async fixShared_retrieve(sharedDataExtensionMap, fixShared_fieldChange) {
if (!Object.keys(sharedDataExtensionMap).length) {
return [];
}
const result = await super.retrieveREST(null, '/hub/v1/contacts/schema/setDefinitions');
const metadataMap = result?.metadata;
if (metadataMap && Object.keys(metadataMap).length) {
const sharedDEs = Object.keys(metadataMap)
const sharedDeIds = Object.keys(metadataMap)
.filter(
(key) =>
metadataMap[key].storageLogicalType === 'ExactTargetSchema' ||
metadataMap[key].storageLogicalType === 'DataExtension'
)
.filter((key) =>
sharedDataExtensionIds.includes(metadataMap[key].storageReferenceID.value)
(asKey) =>
metadataMap[asKey].storageLogicalType === 'ExactTargetSchema' ||
metadataMap[asKey].storageLogicalType === 'DataExtension'
)
.filter((asKey) => {
// check if dataExtension ID is found on any attributeSet of this BU
if (sharedDataExtensionMap[metadataMap[asKey].storageReferenceID.value]) {
Util.logger.debug(
` shared dataExtension ID ${metadataMap[asKey].storageReferenceID.value} found in attributeGroup ${asKey}`
);
return true;
} else {
return false;
}
})
.filter((asKey) => {
// check if any of the updated dataExtension fields dont exist on the attributeSet or are out of date
const deKey =
sharedDataExtensionMap[metadataMap[asKey].storageReferenceID.value];
const asFields = metadataMap[asKey].valueDefinitions;
if (!fixShared_fieldChange[deKey]) {
Util.logger.debug(
` - No changed fields found. Assuming re-deploy with --fixShared flag`
);
return true;
}
const deFields = Object.values(fixShared_fieldChange[deKey]);
return deFields.some((deField) => {
const search = asFields.filter((asf) => asf.name === deField.Name);
if (!search.length) {
Util.logger.debug(
Util.getGrayMsg(
` - Field ${deField.Name} not found in attributeSet`
)
);
return true;
}
const asField = search[0];
if (asField.dataType !== deField.FieldType) {
Util.logger.debug(
Util.getGrayMsg(
` - Field ${deField.Name} FieldType changed (old: ${asField.dataType}; new: ${deField.FieldType})`
)
);
return true;
}
if (
(asField.defaultValue && deField.DefaultValue !== '') ||
(deField.FieldType === 'Boolean' &&
deField.DefaultValue !== '' &&
deField.DefaultValue
? 'True'
: 'False' !== asField.defaultValue) ||
(deField.FieldType !== 'Boolean' &&
deField.DefaultValue !== asField.defaultValue)
) {
Util.logger.debug(
Util.getGrayMsg(
` - Field ${deField.Name} DefaultValue changed (old: ${asField.defaultValue}; new: ${deField.DefaultValue})`
)
);
return true;
}
// some field types don't carry the length property. reset to 0 to ease comparison
asField.length ||= 0;
if (asField.length !== deField.MaxLength) {
Util.logger.debug(
Util.getGrayMsg(
` - Field ${deField.Name} MaxLength changed (old: ${asField.length}; new: ${deField.MaxLength})`
)
);
return true;
}
if (asField.isNullable !== deField.IsRequired) {
Util.logger.debug(
Util.getGrayMsg(
` - Field ${deField.Name} IsRequired changed (old: ${asField.isNullable}; new: ${deField.IsRequired})`
)
);
return true;
}
if (asField.isPrimaryKey !== deField.IsPrimaryKey) {
Util.logger.debug(
Util.getGrayMsg(
` - Field ${deField.Name} IsPrimaryKey changed (old: ${asField.isPrimaryKey}; new: ${deField.IsPrimaryKey})`
)
);
return true;
}
return false;
});
})
.map((key) => metadataMap[key].storageReferenceID.value)
.filter(Boolean);
return sharedDEs;
return sharedDeIds;
} else {
// nothing to do - return empty array
return [];
Expand Down
24 changes: 14 additions & 10 deletions lib/metadataTypes/DataExtension.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -389,16 +389,19 @@ class DataExtension extends MetadataType {
return;
}

const sharedDataExtensionsKeys = this.deployedSharedKeys.filter((key) =>
DataExtensionField.fixShared_fieldChange.includes(key)
);
const sharedDataExtensionsKeys = this.deployedSharedKeys;
this.deployedSharedKeys = null;
if (!sharedDataExtensionsKeys.length) {
// only if updates were made could the issue in https://issues.salesforce.com/#q=W-11031095 affect data designer
if (
!sharedDataExtensionsKeys.filter(
(deKey) =>
DataExtensionField.fixShared_fieldChange[deKey] &&
Object.keys(DataExtensionField.fixShared_fieldChange[deKey])?.length
)
) {
Util.logger.debug(
`Skipping fixShared logic because no fields were changed on updated Shared Data Extensions`
` - No changed fields found. Assuming re-deploy with --fixShared flag`
);
return;
return true;
}

if (Util.OPTIONS.fixShared) {
Expand Down Expand Up @@ -532,8 +535,9 @@ class DataExtension extends MetadataType {
AttributeSet.properties = this.properties;
AttributeSet.buObject = buObjectChildBu;
AttributeSet.client = clientChildBu;
const sharedDeIdsUsedOnBU = await AttributeSet.retrieveForSharedDEs(
Object.keys(sharedDataExtensionMap)
const sharedDeIdsUsedOnBU = await AttributeSet.fixShared_retrieve(
sharedDataExtensionMap,
DataExtensionField.fixShared_fieldChange
);
if (sharedDeIdsUsedOnBU.length) {
let sharedDataExtensionsKeys = sharedDeIdsUsedOnBU.map(
Expand Down Expand Up @@ -791,7 +795,7 @@ class DataExtension extends MetadataType {

/**
* get shared dataExtensions from parent BU and merge them into the cache
* helper for {@link DataExtension.retrieve} and for AttributeSet.retrieveForSharedDEs
* helper for {@link DataExtension.retrieve} and for AttributeSet.fixShared_retrieve
*
* @param {string[]} [additionalFields] Returns specified fields even if their retrieve definition is not set to true
* @returns {Promise.<TYPE.DataExtensionMap>} keyField => metadata map
Expand Down
17 changes: 14 additions & 3 deletions lib/metadataTypes/DataExtensionField.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,10 @@ class DataExtensionField extends MetadataType {
* @returns {Promise.<Object.<string, TYPE.DataExtensionFieldItem>>} existing fields by their original name to allow re-adding FieldType after update
*/
static async prepareDeployColumnsOnUpdate(deployColumns, deKey) {
// create list of DE keys that had changes to their fields to be able to use it as a filter in the --fixShared logic
this.fixShared_fieldChange ||= {};
this.fixShared_fieldChange[deKey] ||= {};

// get row count to know which field restrictions apply
let hasData = false;
try {
Expand Down Expand Up @@ -189,6 +193,9 @@ class DataExtensionField extends MetadataType {
Util.logger.verbose(`no change - removed field [${deKey}].[${item.Name}]`);
continue;
}
// track name of changed field
this.fixShared_fieldChange[deKey][item.Name] = JSON.parse(JSON.stringify(item));
this.fixShared_fieldChange[deKey][item.Name].FieldType = itemOld.FieldType;

// set the ObjectId for clear identification during update
item.ObjectID = itemOld.ObjectID;
Expand Down Expand Up @@ -217,6 +224,8 @@ class DataExtensionField extends MetadataType {
}
// Field doesn't exist in target, therefore Remove ObjectID if present
delete item.ObjectID;

this.fixShared_fieldChange[deKey][item.Name] = JSON.parse(JSON.stringify(item));
}
if (Util.isTrue(item.IsPrimaryKey) && Util.isFalse(item.IsRequired)) {
// applicable: with or without data
Expand All @@ -232,12 +241,14 @@ class DataExtensionField extends MetadataType {
`- Invalid value for 'IsRequired' of [${deKey}].[${item.Name}]. Found '${item.IsRequired}' instead of 'true'/'false'. Removing field from deploy!`
);
deployColumns.splice(i, 1);
delete this.fixShared_fieldChange[deKey][item.Name];
}
if (!Util.isTrue(item.IsPrimaryKey) && !Util.isFalse(item.IsPrimaryKey)) {
Util.logger.error(
`- Invalid value for 'IsPrimaryKey' of [${deKey}].[${item.Name}]. Found '${item.IsPrimaryKey}' instead of 'true'/'false'. Removing field from deploy!`
);
deployColumns.splice(i, 1);
delete this.fixShared_fieldChange[deKey][item.Name];
}
}

Expand All @@ -249,9 +260,9 @@ class DataExtensionField extends MetadataType {
)
);
// create list of DE keys that had changes to their fields to be able to use it as a filter in the --fixShared logic
this.fixShared_fieldChange ||= [];
if (deployColumns.length) {
this.fixShared_fieldChange.push(deKey);
if (!deployColumns.length || !Object.keys(this.fixShared_fieldChange[deKey]).length) {
// no changed fields found. remove entry from list for easier processing
delete this.fixShared_fieldChange[deKey];
}
return existingFieldByName;
}
Expand Down

0 comments on commit 7947ce3

Please sign in to comment.