Skip to content

Commit

Permalink
Merge pull request #1480 from Accenture/task/1470-improve_folder_caching
Browse files Browse the repository at this point in the history 
Task/1470 improve folder caching
  • Loading branch information
@JoernBerkefeld
JoernBerkefeld authored Jul 22, 2024
2 parents 5bb9ecf + 7225a8e commit 8a4a387
Show file tree
Hide file tree
Showing 48 changed files with 3,210 additions and 173 deletions.
2 changes: 1 addition & 1 deletion @types/lib/Deployer.d.ts.map
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion @types/lib/metadataTypes/Folder.d.ts.map
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

16 changes: 9 additions & 7 deletions lib/Deployer.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -298,13 +298,6 @@ class Deployer {
}
}

if (!Util.OPTIONS.fromRetrieve) {
await Deployer.createFolderDefinitions(
this.deployDir,
this.metadata,
Object.keys(this.metadata)
);
}
const foundDeployTypes = Object.keys(this.metadata)
// remove empty types
.filter((type) => Object.keys(this.metadata[type]).length)
Expand All @@ -318,6 +311,15 @@ class Deployer {
throw new Error('No metadata found for deployment');
}
const deployOrder = Util.getMetadataHierachy(foundDeployTypes);

if (!Util.OPTIONS.fromRetrieve) {
// run this AFTER identifying deployOrder or else ALL folders will be cached
await Deployer.createFolderDefinitions(
this.deployDir,
this.metadata,
Object.keys(this.metadata)
);
}
// build cache, including all metadata types which will be deployed (Avoids retrieve later)
for (const metadataType in deployOrder) {
const type = metadataType;
Expand Down
43 changes: 22 additions & 21 deletions lib/metadataTypes/Folder.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,12 +38,17 @@ class Folder extends MetadataType {
if (key) {
Util.logger.error(`Folder.retrieve() does not support key parameter`);
}
const queryAllFolders = await this.retrieveHelper(additionalFields, true, subTypeArr);
const queryAllFolders = await this.retrieveHelper(additionalFields, false, subTypeArr);

if (this.buObject.eid !== this.buObject.mid) {
queryAllFolders.push(
...(await this.retrieveHelper(additionalFields, false, subTypeArr))
);
const selectedParentTypes = subTypeArr?.length
? subTypeArr.filter((item) => this.definition.folderTypesFromParent.includes(item))
: subTypeArr;
if (!subTypeArr?.length || selectedParentTypes.length) {
queryAllFolders.push(
...(await this.retrieveHelper(additionalFields, true, selectedParentTypes))
);
}
}
const sortPairs = toposort(queryAllFolders.map((a) => [a.ParentFolder.ID, a.ID]));
const idMap = {};
Expand Down Expand Up @@ -578,23 +583,19 @@ class Folder extends MetadataType {
static async retrieveHelper(additionalFields, queryAllAccounts, contentTypeList) {
const options = { QueryAllAccounts: !!queryAllAccounts };
if (contentTypeList) {
for (const contentType of contentTypeList) {
options.filter = options.filter
? {
leftOperand: {
leftOperand: 'ContentType',
operator: 'equals',
rightOperand: contentType,
},
operator: 'OR',
rightOperand: options.filter,
}
: {
leftOperand: 'ContentType',
operator: 'equals',
rightOperand: contentType,
};
}
const newFilter = {
leftOperand: 'ContentType',
operator: contentTypeList.length === 1 ? 'equals' : 'IN',
rightOperand:
contentTypeList.length === 1 ? contentTypeList[0] : contentTypeList.sort(),
};
options.filter = options.filter
? {
leftOperand: newFilter,
operator: 'OR',
rightOperand: options.filter,
}
: newFilter;
}
const response = await this.client.soap.retrieveBulk(
'DataFolder',
Expand Down
2 changes: 1 addition & 1 deletion lib/metadataTypes/definitions/Folder.definition.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,10 @@ export default {
'dataextension',
'filteractivity',
'filterdefinition',
'hidden',
'journey',
'list',
'mysubs',
'mysubs',
'publication',
'queryactivity',
'salesforcedataextension',
Expand Down
2 changes: 1 addition & 1 deletion lib/util/util.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1045,7 +1045,7 @@ export const Util = {
getTypeKeyCount(selectedTypes) {
return Object.keys(selectedTypes).reduce(
(previousValue, type) =>
selectedTypes[type] ? previousValue + selectedTypes[type].length : previousValue,
previousValue + (selectedTypes[type] ? selectedTypes[type].length : 0),
0
);
},
Expand Down
48 changes: 12 additions & 36 deletions test/general.test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ describe('GENERAL', () => {

assert.equal(
testUtils.getAPIHistoryLength(),
38,
36,
'Unexpected number of requests made. Run testUtils.logAPIHistoryDebug() to see the requests'
);
return;
Expand Down Expand Up @@ -110,7 +110,7 @@ describe('GENERAL', () => {

assert.equal(
testUtils.getAPIHistoryLength(),
38,
36,
'Unexpected number of requests made. Run testUtils.logAPIHistoryDebug() to see the requests'
);
return;
Expand Down Expand Up @@ -159,7 +159,7 @@ describe('GENERAL', () => {

assert.equal(
testUtils.getAPIHistoryLength(),
38,
36,
'Unexpected number of requests made. Run testUtils.logAPIHistoryDebug() to see the requests'
);
return;
Expand Down Expand Up @@ -226,7 +226,7 @@ describe('GENERAL', () => {

assert.equal(
testUtils.getAPIHistoryLength(),
88,
83,
'Unexpected number of requests made. Run testUtils.logAPIHistoryDebug() to see the requests'
);
return;
Expand Down Expand Up @@ -287,7 +287,7 @@ describe('GENERAL', () => {

assert.equal(
testUtils.getAPIHistoryLength(),
88,
83,
'Unexpected number of requests made. Run testUtils.logAPIHistoryDebug() to see the requests'
);
return;
Expand Down Expand Up @@ -348,7 +348,7 @@ describe('GENERAL', () => {

assert.equal(
testUtils.getAPIHistoryLength(),
88,
83,
'Unexpected number of requests made. Run testUtils.logAPIHistoryDebug() to see the requests'
);
return;
Expand Down Expand Up @@ -695,7 +695,7 @@ describe('GENERAL', () => {
// download first before we test buildTemplate
await handler.retrieve('testInstance/testBU', ['automation', 'query']);

const expectedApiCallsRetrieve = 25;
const expectedApiCallsRetrieve = 24;
assert.equal(
testUtils.getAPIHistoryLength(),
expectedApiCallsRetrieve,
Expand Down Expand Up @@ -800,15 +800,7 @@ describe('GENERAL', () => {
});

it('buildTemplate + buildDefinition for multiple types with keys and --retrieve', async () => {
// download first before we test buildTemplate
// await handler.retrieve('testInstance/testBU', ['automation', 'query']);

const expectedApiCallsRetrieve = 28;
// assert.equal(
// testUtils.getAPIHistoryLength(),
// expectedApiCallsRetrieve,
// 'Unexpected number of requests made. Run testUtils.logAPIHistoryDebug() to see the requests'
// );
const expectedApiCallsRetrieve = 27;

// preparation
const argvMetadata = [
Expand Down Expand Up @@ -1041,15 +1033,7 @@ describe('GENERAL', () => {
});

it('buildTemplate + buildDefinition for multiple types with keys and --dependencies and --retrieve', async () => {
// download first before we test buildTemplate
// await handler.retrieve('testInstance/testBU');

const expectedApiCallsRetrieve = 78;
// assert.equal(
// testUtils.getAPIHistoryLength(),
// expectedApiCallsRetrieve,
// 'Unexpected number of requests made. Run testUtils.logAPIHistoryDebug() to see the requests'
// );

// preparation
const argvMetadata = [
Expand Down Expand Up @@ -1174,7 +1158,7 @@ describe('GENERAL', () => {
// download first before we test buildTemplate
await handler.retrieve('testInstance/testBU', ['automation', 'query']);

const expectedApiCallsRetrieve = 25;
const expectedApiCallsRetrieve = 24;
assert.equal(
testUtils.getAPIHistoryLength(),
expectedApiCallsRetrieve,
Expand Down Expand Up @@ -1366,7 +1350,7 @@ describe('GENERAL', () => {
// download first before we test buildTemplate
await handler.retrieve('testInstance/testBU', ['automation', 'query']);

const expectedApiCallsRetrieve = 25;
const expectedApiCallsRetrieve = 24;
assert.equal(
testUtils.getAPIHistoryLength(),
expectedApiCallsRetrieve,
Expand Down Expand Up @@ -1567,15 +1551,7 @@ describe('GENERAL', () => {
});

it('build multiple type with keys and --dependencies and --retrieve', async () => {
// download everything before we test buildTemplate
// await handler.retrieve('testInstance/testBU');

const expectedApiCallsRetrieve = 78;
// assert.equal(
// testUtils.getAPIHistoryLength(),
// expectedApiCallsRetrieve,
// 'Unexpected number of requests made. Run testUtils.logAPIHistoryDebug() to see the requests'
// );

// preparation
const argvMetadata = [
Expand Down Expand Up @@ -1682,7 +1658,7 @@ describe('GENERAL', () => {
// download first before we test buildTemplate
await handler.retrieve('testInstance/testBU', ['automation', 'query']);

const expectedApiCallsRetrieve = 25;
const expectedApiCallsRetrieve = 24;
assert.equal(
testUtils.getAPIHistoryLength(),
expectedApiCallsRetrieve,
Expand Down Expand Up @@ -2308,7 +2284,7 @@ describe('GENERAL', () => {
);
assert.equal(
testUtils.getAPIHistoryLength(),
9,
8,
'Unexpected number of requests made. Run testUtils.logAPIHistoryDebug() to see the requests'
);
});
Expand Down
Loading

0 comments on commit 8a4a387

Please sign in to comment.