From 524abfbfcde2ca628b60b48a4a208b832a535f3b Mon Sep 17 00:00:00 2001 From: Ariel Chinn Date: Tue, 15 Oct 2024 17:57:53 +0300 Subject: [PATCH] =?UTF-8?q?Revert=20"SALTO-4990:=20Salesforce:=20Remodel?= =?UTF-8?q?=20picklist=20value=20lists=20to=20maps,=20to=20allo=E2=80=A6"?= =?UTF-8?q?=20(#6680)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit b367e6505c0c94622281f52be8401cc535ae9276. --- packages/salesforce-adapter/jest.config.js | 2 +- packages/salesforce-adapter/src/adapter.ts | 3 +- .../src/change_validator.ts | 2 - .../change_validators/multiple_defaults.ts | 47 +-- .../src/change_validators/ordered_maps.ts | 137 -------- .../src/fetch_profile/fetch_profile.ts | 1 - .../src/filters/convert_maps.ts | 316 ++++-------------- packages/salesforce-adapter/src/types.ts | 4 - .../change_validators/ordered_maps.test.ts | 170 ---------- .../test/filters/convert_maps.test.ts | 121 +------ 10 files changed, 74 insertions(+), 729 deletions(-) delete mode 100644 packages/salesforce-adapter/src/change_validators/ordered_maps.ts delete mode 100644 packages/salesforce-adapter/test/change_validators/ordered_maps.test.ts diff --git a/packages/salesforce-adapter/jest.config.js b/packages/salesforce-adapter/jest.config.js index 07a5a75a757..a4a84f14bd7 100644 --- a/packages/salesforce-adapter/jest.config.js +++ b/packages/salesforce-adapter/jest.config.js @@ -16,7 +16,7 @@ module.exports = deepMerge(require('../../jest.base.config.js'), { : undefined, coverageThreshold: { global: { - branches: 87.75, + branches: 87.8, functions: 94, lines: 95, statements: 95, diff --git a/packages/salesforce-adapter/src/adapter.ts b/packages/salesforce-adapter/src/adapter.ts index 0415cfe39ed..bf177bd07a9 100644 --- a/packages/salesforce-adapter/src/adapter.ts +++ b/packages/salesforce-adapter/src/adapter.ts @@ -197,10 +197,9 @@ export const allFilters: Array = [ profilePermissionsFilter, // emailTemplateFilter should run before convertMapsFilter emailTemplateFilter, - // standardValueSetFilter should run before convertMapsFilter - standardValueSetFilter, // convertMapsFilter should run before profile fieldReferencesFilter convertMapsFilter, + standardValueSetFilter, flowFilter, customObjectInstanceReferencesFilter, cpqReferencableFieldReferencesFilter, diff --git a/packages/salesforce-adapter/src/change_validator.ts b/packages/salesforce-adapter/src/change_validator.ts index 21ebd49909e..4306bd691e1 100644 --- a/packages/salesforce-adapter/src/change_validator.ts +++ b/packages/salesforce-adapter/src/change_validator.ts @@ -45,7 +45,6 @@ import elementApiVersionValidator from './change_validators/element_api_version' import cpqBillingStartDate from './change_validators/cpq_billing_start_date' import cpqBillingTriggers from './change_validators/cpq_billing_triggers' import managedApexComponent from './change_validators/managed_apex_component' -import orderedMaps from './change_validators/ordered_maps' import SalesforceClient from './client/client' import { ChangeValidatorName, DEPLOY_CONFIG, FetchProfile, SalesforceConfig } from './types' import { buildFetchProfile } from './fetch_profile/fetch_profile' @@ -105,7 +104,6 @@ export const changeValidators: Record cpqBillingStartDate, cpqBillingTriggers: () => cpqBillingTriggers, managedApexComponent: () => managedApexComponent, - orderedMaps: ({ fetchProfile }) => orderedMaps(fetchProfile), ..._.mapValues(getDefaultChangeValidators(), validator => () => validator), } diff --git a/packages/salesforce-adapter/src/change_validators/multiple_defaults.ts b/packages/salesforce-adapter/src/change_validators/multiple_defaults.ts index cc1bfd714a9..b7c5cb0e5f4 100644 --- a/packages/salesforce-adapter/src/change_validators/multiple_defaults.ts +++ b/packages/salesforce-adapter/src/change_validators/multiple_defaults.ts @@ -21,7 +21,6 @@ import { Value, Values, isReferenceExpression, - getField, } from '@salto-io/adapter-api' import { safeJsonStringify } from '@salto-io/adapter-utils' import { collections } from '@salto-io/lowerdash' @@ -39,13 +38,8 @@ type FieldDef = { const FIELD_NAME_TO_INNER_CONTEXT_FIELD: Record = { applicationVisibilities: { name: 'application' }, recordTypeVisibilities: { name: 'recordType', nested: true }, - - // TODO(SALTO-4990): Remove once picklistsAsMaps FF is deployed and removed. standardValue: { name: 'label' }, customValue: { name: 'label' }, - - 'standardValue.values': { name: 'label' }, - 'customValue.values': { name: 'label' }, } type ValueSetInnerObject = { @@ -53,31 +47,14 @@ type ValueSetInnerObject = { label: string } & Values[] -type FieldWithValueSetList = Field & { +type FieldWithValueSet = Field & { annotations: { valueSet: Array } } -// TODO(SALTO-4990): Remove once picklistsAsMaps FF is deployed and removed. -type FieldWithValueSetOrderedMap = Field & { - annotations: { - valueSet: { - values: Array - } - } -} - -type FieldWithValueSet = FieldWithValueSetList | FieldWithValueSetOrderedMap - -const isFieldWithValueSetList = (field: Field): field is FieldWithValueSetList => - _.isArray(field.annotations[FIELD_ANNOTATIONS.VALUE_SET]) - -const isFieldWithOrderedMapValueSet = (field: Field): field is FieldWithValueSetOrderedMap => - _.isArray(field.annotations[FIELD_ANNOTATIONS.VALUE_SET]?.order) - const isFieldWithValueSet = (field: Field): field is FieldWithValueSet => - isFieldWithValueSetList(field) || isFieldWithOrderedMapValueSet(field) + _.isArray(field.annotations[FIELD_ANNOTATIONS.VALUE_SET]) const formatContext = (context: Value): string => { if (isReferenceExpression(context)) { @@ -107,9 +84,7 @@ const createFieldChangeError = (field: Field, contexts: string[]): ChangeError = }) const getPicklistMultipleDefaultsErrors = (field: FieldWithValueSet): ChangeError[] => { - const contexts = ( - isFieldWithValueSetList(field) ? field.annotations.valueSet : Object.values(field.annotations.valueSet.values) - ) + const contexts = field.annotations.valueSet .filter(obj => obj.default) .map(obj => obj[LABEL]) .map(formatContext) @@ -137,9 +112,6 @@ const getInstancesMultipleDefaultsErrors = async (after: InstanceElement): Promi valueName: string, ): Promise => { const defaultObjects = await getDefaultObjectsList(value, fieldType) - if (!_.isArray(defaultObjects)) { - return undefined - } const contexts = defaultObjects .filter(val => val.default) .map(obj => obj[valueName]) @@ -158,18 +130,17 @@ const getInstancesMultipleDefaultsErrors = async (after: InstanceElement): Promi return [] } - const errors: ChangeError[] = await awu(Object.keys(FIELD_NAME_TO_INNER_CONTEXT_FIELD)) - .filter(fieldPath => _.has(after.value, fieldPath)) - .flatMap(async fieldPath => { - const value = _.get(after.value, fieldPath) - const field = await getField(await after.getType(), fieldPath.split('.')) + const errors: ChangeError[] = await awu(Object.entries(after.value)) + .filter(([fieldName]) => Object.keys(FIELD_NAME_TO_INNER_CONTEXT_FIELD).includes(fieldName)) + .flatMap(async ([fieldName, value]) => { + const field = (await after.getType()).fields[fieldName] if (field === undefined) { // Can happen if the field exists in the instance but not in the type. return [] } const fieldType = await field.getType() - const valueName = FIELD_NAME_TO_INNER_CONTEXT_FIELD[fieldPath].name - if (_.isPlainObject(value) && FIELD_NAME_TO_INNER_CONTEXT_FIELD[fieldPath].nested) { + const valueName = FIELD_NAME_TO_INNER_CONTEXT_FIELD[fieldName].name + if (_.isPlainObject(value) && FIELD_NAME_TO_INNER_CONTEXT_FIELD[fieldName].nested) { return awu(Object.entries(value)).flatMap(async ([_key, innerValue]) => { const startLevelType = isMapType(fieldType) ? await fieldType.getInnerType() : fieldType const defaultsContexts = await findMultipleDefaults(innerValue, startLevelType, valueName) diff --git a/packages/salesforce-adapter/src/change_validators/ordered_maps.ts b/packages/salesforce-adapter/src/change_validators/ordered_maps.ts deleted file mode 100644 index ec280803477..00000000000 --- a/packages/salesforce-adapter/src/change_validators/ordered_maps.ts +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Copyright 2024 Salto Labs Ltd. - * Licensed under the Salto Terms of Use (the "License"); - * You may not use this file except in compliance with the License. You may obtain a copy of the License at https://www.salto.io/terms-of-use - * - * CERTAIN THIRD PARTY SOFTWARE MAY BE CONTAINED IN PORTIONS OF THE SOFTWARE. See NOTICE FILE AT https://github.com/salto-io/salto/blob/main/NOTICES - */ -import _ from 'lodash' -import { - ChangeError, - ChangeValidator, - Element, - ElemID, - getChangeData, - isFieldChange, - isInstanceChange, - isObjectTypeChange, - isReferenceExpression, - Value, -} from '@salto-io/adapter-api' -import { collections } from '@salto-io/lowerdash' -import { - metadataTypeToFieldToMapDef, - annotationDefsByType, - findInstancesToConvert, - getElementValueOrAnnotations, - getChangesWithFieldType, -} from '../filters/convert_maps' -import { FetchProfile } from '../types' - -const { awu } = collections.asynciterable - -export const getOrderedMapErrors = (element: Element, fieldName: string): ChangeError[] => { - const elementValues = getElementValueOrAnnotations(element) - const fieldValue = _.get(elementValues, fieldName) - if (fieldValue === undefined) { - return [] - } - const { values, order } = fieldValue - if (order === undefined || values === undefined) { - return [ - { - elemID: element.elemID, - severity: 'Error', - message: 'Missing field in ordered map', - detailedMessage: `Missing order or values fields in field ${fieldName}`, - }, - ] - } - const valueElemIds: ElemID[] = Object.keys(values).map(key => element.elemID.createNestedID(fieldName, 'values', key)) - const foundValueElemIds: ElemID[] = [] - const errors: ChangeError[] = [] - order.forEach((valueRef: Value) => { - if ( - !isReferenceExpression(valueRef) || - !valueElemIds.map(elemID => elemID.getFullName()).includes(valueRef.elemID.getFullName()) - ) { - errors.push({ - elemID: element.elemID, - severity: 'Error', - message: 'Invalid reference in ordered map', - detailedMessage: `Invalid reference in field ${fieldName}.order: ${valueRef.elemID?.getFullName() ?? valueRef}. Only reference to internal value keys are allowed.`, - }) - return - } - if (foundValueElemIds.map(elemID => elemID.getFullName()).includes(valueRef.elemID.getFullName())) { - errors.push({ - elemID: element.elemID, - severity: 'Error', - message: 'Duplicate reference in ordered map', - detailedMessage: `Duplicate reference in field ${fieldName}.order: ${valueRef.elemID.name}`, - }) - } - foundValueElemIds.push(valueRef.elemID) - }) - const missingElemIds = valueElemIds.filter( - valueElemId => !foundValueElemIds.map(elemID => elemID.getFullName()).includes(valueElemId.getFullName()), - ) - if (!_.isEmpty(missingElemIds)) { - errors.push({ - elemID: element.elemID, - severity: 'Error', - message: 'Missing reference in ordered map', - detailedMessage: `Missing reference in field ${fieldName}.order: ${missingElemIds.map(elemID => elemID.name).join(', ')}`, - }) - } - return errors -} - -const changeValidator: (fetchProfile: FetchProfile) => ChangeValidator = fetchProfile => { - if (!fetchProfile.isFeatureEnabled('picklistsAsMaps')) { - return async () => [] - } - return async changes => { - const instanceErrors: ChangeError[] = await awu(Object.keys(metadataTypeToFieldToMapDef)) - .flatMap(async targetMetadataType => { - const instances = await findInstancesToConvert( - changes.filter(isInstanceChange).map(getChangeData), - targetMetadataType, - ) - if (_.isEmpty(instances)) { - return [] - } - const fieldNames = Object.entries(metadataTypeToFieldToMapDef[targetMetadataType]) - .filter(([_fieldName, mapDef]) => mapDef.maintainOrder) - .map(([fieldName, _mapDef]) => fieldName) - - return fieldNames.flatMap(fieldName => instances.flatMap(instance => getOrderedMapErrors(instance, fieldName))) - }) - .toArray() - - const objectTypeErrors: ChangeError[] = await awu(Object.keys(annotationDefsByType)) - .flatMap(async fieldType => { - const fieldNames = Object.entries(annotationDefsByType[fieldType]) - .filter(([_fieldName, annotationDef]) => annotationDef.maintainOrder) - .map(([fieldName, _mapDef]) => fieldName) - - const fieldChanges = getChangesWithFieldType(changes, fieldType) - return fieldChanges - .flatMap(change => { - if (isFieldChange(change)) { - return [getChangeData(change)] - } - if (isObjectTypeChange(change)) { - const objectType = getChangeData(change) - return Object.values(objectType.fields).filter(field => field.refType.elemID.typeName === fieldType) - } - return [] - }) - .flatMap(field => fieldNames.flatMap(fieldName => getOrderedMapErrors(field, fieldName))) - }) - .toArray() - return instanceErrors.concat(objectTypeErrors) - } -} - -export default changeValidator diff --git a/packages/salesforce-adapter/src/fetch_profile/fetch_profile.ts b/packages/salesforce-adapter/src/fetch_profile/fetch_profile.ts index 6899526d1a8..3a48fb35a66 100644 --- a/packages/salesforce-adapter/src/fetch_profile/fetch_profile.ts +++ b/packages/salesforce-adapter/src/fetch_profile/fetch_profile.ts @@ -50,7 +50,6 @@ const optionalFeaturesDefaultValues: OptionalFeaturesDefaultValues = { logDiffsFromParsingXmlNumbers: true, extendTriggersMetadata: false, removeReferenceFromFilterItemToRecordType: false, - picklistsAsMaps: false, } type BuildFetchProfileParams = { diff --git a/packages/salesforce-adapter/src/filters/convert_maps.ts b/packages/salesforce-adapter/src/filters/convert_maps.ts index 38c3ab7fba0..76d39b4d617 100644 --- a/packages/salesforce-adapter/src/filters/convert_maps.ts +++ b/packages/salesforce-adapter/src/filters/convert_maps.ts @@ -27,12 +27,6 @@ import { getDeepInnerType, isObjectType, getField, - isFieldChange, - ReferenceExpression, - TypeElement, - ElemID, - isObjectTypeChange, - BuiltinTypes, } from '@salto-io/adapter-api' import { collections, values as lowerdashValues } from '@salto-io/lowerdash' import { naclCase, applyFunctionToChangeData } from '@salto-io/adapter-utils' @@ -51,8 +45,6 @@ import { INSTANCE_FULL_NAME_FIELD, } from '../constants' import { metadataType } from '../transformers/transformer' -import { GLOBAL_VALUE_SET } from './global_value_sets' -import { STANDARD_VALUE_SET } from './standard_value_sets' const { awu } = collections.asynciterable const { isDefined } = lowerdashValues @@ -70,33 +62,8 @@ type MapDef = { mapToList?: boolean // with which mapper should we parse the key mapper?: (string: string) => string[] - // keep a separate list of references for each value to preserve the order - // Note: this is only supported for one-level maps (nested maps are not supported) - maintainOrder?: boolean } -const ORDERED_MAP_VALUES_FIELD = 'values' -const ORDERED_MAP_ORDER_FIELD = 'order' - -const createOrderedMapType = (innerType: T): ObjectType => - new ObjectType({ - elemID: new ElemID('salesforce', `OrderedMap<${innerType.elemID.name}>`), - fields: { - [ORDERED_MAP_VALUES_FIELD]: { - refType: new MapType(innerType), - annotations: { - [CORE_ANNOTATIONS.REQUIRED]: true, - }, - }, - [ORDERED_MAP_ORDER_FIELD]: { - refType: new ListType(BuiltinTypes.STRING), - annotations: { - [CORE_ANNOTATIONS.REQUIRED]: true, - }, - }, - }, - }) - /** * Convert a string value into the map index keys. * Note: Reference expressions are not supported yet (the resolved value is not populated in fetch) @@ -169,20 +136,6 @@ const SHARING_RULES_MAP_FIELD_DEF: Record = { sharingOwnerRules: { key: INSTANCE_FULL_NAME_FIELD }, } -const PICKLIST_MAP_FIELD_DEF: MapDef = { - key: 'fullName', - maintainOrder: true, - mapper: (val: string): string[] => [naclCase(val)], -} - -const GLOBAL_VALUE_SET_MAP_FIELD_DEF: Record = { - customValue: PICKLIST_MAP_FIELD_DEF, -} - -const STANDARD_VALUE_SET_MAP_FIELD_DEF: Record = { - standardValue: PICKLIST_MAP_FIELD_DEF, -} - export const metadataTypeToFieldToMapDef: Record> = { [BUSINESS_HOURS_METADATA_TYPE]: BUSINESS_HOURS_MAP_FIELD_DEF, [EMAIL_TEMPLATE_METADATA_TYPE]: EMAIL_TEMPLATE_MAP_FIELD_DEF, @@ -191,33 +144,19 @@ export const metadataTypeToFieldToMapDef: Record> [MUTING_PERMISSION_SET_METADATA_TYPE]: PERMISSIONS_SET_MAP_FIELD_DEF, [LIGHTNING_COMPONENT_BUNDLE_METADATA_TYPE]: LIGHTNING_COMPONENT_BUNDLE_MAP, [SHARING_RULES_TYPE]: SHARING_RULES_MAP_FIELD_DEF, - [GLOBAL_VALUE_SET]: GLOBAL_VALUE_SET_MAP_FIELD_DEF, - [STANDARD_VALUE_SET]: STANDARD_VALUE_SET_MAP_FIELD_DEF, } -export const annotationDefsByType: Record> = { - Picklist: { - valueSet: PICKLIST_MAP_FIELD_DEF, - }, - MultiselectPicklist: { - valueSet: PICKLIST_MAP_FIELD_DEF, - }, -} - -export const getElementValueOrAnnotations = (element: Element): Values => - isInstanceElement(element) ? element.value : element.annotations - /** - * Convert the specified element fields into maps. + * Convert the specified instance fields into maps. * Choose between unique maps and lists based on each field's conversion definition. If a field * should use a unique map but fails due to conflicts, convert it to a list map, and include it * in the returned list so that it can be converted across the board. * - * @param element The instance to modify - * @param mapFieldDef The definitions of the fields to covert + * @param instance The instance to modify + * @param instanceMapFieldDef The definitions of the fields to covert * @returns The list of fields that were converted to non-unique due to duplicates */ -const convertArraysToMaps = (element: Element, mapFieldDef: Record): string[] => { +const convertArraysToMaps = (instance: InstanceElement, instanceMapFieldDef: Record): string[] => { // fields that were intended to be unique, but have multiple values under to the same map key const nonUniqueMapFields: string[] = [] @@ -232,44 +171,28 @@ const convertArraysToMaps = (element: Element, mapFieldDef: Record keyFunc(item)) } - Object.entries(mapFieldDef) - .filter(([fieldName]) => _.get(getElementValueOrAnnotations(element), fieldName) !== undefined) + Object.entries(instanceMapFieldDef) + .filter(([fieldName]) => _.get(instance.value, fieldName) !== undefined) .forEach(([fieldName, mapDef]) => { const mapper = mapDef.mapper ?? defaultMapper - const elementValues = getElementValueOrAnnotations(element) if (mapDef.nested) { const firstLevelGroups = _.groupBy( - makeArray(_.get(elementValues, fieldName)), + makeArray(_.get(instance.value, fieldName)), item => mapper(item[mapDef.key])[0], ) _.set( - elementValues, + instance.value, fieldName, _.mapValues(firstLevelGroups, firstLevelValues => convertField(firstLevelValues, item => mapper(item[mapDef.key])[1], !!mapDef.mapToList, fieldName), ), ) - } else if (mapDef.maintainOrder) { - const originalFieldValue = makeArray(_.get(elementValues, fieldName)) - _.set(elementValues, fieldName, { - [ORDERED_MAP_VALUES_FIELD]: convertField( - originalFieldValue, - item => mapper(item[mapDef.key])[0], - !!mapDef.mapToList, - fieldName, - ), - [ORDERED_MAP_ORDER_FIELD]: originalFieldValue - .map(item => mapper(item[mapDef.key])[0]) - .map( - name => new ReferenceExpression(element.elemID.createNestedID(fieldName, ORDERED_MAP_VALUES_FIELD, name)), - ), - }) } else { _.set( - elementValues, + instance.value, fieldName, convertField( - makeArray(_.get(elementValues, fieldName)), + makeArray(_.get(instance.value, fieldName)), item => mapper(item[mapDef.key])[0], !!mapDef.mapToList, fieldName, @@ -283,28 +206,24 @@ const convertArraysToMaps = (element: Element, mapFieldDef: Record, + instanceMapFieldDef: Record, ): void => { nonUniqueMapFields.forEach(fieldName => { - if (mapFieldDef[fieldName]?.nested) { + if (instanceMapFieldDef[fieldName]?.nested) { _.set( - getElementValueOrAnnotations(element), + instance.value, fieldName, - _.mapValues(_.get(getElementValueOrAnnotations(element), fieldName), val => _.mapValues(val, makeArray)), + _.mapValues(_.get(instance.value, fieldName), val => _.mapValues(val, makeArray)), ) } else { - _.set( - getElementValueOrAnnotations(element), - fieldName, - _.mapValues(_.get(getElementValueOrAnnotations(element), fieldName), makeArray), - ) + _.set(instance.value, fieldName, _.mapValues(_.get(instance.value, fieldName), makeArray)) } }) } @@ -317,7 +236,7 @@ const convertValuesToMapArrays = ( * @param instanceMapFieldDef The original field mapping definition */ const updateFieldTypes = async ( - instanceType: ObjectType | TypeElement, + instanceType: ObjectType, nonUniqueMapFields: string[], instanceMapFieldDef: Record, ): Promise => { @@ -333,8 +252,6 @@ const updateFieldTypes = async ( } if (mapDef.nested) { field.refType = createRefToElmWithValue(new MapType(new MapType(innerType))) - } else if (mapDef.maintainOrder) { - field.refType = createRefToElmWithValue(createOrderedMapType(innerType)) } else { field.refType = createRefToElmWithValue(new MapType(innerType)) } @@ -354,128 +271,74 @@ const updateFieldTypes = async ( }) } -const updateAnnotationRefTypes = async ( - typeElement: TypeElement, - nonUniqueMapFields: string[], - mapFieldDef: Record, -): Promise => { - Object.entries(mapFieldDef).forEach(async ([fieldName, mapDef]) => { - const fieldType = _.get(typeElement.annotationRefTypes, fieldName).type - // navigate to the right field type - if (isDefined(fieldType) && !isMapType(fieldType)) { - let innerType = isContainerType(fieldType) ? await fieldType.getInnerType() : fieldType - if (mapDef.mapToList || nonUniqueMapFields.includes(fieldName)) { - innerType = new ListType(innerType) - } - if (mapDef.nested) { - typeElement.annotationRefTypes[fieldName] = createRefToElmWithValue(new MapType(new MapType(innerType))) - } else if (mapDef.maintainOrder) { - typeElement.annotationRefTypes[fieldName] = createRefToElmWithValue(createOrderedMapType(innerType)) - } else { - typeElement.annotationRefTypes[fieldName] = createRefToElmWithValue(new MapType(innerType)) - } - - // make the key field required - const deepInnerType = await getDeepInnerType(innerType) - if (isObjectType(deepInnerType)) { - const keyFieldType = deepInnerType.fields[mapDef.key] - if (!keyFieldType) { - log.error('could not find key field %s for type %s', mapDef.key, fieldType.elemID.getFullName()) - return - } - keyFieldType.annotations[CORE_ANNOTATIONS.REQUIRED] = true - } - } - }) -} - -const convertElementFieldsToMaps = async ( - elementsToConvert: Element[], - mapFieldDef: Record, +const convertInstanceFieldsToMaps = async ( + instancesToConvert: InstanceElement[], + instanceMapFieldDef: Record, ): Promise => { const nonUniqueMapFields = _.uniq( - elementsToConvert.flatMap(element => { - const nonUniqueFields = convertArraysToMaps(element, mapFieldDef) + instancesToConvert.flatMap(instance => { + const nonUniqueFields = convertArraysToMaps(instance, instanceMapFieldDef) if (nonUniqueFields.length > 0) { - log.info(`Instance ${element.elemID.getFullName()} has non-unique map fields: ${nonUniqueFields}`) + log.info(`Instance ${instance.elemID.getFullName()} has non-unique map fields: ${nonUniqueFields}`) } return nonUniqueFields }), ) if (nonUniqueMapFields.length > 0) { - elementsToConvert.forEach(element => { - convertValuesToMapArrays(element, nonUniqueMapFields, mapFieldDef) + instancesToConvert.forEach(instance => { + convertValuesToMapArrays(instance, nonUniqueMapFields, instanceMapFieldDef) }) } return nonUniqueMapFields } /** - * Convert element field values from maps back to arrays before deploy. + * Convert instance field values from maps back to arrays before deploy. * - * @param changes The changes to deploy - * @param mapFieldDef The definitions of the fields to convert - * @param elementType The type of the elements to convert + * @param instanceChanges The instance changes to deploy + * @param instanceMapFieldDef The definitions of the fields to covert */ const convertFieldsBackToLists = async ( - changes: ReadonlyArray>, - mapFieldDef: Record, - elementType: string, + instanceChanges: ReadonlyArray>, + instanceMapFieldDef: Record, ): Promise => { const toVals = (values: Values): Values[] => Object.values(values).flat() - const backToArrays = (baseElement: Element): Element => { - const elementsToConvert = [] - if (isObjectType(baseElement)) { - Object.values(baseElement.fields) - .filter(field => field.refType.elemID.typeName === elementType) - .forEach(field => elementsToConvert.push(field)) - } else { - elementsToConvert.push(baseElement) - } - elementsToConvert.forEach(element => { - Object.keys(mapFieldDef) - .filter(fieldName => getElementValueOrAnnotations(element)[fieldName] !== undefined) - .forEach(fieldName => { - const elementValues = getElementValueOrAnnotations(element) - if (Array.isArray(_.get(elementValues, fieldName))) { - // should not happen - return - } + const backToArrays = (instance: InstanceElement): InstanceElement => { + Object.keys(instanceMapFieldDef) + .filter(fieldName => _.get(instance.value, fieldName) !== undefined) + .forEach(fieldName => { + if (Array.isArray(_.get(instance.value, fieldName))) { + // should not happen + return + } - if (mapFieldDef[fieldName].nested) { - // first convert the inner levels to arrays, then merge into one array - _.set(elementValues, fieldName, _.mapValues(elementValues[fieldName], toVals)) - } - if (mapFieldDef[fieldName].maintainOrder) { - // OrderedMap keeps the order in a list of references, so we just need to override the top-level OrderedMap - // with this list. - _.set(elementValues, fieldName, elementValues[fieldName][ORDERED_MAP_ORDER_FIELD]) - } else { - _.set(elementValues, fieldName, toVals(elementValues[fieldName])) - } - }) - }) - return baseElement + if (instanceMapFieldDef[fieldName].nested) { + // first convert the inner levels to arrays, then merge into one array + _.set(instance.value, fieldName, _.mapValues(_.get(instance.value, fieldName), toVals)) + } + _.set(instance.value, fieldName, toVals(_.get(instance.value, fieldName))) + }) + return instance } - await awu(changes).forEach(change => applyFunctionToChangeData(change, backToArrays)) + await awu(instanceChanges).forEach(instanceChange => applyFunctionToChangeData(instanceChange, backToArrays)) } /** - * Convert an element's field values from arrays back to maps after deploy. + * Convert instance's field values from arrays back to maps after deploy. * - * @param changes The changes to deploy - * @param mapFieldDef The definitions of the fields to covert + * @param instanceChanges The instance changes to deploy + * @param instanceMapFieldDef The definitions of the fields to covert */ const convertFieldsBackToMaps = ( - changes: ReadonlyArray>, - mapFieldDef: Record, + instanceChanges: ReadonlyArray>, + instanceMapFieldDef: Record, ): void => { - changes.forEach(change => - applyFunctionToChangeData(change, element => { - convertArraysToMaps(element, mapFieldDef) - return element + instanceChanges.forEach(instanceChange => + applyFunctionToChangeData(instanceChange, instance => { + convertArraysToMaps(instance, instanceMapFieldDef) + return instance }), ) } @@ -516,25 +379,6 @@ export const getInstanceChanges = ( .filter(async change => (await metadataType(getChangeData(change))) === targetMetadataType) .toArray() -/** Get all changes that contain a specific field type. - * - * @return All changes that are either field changes of the specified type or object changes that contain fields of the - * specified type - */ -export const getChangesWithFieldType = (changes: ReadonlyArray, fieldType: string): Change[] => { - const fieldChanges: Change[] = changes - .filter(isFieldChange) - .filter(async change => getChangeData(change).getTypeSync().elemID.typeName === fieldType) - - const objectTypeChanges = changes - .filter(isObjectTypeChange) - .filter(change => - Object.values(getChangeData(change).fields).some(field => field.refType.elemID.typeName === fieldType), - ) - - return fieldChanges.concat(objectTypeChanges) -} - export const findInstancesToConvert = (elements: Element[], targetMetadataType: string): Promise => { const instances = elements.filter(isInstanceElement) return awu(instances) @@ -555,18 +399,14 @@ export const findTypeToConvert = async ( } /** - * Convert certain elements' fields into maps, so that they are easier to view, + * Convert certain instances' fields into maps, so that they are easier to view, * could be referenced, and can be split across multiple files. */ const filter: FilterCreator = ({ config }) => ({ name: 'convertMapsFilter', onFetch: async (elements: Element[]) => { await awu(Object.keys(metadataTypeToFieldToMapDef)).forEach(async targetMetadataType => { - if ( - (targetMetadataType === SHARING_RULES_TYPE && !config.fetchProfile.isFeatureEnabled('sharingRulesMaps')) || - ([GLOBAL_VALUE_SET, STANDARD_VALUE_SET].includes(targetMetadataType) && - !config.fetchProfile.isFeatureEnabled('picklistsAsMaps')) - ) { + if (targetMetadataType === SHARING_RULES_TYPE && !config.fetchProfile.isFeatureEnabled('sharingRulesMaps')) { return } const instancesToConvert = await findInstancesToConvert(elements, targetMetadataType) @@ -576,27 +416,11 @@ const filter: FilterCreator = ({ config }) => ({ if (instancesToConvert.length === 0) { await updateFieldTypes(typeToConvert, [], mapFieldDef) } else { - const nonUniqueMapFields = await convertElementFieldsToMaps(instancesToConvert, mapFieldDef) + const nonUniqueMapFields = await convertInstanceFieldsToMaps(instancesToConvert, mapFieldDef) await updateFieldTypes(typeToConvert, nonUniqueMapFields, mapFieldDef) } } }) - - const fields = elements.filter(isObjectType).flatMap(obj => Object.values(obj.fields)) - await awu(Object.entries(annotationDefsByType)).forEach(async ([fieldType, annotationToMapDef]) => { - if ( - ['Picklist', 'MultiselectPicklist'].includes(fieldType) && - !config.fetchProfile.isFeatureEnabled('picklistsAsMaps') - ) { - return - } - const fieldsToConvert = fields.filter(field => field.refType.elemID.typeName === fieldType) - if (fieldsToConvert.length === 0) { - return - } - const nonUniqueMapFields = await convertElementFieldsToMaps(fieldsToConvert, annotationToMapDef) - await updateAnnotationRefTypes(await fieldsToConvert[0].getType(), nonUniqueMapFields, annotationToMapDef) - }) }, preDeploy: async changes => { @@ -609,20 +433,11 @@ const filter: FilterCreator = ({ config }) => ({ // since transformElement and salesforce do not require list fields to be defined as lists, // we only mark fields as lists of their map inner value is a list, // so that we can convert the object back correctly in onDeploy - await convertFieldsBackToLists(instanceChanges, mapFieldDef, targetMetadataType) + await convertFieldsBackToLists(instanceChanges, mapFieldDef) const instanceType = await getChangeData(instanceChanges[0]).getType() await convertFieldTypesBackToLists(instanceType, mapFieldDef) }) - - await awu(Object.keys(annotationDefsByType)).forEach(async fieldType => { - const elementsWithFieldType = getChangesWithFieldType(changes, fieldType) - if (elementsWithFieldType.length === 0) { - return - } - const mapFieldDef = annotationDefsByType[fieldType] - await convertFieldsBackToLists(elementsWithFieldType, mapFieldDef, fieldType) - }) }, onDeploy: async changes => { @@ -643,15 +458,6 @@ const filter: FilterCreator = ({ config }) => ({ .toArray() await updateFieldTypes(instanceType, nonUniqueMapFields, mapFieldDef) }) - - await awu(Object.keys(annotationDefsByType)).forEach(async fieldType => { - const fieldsChanges = getChangesWithFieldType(changes, fieldType) - if (fieldsChanges.length === 0) { - return - } - const mapFieldDef = annotationDefsByType[fieldType] - convertFieldsBackToMaps(fieldsChanges, mapFieldDef) - }) }, }) diff --git a/packages/salesforce-adapter/src/types.ts b/packages/salesforce-adapter/src/types.ts index 17aff844b14..e39548f3fca 100644 --- a/packages/salesforce-adapter/src/types.ts +++ b/packages/salesforce-adapter/src/types.ts @@ -127,7 +127,6 @@ export type OptionalFeatures = { logDiffsFromParsingXmlNumbers?: boolean extendTriggersMetadata?: boolean removeReferenceFromFilterItemToRecordType?: boolean - picklistsAsMaps?: boolean } export type ChangeValidatorName = @@ -167,7 +166,6 @@ export type ChangeValidatorName = | 'cpqBillingStartDate' | 'cpqBillingTriggers' | 'managedApexComponent' - | 'orderedMaps' type ChangeValidatorConfig = Partial> @@ -840,7 +838,6 @@ const optionalFeaturesType = createMatchingObjectType({ logDiffsFromParsingXmlNumbers: { refType: BuiltinTypes.BOOLEAN }, extendTriggersMetadata: { refType: BuiltinTypes.BOOLEAN }, removeReferenceFromFilterItemToRecordType: { refType: BuiltinTypes.BOOLEAN }, - picklistsAsMaps: { refType: BuiltinTypes.BOOLEAN }, }, annotations: { [CORE_ANNOTATIONS.ADDITIONAL_PROPERTIES]: false, @@ -888,7 +885,6 @@ const changeValidatorConfigType = createMatchingObjectType { - const changeValidator = changeValidatorCreator( - buildFetchProfile({ fetchParams: { optionalFeatures: { picklistsAsMaps: true } } }), - ) - describe('InstanceElement with ordered map', () => { - const gvsType = new ObjectType({ - elemID: new ElemID(SALESFORCE, GLOBAL_VALUE_SET), - annotations: { [METADATA_TYPE]: GLOBAL_VALUE_SET }, - }) - const gvs = new InstanceElement('MyGVS', gvsType, { - customValue: { - values: { - val1: 'value1', - val2: 'value2', - }, - }, - }) - - it('should return an error when the order field is missing', async () => { - const errors = await changeValidator([toChange({ after: gvs })]) - expect(errors).toHaveLength(1) - expect(errors[0]).toMatchObject({ - elemID: gvs.elemID, - severity: 'Error', - message: 'Missing field in ordered map', - detailedMessage: 'Missing order or values fields in field customValue', - }) - }) - - it('should return an error when a field ref is missing', async () => { - gvs.value.customValue.order = [ - new ReferenceExpression(gvs.elemID.createNestedID('customValue', 'values', 'val1')), - ] - const errors = await changeValidator([toChange({ after: gvs })]) - expect(errors).toHaveLength(1) - expect(errors[0]).toMatchObject({ - elemID: gvs.elemID, - severity: 'Error', - message: 'Missing reference in ordered map', - detailedMessage: 'Missing reference in field customValue.order: val2', - }) - }) - - it('should return an error when a ref is invalid', async () => { - gvs.value.customValue.order = [ - new ReferenceExpression(gvs.elemID.createNestedID('customValue', 'values', 'val1')), - new ReferenceExpression(gvs.elemID.createNestedID('customValue', 'values', 'val2')), - 'invalid', - ] - const errors = await changeValidator([toChange({ after: gvs })]) - expect(errors).toHaveLength(1) - expect(errors[0]).toMatchObject({ - elemID: gvs.elemID, - severity: 'Error', - message: 'Invalid reference in ordered map', - detailedMessage: - 'Invalid reference in field customValue.order: invalid. Only reference to internal value keys are allowed.', - }) - }) - - it('should return an error for duplicate field refs', async () => { - gvs.value.customValue.order = [ - new ReferenceExpression(gvs.elemID.createNestedID('customValue', 'values', 'val1')), - new ReferenceExpression(gvs.elemID.createNestedID('customValue', 'values', 'val1')), - new ReferenceExpression(gvs.elemID.createNestedID('customValue', 'values', 'val2')), - ] - const errors = await changeValidator([toChange({ after: gvs })]) - expect(errors).toHaveLength(1) - expect(errors[0]).toMatchObject({ - elemID: gvs.elemID, - severity: 'Error', - message: 'Duplicate reference in ordered map', - detailedMessage: 'Duplicate reference in field customValue.order: val1', - }) - }) - }) - - describe('ObjectType with ordered map', () => { - const account = new ObjectType({ - elemID: new ElemID(SALESFORCE, 'Account'), - fields: { - CustomerPriority__c: { - refType: Types.primitiveDataTypes.Picklist, - annotations: { - valueSet: { - values: { - High: 'High', - Low: 'Low', - }, - }, - }, - }, - }, - }) - - const fieldElemID = account.elemID.createNestedID('field', 'CustomerPriority__c') - - it('should return an error when the order field is missing', async () => { - const errors = await changeValidator([toChange({ after: account })]) - expect(errors).toHaveLength(1) - expect(errors[0]).toMatchObject({ - elemID: fieldElemID, - severity: 'Error', - message: 'Missing field in ordered map', - detailedMessage: 'Missing order or values fields in field valueSet', - }) - }) - - it('should return an error when a field ref is missing', async () => { - account.fields.CustomerPriority__c.annotations.valueSet.order = [ - new ReferenceExpression(fieldElemID.createNestedID('valueSet', 'values', 'High')), - ] - const errors = await changeValidator([toChange({ after: account })]) - expect(errors).toHaveLength(1) - expect(errors[0]).toMatchObject({ - elemID: fieldElemID, - severity: 'Error', - message: 'Missing reference in ordered map', - detailedMessage: 'Missing reference in field valueSet.order: Low', - }) - }) - - it('should return an error when a ref is invalid', async () => { - account.fields.CustomerPriority__c.annotations.valueSet.order = [ - new ReferenceExpression(fieldElemID.createNestedID('valueSet', 'values', 'High')), - new ReferenceExpression(fieldElemID.createNestedID('valueSet', 'values', 'Low')), - 'invalid', - ] - const errors = await changeValidator([toChange({ after: account })]) - expect(errors).toHaveLength(1) - expect(errors[0]).toMatchObject({ - elemID: fieldElemID, - severity: 'Error', - message: 'Invalid reference in ordered map', - detailedMessage: - 'Invalid reference in field valueSet.order: invalid. Only reference to internal value keys are allowed.', - }) - }) - - it('should return an error for duplicate field refs', async () => { - account.fields.CustomerPriority__c.annotations.valueSet.order = [ - new ReferenceExpression(fieldElemID.createNestedID('valueSet', 'values', 'High')), - new ReferenceExpression(fieldElemID.createNestedID('valueSet', 'values', 'High')), - new ReferenceExpression(fieldElemID.createNestedID('valueSet', 'values', 'Low')), - ] - const errors = await changeValidator([toChange({ after: account })]) - expect(errors).toHaveLength(1) - expect(errors[0]).toMatchObject({ - elemID: fieldElemID, - severity: 'Error', - message: 'Duplicate reference in ordered map', - detailedMessage: 'Duplicate reference in field valueSet.order: High', - }) - }) - }) -}) diff --git a/packages/salesforce-adapter/test/filters/convert_maps.test.ts b/packages/salesforce-adapter/test/filters/convert_maps.test.ts index f280a1ff246..b9cd63fe4a3 100644 --- a/packages/salesforce-adapter/test/filters/convert_maps.test.ts +++ b/packages/salesforce-adapter/test/filters/convert_maps.test.ts @@ -15,16 +15,12 @@ import { Change, toChange, isObjectType, - PrimitiveType, - TypeReference, } from '@salto-io/adapter-api' import filterCreator from '../../src/filters/convert_maps' -import { generateProfileType, generatePermissionSetType, defaultFilterContext, createCustomObjectType } from '../utils' -import { createInstanceElement, Types } from '../../src/transformers/transformer' +import { generateProfileType, generatePermissionSetType, defaultFilterContext } from '../utils' +import { createInstanceElement } from '../../src/transformers/transformer' import { mockTypes } from '../mock_elements' import { FilterWith } from './mocks' -import { buildFetchProfile } from '../../src/fetch_profile/fetch_profile' -import { FIELD_ANNOTATIONS } from '../../src/constants' type layoutAssignmentType = { layout: string; recordType?: string } @@ -596,117 +592,4 @@ describe('Convert maps filter', () => { }) }) }) - - describe('Maintain order', () => { - const gvsType = mockTypes.GlobalValueSet - const gvs = new InstanceElement('MyGVS', gvsType, { - customValue: [ - { fullName: 'val1', default: true, label: 'value1' }, - { fullName: 'val2', default: false, label: 'value2' }, - ], - }) - let elements: Element[] - type FilterType = FilterWith<'onFetch'> - let filter: FilterType - beforeAll(async () => { - elements = [gvs, gvsType] - filter = filterCreator({ - config: { - ...defaultFilterContext, - fetchProfile: buildFetchProfile({ fetchParams: { optionalFeatures: { picklistsAsMaps: true } } }), - }, - }) as FilterType - await filter.onFetch(elements) - }) - - it('should convert field type to ordered map', async () => { - const fieldType = await gvsType.fields.customValue.getType() - expect(fieldType.elemID.typeName).toEqual('OrderedMap') - }) - - it('should convert instance value to map ', () => { - expect(gvs.value.customValue.values).toBeDefined() - expect(gvs.value.customValue.values).toEqual({ - val1: { fullName: 'val1', default: true, label: 'value1' }, - val2: { fullName: 'val2', default: false, label: 'value2' }, - }) - }) - }) - - describe('Convert CustomObject field annotations by type', () => { - let picklistType: PrimitiveType - let multiselectPicklistType: PrimitiveType - let myCustomObj: ObjectType - let elements: Element[] - type FilterType = FilterWith<'onFetch'> - let filter: FilterType - beforeEach(async () => { - // Clone the types to avoid changing the original types and affecting other tests. - picklistType = Types.primitiveDataTypes.Picklist.clone() - multiselectPicklistType = Types.primitiveDataTypes.MultiselectPicklist.clone() - myCustomObj = createCustomObjectType('MyCustomObj', { - fields: { - myPicklist: { - refType: picklistType, - annotations: { - [FIELD_ANNOTATIONS.VALUE_SET]: [ - { fullName: 'val1', default: true, label: 'value1' }, - { fullName: 'val2', default: false, label: 'value2' }, - ], - }, - }, - myMultiselectPicklist: { - refType: multiselectPicklistType, - annotations: { - [FIELD_ANNOTATIONS.VALUE_SET]: [ - { fullName: 'val1', default: true, label: 'value1' }, - { fullName: 'val2', default: false, label: 'value2' }, - ], - }, - }, - }, - }) - - elements = [myCustomObj] - filter = filterCreator({ - config: { - ...defaultFilterContext, - fetchProfile: buildFetchProfile({ fetchParams: { optionalFeatures: { picklistsAsMaps: true } } }), - }, - }) as FilterType - await filter.onFetch(elements) - }) - - it('should convert Picklist valueSet type to ordered map', async () => { - expect(myCustomObj.fields.myPicklist.getTypeSync()).toEqual(picklistType) - const valueSetType = picklistType.annotationRefTypes.valueSet as TypeReference - expect(valueSetType.elemID.typeName).toEqual('OrderedMap') - expect(valueSetType.type?.fields.values.refType.elemID.typeName).toEqual('Map') - expect(valueSetType.type?.fields.order.refType.elemID.typeName).toEqual('List') - }) - - it('should convert MultiselectPicklist valueSet type to ordered map', async () => { - expect(myCustomObj.fields.myMultiselectPicklist.getTypeSync()).toEqual(multiselectPicklistType) - const valueSetType = multiselectPicklistType.annotationRefTypes.valueSet as TypeReference - expect(valueSetType.elemID.typeName).toEqual('OrderedMap') - expect(valueSetType.type?.fields.values.refType.elemID.typeName).toEqual('Map') - expect(valueSetType.type?.fields.order.refType.elemID.typeName).toEqual('List') - }) - - it('should convert annotation value to map (Picklist)', () => { - expect(myCustomObj.fields.myPicklist.annotations.valueSet.values).toBeDefined() - expect(myCustomObj.fields.myPicklist.annotations.valueSet.values).toEqual({ - val1: { fullName: 'val1', default: true, label: 'value1' }, - val2: { fullName: 'val2', default: false, label: 'value2' }, - }) - }) - - it('should convert annotation value to map (MultiselectPicklist)', () => { - expect(myCustomObj.fields.myMultiselectPicklist.annotations.valueSet.values).toBeDefined() - expect(myCustomObj.fields.myMultiselectPicklist.annotations.valueSet.values).toEqual({ - val1: { fullName: 'val1', default: true, label: 'value1' }, - val2: { fullName: 'val2', default: false, label: 'value2' }, - }) - }) - }) })