diff --git a/packages/adapter-api/src/change.ts b/packages/adapter-api/src/change.ts index 76831595aff..dfa9db7cdc5 100644 --- a/packages/adapter-api/src/change.ts +++ b/packages/adapter-api/src/change.ts @@ -18,7 +18,7 @@ import { } from '@salto-io/dag' import { values as lowerDashValues } from '@salto-io/lowerdash' import { - ObjectType, InstanceElement, Field, isInstanceElement, isObjectType, isField, TypeElement, + ObjectType, InstanceElement, Field, isInstanceElement, isObjectType, isField, TopLevelElement, } from './elements' import { ElemID } from './element_id' import { Values, Value } from './values' @@ -27,7 +27,7 @@ const { isDefined } = lowerDashValues export { ActionName } -export type ChangeDataType = TypeElement | InstanceElement | Field +export type ChangeDataType = TopLevelElement | Field export type AdditionChange = AdditionDiff export type ModificationChange = ModificationDiff export type RemovalChange = RemovalDiff diff --git a/packages/adapter-api/src/elements.ts b/packages/adapter-api/src/elements.ts index 1f15842d2c7..aace6bfa198 100644 --- a/packages/adapter-api/src/elements.ts +++ b/packages/adapter-api/src/elements.ts @@ -148,6 +148,7 @@ export enum PrimitiveTypes { export type ContainerType = ListType | MapType export type TypeElement = PrimitiveType | ObjectType | ContainerType +export type TopLevelElement = TypeElement | InstanceElement export type TypeMap = Record type TypeOrRef = T | TypeReference export type TypeRefMap = Record diff --git a/packages/adapter-components/src/add_alias.ts b/packages/adapter-components/src/add_alias.ts index 620501d703a..ebfaf023ba0 100644 --- a/packages/adapter-components/src/add_alias.ts +++ b/packages/adapter-components/src/add_alias.ts @@ -14,7 +14,7 @@ * limitations under the License. */ import _ from 'lodash' -import { InstanceElement, CORE_ANNOTATIONS, ElemID, INSTANCE_ANNOTATIONS, isReferenceExpression, ObjectType, isInstanceElement, Value } from '@salto-io/adapter-api' +import { CORE_ANNOTATIONS, ElemID, INSTANCE_ANNOTATIONS, isReferenceExpression, isInstanceElement, Value, TopLevelElement } from '@salto-io/adapter-api' import { logger } from '@salto-io/logging' const log = logger(module) @@ -33,12 +33,11 @@ export type AliasData = { separator?: string } -type SupportedElement = ObjectType | InstanceElement const isInstanceAnnotation = (field: string): boolean => Object.values(INSTANCE_ANNOTATIONS).includes(field.split(ElemID.NAMESPACE_SEPARATOR)[0]) -const isValidAlias = (aliasParts: (string | undefined)[], element: SupportedElement): boolean => +const isValidAlias = (aliasParts: (string | undefined)[], element: TopLevelElement): boolean => aliasParts.every((val, index) => { if (val === undefined) { log.debug(`for element ${element.elemID.getFullName()}, component number ${index} in the alias map resulted in undefined`) @@ -47,16 +46,16 @@ const isValidAlias = (aliasParts: (string | undefined)[], element: SupportedElem return true }) -const getFieldValue = (element: SupportedElement, fieldName: string): Value => ( +const getFieldValue = (element: TopLevelElement, fieldName: string): Value => ( !isInstanceElement(element) || isInstanceAnnotation(fieldName) ? _.get(element.annotations, fieldName) : _.get(element.value, fieldName) ) const getAliasFromField = ({ element, component, elementsById }:{ - element: SupportedElement + element: TopLevelElement component: AliasComponent - elementsById: Record + elementsById: Record }): string | undefined => { const { fieldName, referenceFieldName } = component @@ -83,8 +82,8 @@ const isConstantComponent = ( ): component is ConstantComponent => 'constant' in component const calculateAlias = ({ element, elementsById, aliasData }: { - element: SupportedElement - elementsById: Record + element: TopLevelElement + elementsById: Record aliasData: AliasData }): string | undefined => { const { aliasComponents, separator = ' ' } = aliasData @@ -105,7 +104,7 @@ export const addAliasToElements = ({ aliasMap, secondIterationGroupNames = [], }: { - elementsMap: Record + elementsMap: Record aliasMap: Record secondIterationGroupNames?: string[] }): void => { diff --git a/packages/core/src/api.ts b/packages/core/src/api.ts index 7b7ad27dbed..f94c40ec4ca 100644 --- a/packages/core/src/api.ts +++ b/packages/core/src/api.ts @@ -36,12 +36,15 @@ import { isAdditionOrModificationChange, ChangeError, AdapterOperations, + TopLevelElement, + isAdditionChange, + isRemovalChange, } from '@salto-io/adapter-api' import { EventEmitter } from 'pietile-eventemitter' import { logger } from '@salto-io/logging' import _ from 'lodash' import { promises, collections, values, objects } from '@salto-io/lowerdash' -import { Workspace, ElementSelector, elementSource, expressions, merger, selectElementIdsByTraversal, isTopLevelSelector } from '@salto-io/workspace' +import { Workspace, ElementSelector, elementSource, expressions, merger, selectElementIdsByTraversal, isTopLevelSelector, pathIndex as pathIndexModule } from '@salto-io/workspace' import { EOL } from 'os' import { buildElementsSourceFromElements, @@ -358,6 +361,42 @@ export const fetchFromWorkspace: FetchFromWorkspaceFunc = async ({ } } +type CalculatePatchFromChangesArgs = { + workspace: Workspace + changes: Change[] + pathIndex: pathIndexModule.PathIndex +} + +export const calculatePatchFromChanges = async ( + { + workspace, + changes, + pathIndex, + }: CalculatePatchFromChangesArgs, +): Promise => { + const [additions, removalAndModifications] = _.partition(changes, isAdditionChange) + const [removals, modifications] = _.partition(removalAndModifications, isRemovalChange) + const beforeElements = [...removals, ...modifications].map(change => change.data.before) + const afterElements = [...additions, ...modifications].map(change => change.data.after) + const unmergedAfterElements = await awu(afterElements) + .flatMap(element => pathIndexModule.splitElementByPath(element, pathIndex)) + .toArray() + const accounts = [...beforeElements, ...afterElements].map(element => element.elemID.adapter) + const partialFetchData = new Map(accounts.map(account => [account, { deletedElements: new Set() }])) + removals.map(change => getChangeData(change).elemID).forEach(id => { + partialFetchData.get(id.adapter)?.deletedElements.add(id.getFullName()) + }) + const result = await calcFetchChanges( + unmergedAfterElements, + afterElements, + elementSource.createInMemoryElementSource(beforeElements), + await workspace.elements(), + partialFetchData, + new Set(accounts), + ) + return result.changes +} + type CalculatePatchArgs = { workspace: Workspace fromDir: string diff --git a/packages/core/test/api.test.ts b/packages/core/test/api.test.ts index bea1e8b4981..a4cf25f208e 100644 --- a/packages/core/test/api.test.ts +++ b/packages/core/test/api.test.ts @@ -14,7 +14,7 @@ * limitations under the License. */ import _ from 'lodash' -import { AdapterOperations, BuiltinTypes, CORE_ANNOTATIONS, Element, ElemID, InstanceElement, ObjectType, PrimitiveType, PrimitiveTypes, Adapter, isObjectType, isEqualElements, isAdditionChange, ChangeDataType, AdditionChange, isInstanceElement, isModificationChange, DetailedChange, ReferenceExpression, Field, getChangeData, toChange, SeverityLevel, GetAdditionalReferencesFunc, Change, FixElementsFunc } from '@salto-io/adapter-api' +import { AdapterOperations, BuiltinTypes, CORE_ANNOTATIONS, Element, ElemID, InstanceElement, ObjectType, PrimitiveType, PrimitiveTypes, Adapter, isObjectType, isEqualElements, isAdditionChange, ChangeDataType, AdditionChange, isInstanceElement, isModificationChange, DetailedChange, ReferenceExpression, Field, getChangeData, toChange, SeverityLevel, GetAdditionalReferencesFunc, Change, FixElementsFunc, isAdditionOrModificationChange } from '@salto-io/adapter-api' import * as workspace from '@salto-io/workspace' import { collections } from '@salto-io/lowerdash' import { mockFunction, MockInterface } from '@salto-io/test-utils' @@ -862,6 +862,172 @@ describe('api.ts', () => { }) }) + describe('calculatePatchFromChanges', () => { + const type = new ObjectType({ elemID: new ElemID('salto', 'type') }) + const modifiedInstance = toChange({ + before: new InstanceElement('modified', type, { name: 'before', label: 'before' }), + after: new InstanceElement('modified', type, { name: 'after', label: 'after' }), + }) + const nonExistModifiedInstance = toChange({ + before: new InstanceElement('nonExistModified', type, { name: 'before' }), + after: new InstanceElement('nonExistModified', type, { name: 'after' }), + }) + const addedInstance = toChange({ + after: new InstanceElement('added', type, { name: 'after' }), + }) + const existingAddedInstance = toChange({ + after: new InstanceElement('existingAdded', type, { name: 'after' }), + }) + const deletedInstance = toChange({ + before: new InstanceElement('deleted', type, { name: 'before' }), + }) + const nonExistDeletedInstance = toChange({ + before: new InstanceElement('nonExistDeleted', type, { name: 'before' }), + }) + const changes = [ + modifiedInstance, + nonExistModifiedInstance, + addedInstance, + existingAddedInstance, + deletedInstance, + nonExistDeletedInstance, + ] + const pathIndex = new workspace.remoteMap.InMemoryRemoteMap( + changes + .filter(isAdditionOrModificationChange) + .map(getChangeData) + .map(element => ({ key: element.elemID.getFullName(), value: [['test']] })) + ) + + let patchChanges: fetch.FetchChange[] + beforeEach(async () => { + const elements = [ + new InstanceElement('modified', type, { name: 'other', label: 'before' }), + new InstanceElement('existingAdded', type, { name: 'other' }), + new InstanceElement('deleted', type, { name: 'other' }), + ] + const ws = mockWorkspace({ + elements, + stateElements: elements, + name: 'workspace', + accounts: ['salto'], + accountToServiceName: { salto: 'salto' }, + }) + + patchChanges = await api.calculatePatchFromChanges({ + workspace: ws, + changes, + pathIndex, + }) + }) + + it('should calculate changes', () => { + expect(patchChanges).toHaveLength(6) + }) + it('should calculate instance modification (conflict)', () => { + expect(patchChanges).toEqual(expect.arrayContaining([ + expect.objectContaining({ + change: expect.objectContaining({ + id: new ElemID('salto', 'type', 'instance', 'modified', 'name'), + data: { before: 'other', after: 'after' }, + }), + serviceChanges: [expect.objectContaining({ + id: new ElemID('salto', 'type', 'instance', 'modified', 'name'), + data: { before: 'before', after: 'after' }, + })], + pendingChanges: [expect.objectContaining({ + id: new ElemID('salto', 'type', 'instance', 'modified', 'name'), + data: { before: 'before', after: 'other' }, + })], + }), + ])) + }) + it('should calculate instance modification', () => { + expect(patchChanges).toEqual(expect.arrayContaining([ + expect.objectContaining({ + change: expect.objectContaining({ + id: new ElemID('salto', 'type', 'instance', 'modified', 'label'), + data: { before: 'before', after: 'after' }, + }), + serviceChanges: [expect.objectContaining({ + id: new ElemID('salto', 'type', 'instance', 'modified', 'label'), + data: { before: 'before', after: 'after' }, + })], + pendingChanges: [], + }), + ])) + }) + it('should calculate non existed instance modification (conflict)', () => { + expect(patchChanges).toEqual(expect.arrayContaining([ + expect.objectContaining({ + change: expect.objectContaining({ + id: new ElemID('salto', 'type', 'instance', 'nonExistModified'), + data: { after: expect.objectContaining({ path: ['test'] }) }, + }), + serviceChanges: [expect.objectContaining({ + id: new ElemID('salto', 'type', 'instance', 'nonExistModified', 'name'), + data: { before: 'before', after: 'after' }, + })], + pendingChanges: [expect.objectContaining({ + id: new ElemID('salto', 'type', 'instance', 'nonExistModified'), + data: { before: expect.any(InstanceElement) }, + })], + }), + ])) + }) + it('should calculate existing instance addition (conflict)', () => { + expect(patchChanges).toEqual(expect.arrayContaining([ + expect.objectContaining({ + change: expect.objectContaining({ + id: new ElemID('salto', 'type', 'instance', 'existingAdded', 'name'), + data: { before: 'other', after: 'after' }, + }), + serviceChanges: [expect.objectContaining({ + id: new ElemID('salto', 'type', 'instance', 'existingAdded'), + data: { after: expect.any(InstanceElement) }, + })], + pendingChanges: [expect.objectContaining({ + id: new ElemID('salto', 'type', 'instance', 'existingAdded'), + data: { after: expect.any(InstanceElement) }, + })], + }), + ])) + }) + it('should calculate instance addition', () => { + expect(patchChanges).toEqual(expect.arrayContaining([ + expect.objectContaining({ + change: expect.objectContaining({ + id: new ElemID('salto', 'type', 'instance', 'added'), + data: { after: expect.objectContaining({ path: ['test'] }) }, + }), + serviceChanges: [expect.objectContaining({ + id: new ElemID('salto', 'type', 'instance', 'added'), + data: { after: expect.any(InstanceElement) }, + })], + pendingChanges: [], + }), + ])) + }) + it('should calculate instance deletion (conflict)', () => { + expect(patchChanges).toEqual(expect.arrayContaining([ + expect.objectContaining({ + change: expect.objectContaining({ + id: new ElemID('salto', 'type', 'instance', 'deleted'), + data: { before: expect.any(InstanceElement) }, + }), + serviceChanges: [expect.objectContaining({ + id: new ElemID('salto', 'type', 'instance', 'deleted'), + data: { before: expect.any(InstanceElement) }, + })], + pendingChanges: [expect.objectContaining({ + id: new ElemID('salto', 'type', 'instance', 'deleted', 'name'), + data: { before: 'before', after: 'other' }, + })], + }), + ])) + }) + }) + describe('calculatePatch', () => { const type = new ObjectType({ elemID: new ElemID('salesforce', 'type'), diff --git a/packages/netsuite-adapter/src/client/types.ts b/packages/netsuite-adapter/src/client/types.ts index b450b76bbad..6ea651810b6 100644 --- a/packages/netsuite-adapter/src/client/types.ts +++ b/packages/netsuite-adapter/src/client/types.ts @@ -13,7 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { Change, ChangeData, ElemID, getChangeData, InstanceElement, isInstanceChange, isObjectType, isObjectTypeChange, ObjectType, Values } from '@salto-io/adapter-api' +import { Change, ChangeData, ElemID, getChangeData, InstanceElement, isInstanceChange, isObjectType, isObjectTypeChange, ObjectType, TopLevelElement, Values } from '@salto-io/adapter-api' import { toCustomRecordTypeInstance } from '../custom_records/custom_record_type' import { NetsuiteFilePathsQueryParams, NetsuiteTypesQueryParams, ObjectID } from '../query' @@ -87,7 +87,7 @@ export type ImportObjectsResult = { } export type DataElementsResult = { - elements: (ObjectType | InstanceElement)[] + elements: TopLevelElement[] requestedTypes: string[] largeTypesError: string[] } diff --git a/packages/netsuite-adapter/src/reference_dependencies.ts b/packages/netsuite-adapter/src/reference_dependencies.ts index 12f3aa77d0d..7b6c56ed923 100644 --- a/packages/netsuite-adapter/src/reference_dependencies.ts +++ b/packages/netsuite-adapter/src/reference_dependencies.ts @@ -17,7 +17,7 @@ import _ from 'lodash' import { logger } from '@salto-io/logging' import { isInstanceElement, isPrimitiveType, ElemID, getFieldType, - isReferenceExpression, Value, isServiceId, isObjectType, ChangeDataType, ObjectType, InstanceElement, + isReferenceExpression, Value, isServiceId, isObjectType, ChangeDataType, TopLevelElement, } from '@salto-io/adapter-api' import { transformElement, TransformFunc } from '@salto-io/adapter-utils' import { values as lowerDashValues, collections } from '@salto-io/lowerdash' @@ -30,7 +30,6 @@ const { awu } = collections.asynciterable const { isDefined } = lowerDashValues const log = logger(module) -type TopLevelElement = ObjectType | InstanceElement const isTopLevelElement = (value: unknown): value is TopLevelElement => isObjectType(value) || isInstanceElement(value) diff --git a/packages/netsuite-adapter/src/suiteapp_config_elements.ts b/packages/netsuite-adapter/src/suiteapp_config_elements.ts index dc2a24f606f..4503e397719 100644 --- a/packages/netsuite-adapter/src/suiteapp_config_elements.ts +++ b/packages/netsuite-adapter/src/suiteapp_config_elements.ts @@ -14,7 +14,7 @@ * limitations under the License. */ import _ from 'lodash' -import { BuiltinTypes, ElemID, getChangeData, InstanceElement, isInstanceElement, ModificationChange, ObjectType } from '@salto-io/adapter-api' +import { BuiltinTypes, ElemID, getChangeData, InstanceElement, isInstanceElement, ModificationChange, ObjectType, TopLevelElement } from '@salto-io/adapter-api' import { NETSUITE, SELECT_OPTION, SETTINGS_PATH, TYPES_PATH } from './constants' import { SUITEAPP_CONFIG_TYPES_TO_TYPE_NAMES, DeployResult } from './types' import { NetsuiteQuery } from './query' @@ -32,9 +32,7 @@ export const getConfigTypes = (): ObjectType[] => ([new ObjectType({ export const toConfigElements = ( configRecords: ConfigRecord[], fetchQuery: NetsuiteQuery -): ( - ObjectType | InstanceElement -)[] => { +): TopLevelElement[] => { const elements = configRecords .flatMap(configRecord => { const typeName = SUITEAPP_CONFIG_TYPES_TO_TYPE_NAMES[configRecord.configType] diff --git a/packages/netsuite-adapter/src/transformer.ts b/packages/netsuite-adapter/src/transformer.ts index 25913cc49fd..6738f495d3d 100644 --- a/packages/netsuite-adapter/src/transformer.ts +++ b/packages/netsuite-adapter/src/transformer.ts @@ -19,11 +19,11 @@ import { OBJECT_SERVICE_ID, OBJECT_NAME, toServiceIdsString, ServiceIds, isInstanceElement, ElemIDType, - TypeElement, BuiltinTypes, ReadOnlyElementsSource, CORE_ANNOTATIONS, TypeReference, + TopLevelElement, } from '@salto-io/adapter-api' import { MapKeyFunc, mapKeysRecursive, TransformFunc, transformValues, GetLookupNameFunc, naclCase, pathNaclCase } from '@salto-io/adapter-utils' import { collections } from '@salto-io/lowerdash' @@ -201,7 +201,7 @@ export const createElements = async ( customizationInfos: CustomizationInfo[], elementsSource: ReadOnlyElementsSource, getElemIdFunc?: ElemIdGetter, -): Promise> => { +): Promise> => { const { standardTypes, additionalTypes, innerAdditionalTypes } = getMetadataTypes() getTopLevelStandardTypes(standardTypes).concat(Object.values(additionalTypes))