SALTO-4804 Add calculatePatchFromChanges API (#4906)

packages/adapter-api/src/change.ts

@@ -18,7 +18,7 @@ import {
 } from '@salto-io/dag'
 import { values as lowerDashValues } from '@salto-io/lowerdash'
 import {
-  ObjectType, InstanceElement, Field, isInstanceElement, isObjectType, isField, TypeElement,
+  ObjectType, InstanceElement, Field, isInstanceElement, isObjectType, isField, TopLevelElement,
 } from './elements'
 import { ElemID } from './element_id'
 import { Values, Value } from './values'
@@ -27,7 +27,7 @@ const { isDefined } = lowerDashValues
 
 export { ActionName }
 
-export type ChangeDataType = TypeElement | InstanceElement | Field
+export type ChangeDataType = TopLevelElement | Field
 export type AdditionChange<T> = AdditionDiff<T>
 export type ModificationChange<T> = ModificationDiff<T>
 export type RemovalChange<T> = RemovalDiff<T>

packages/adapter-api/src/elements.ts

@@ -148,6 +148,7 @@ export enum PrimitiveTypes {
 
 export type ContainerType = ListType | MapType
 export type TypeElement = PrimitiveType | ObjectType | ContainerType
+export type TopLevelElement = TypeElement | InstanceElement
 export type TypeMap = Record<string, TypeElement>
 type TypeOrRef<T extends TypeElement = TypeElement> = T | TypeReference
 export type TypeRefMap = Record<string, TypeOrRef>

packages/adapter-components/src/add_alias.ts

@@ -14,7 +14,7 @@
 * limitations under the License.
 */
 import _ from 'lodash'
-import { InstanceElement, CORE_ANNOTATIONS, ElemID, INSTANCE_ANNOTATIONS, isReferenceExpression, ObjectType, isInstanceElement, Value } from '@salto-io/adapter-api'
+import { CORE_ANNOTATIONS, ElemID, INSTANCE_ANNOTATIONS, isReferenceExpression, isInstanceElement, Value, TopLevelElement } from '@salto-io/adapter-api'
 import { logger } from '@salto-io/logging'
 
 const log = logger(module)
@@ -33,12 +33,11 @@ export type AliasData<T extends Component[] = Component[]> = {
   separator?: string
 }
 
-type SupportedElement = ObjectType | InstanceElement
 
 const isInstanceAnnotation = (field: string): boolean =>
   Object.values(INSTANCE_ANNOTATIONS).includes(field.split(ElemID.NAMESPACE_SEPARATOR)[0])
 
-const isValidAlias = (aliasParts: (string | undefined)[], element: SupportedElement): boolean =>
+const isValidAlias = (aliasParts: (string | undefined)[], element: TopLevelElement): boolean =>
   aliasParts.every((val, index) => {
     if (val === undefined) {
       log.debug(`for element ${element.elemID.getFullName()}, component number ${index} in the alias map resulted in undefined`)
@@ -47,16 +46,16 @@ const isValidAlias = (aliasParts: (string | undefined)[], element: SupportedElem
     return true
   })
 
-const getFieldValue = (element: SupportedElement, fieldName: string): Value => (
+const getFieldValue = (element: TopLevelElement, fieldName: string): Value => (
   !isInstanceElement(element) || isInstanceAnnotation(fieldName)
     ? _.get(element.annotations, fieldName)
     : _.get(element.value, fieldName)
 )
 
 const getAliasFromField = ({ element, component, elementsById }:{
-  element: SupportedElement
+  element: TopLevelElement
   component: AliasComponent
-  elementsById: Record<string, SupportedElement>
+  elementsById: Record<string, TopLevelElement>
 }): string | undefined => {
   const { fieldName, referenceFieldName } = component
 
@@ -83,8 +82,8 @@ const isConstantComponent = (
 ): component is ConstantComponent => 'constant' in component
 
 const calculateAlias = ({ element, elementsById, aliasData }: {
-  element: SupportedElement
-  elementsById: Record<string, SupportedElement>
+  element: TopLevelElement
+  elementsById: Record<string, TopLevelElement>
   aliasData: AliasData
 }): string | undefined => {
   const { aliasComponents, separator = ' ' } = aliasData
@@ -105,7 +104,7 @@ export const addAliasToElements = ({
   aliasMap,
   secondIterationGroupNames = [],
 }: {
-  elementsMap: Record<string, SupportedElement[]>
+  elementsMap: Record<string, TopLevelElement[]>
   aliasMap: Record<string, AliasData>
   secondIterationGroupNames?: string[]
 }): void => {

packages/core/src/api.ts

@@ -36,12 +36,15 @@ import {
   isAdditionOrModificationChange,
   ChangeError,
   AdapterOperations,
+  TopLevelElement,
+  isAdditionChange,
+  isRemovalChange,
 } from '@salto-io/adapter-api'
 import { EventEmitter } from 'pietile-eventemitter'
 import { logger } from '@salto-io/logging'
 import _ from 'lodash'
 import { promises, collections, values, objects } from '@salto-io/lowerdash'
-import { Workspace, ElementSelector, elementSource, expressions, merger, selectElementIdsByTraversal, isTopLevelSelector } from '@salto-io/workspace'
+import { Workspace, ElementSelector, elementSource, expressions, merger, selectElementIdsByTraversal, isTopLevelSelector, pathIndex as pathIndexModule } from '@salto-io/workspace'
 import { EOL } from 'os'
 import {
   buildElementsSourceFromElements,
@@ -358,6 +361,42 @@ export const fetchFromWorkspace: FetchFromWorkspaceFunc = async ({
   }
 }
 
+type CalculatePatchFromChangesArgs = {
+  workspace: Workspace
+  changes: Change<TopLevelElement>[]
+  pathIndex: pathIndexModule.PathIndex
+}
+
+export const calculatePatchFromChanges = async (
+  {
+    workspace,
+    changes,
+    pathIndex,
+  }: CalculatePatchFromChangesArgs,
+): Promise<FetchChange[]> => {
+  const [additions, removalAndModifications] = _.partition(changes, isAdditionChange)
+  const [removals, modifications] = _.partition(removalAndModifications, isRemovalChange)
+  const beforeElements = [...removals, ...modifications].map(change => change.data.before)
+  const afterElements = [...additions, ...modifications].map(change => change.data.after)
+  const unmergedAfterElements = await awu(afterElements)
+    .flatMap(element => pathIndexModule.splitElementByPath(element, pathIndex))
+    .toArray()
+  const accounts = [...beforeElements, ...afterElements].map(element => element.elemID.adapter)
+  const partialFetchData = new Map(accounts.map(account => [account, { deletedElements: new Set<string>() }]))
+  removals.map(change => getChangeData(change).elemID).forEach(id => {
+    partialFetchData.get(id.adapter)?.deletedElements.add(id.getFullName())
+  })
+  const result = await calcFetchChanges(
+    unmergedAfterElements,
+    afterElements,
+    elementSource.createInMemoryElementSource(beforeElements),
+    await workspace.elements(),
+    partialFetchData,
+    new Set(accounts),
+  )
+  return result.changes
+}
+
 type CalculatePatchArgs = {
   workspace: Workspace
   fromDir: string

packages/core/test/api.test.ts

@@ -14,7 +14,7 @@
 * limitations under the License.
 */
 import _ from 'lodash'
-import { AdapterOperations, BuiltinTypes, CORE_ANNOTATIONS, Element, ElemID, InstanceElement, ObjectType, PrimitiveType, PrimitiveTypes, Adapter, isObjectType, isEqualElements, isAdditionChange, ChangeDataType, AdditionChange, isInstanceElement, isModificationChange, DetailedChange, ReferenceExpression, Field, getChangeData, toChange, SeverityLevel, GetAdditionalReferencesFunc, Change, FixElementsFunc } from '@salto-io/adapter-api'
+import { AdapterOperations, BuiltinTypes, CORE_ANNOTATIONS, Element, ElemID, InstanceElement, ObjectType, PrimitiveType, PrimitiveTypes, Adapter, isObjectType, isEqualElements, isAdditionChange, ChangeDataType, AdditionChange, isInstanceElement, isModificationChange, DetailedChange, ReferenceExpression, Field, getChangeData, toChange, SeverityLevel, GetAdditionalReferencesFunc, Change, FixElementsFunc, isAdditionOrModificationChange } from '@salto-io/adapter-api'
 import * as workspace from '@salto-io/workspace'
 import { collections } from '@salto-io/lowerdash'
 import { mockFunction, MockInterface } from '@salto-io/test-utils'
@@ -862,6 +862,172 @@ describe('api.ts', () => {
     })
   })
 
+  describe('calculatePatchFromChanges', () => {
+    const type = new ObjectType({ elemID: new ElemID('salto', 'type') })
+    const modifiedInstance = toChange({
+      before: new InstanceElement('modified', type, { name: 'before', label: 'before' }),
+      after: new InstanceElement('modified', type, { name: 'after', label: 'after' }),
+    })
+    const nonExistModifiedInstance = toChange({
+      before: new InstanceElement('nonExistModified', type, { name: 'before' }),
+      after: new InstanceElement('nonExistModified', type, { name: 'after' }),
+    })
+    const addedInstance = toChange({
+      after: new InstanceElement('added', type, { name: 'after' }),
+    })
+    const existingAddedInstance = toChange({
+      after: new InstanceElement('existingAdded', type, { name: 'after' }),
+    })
+    const deletedInstance = toChange({
+      before: new InstanceElement('deleted', type, { name: 'before' }),
+    })
+    const nonExistDeletedInstance = toChange({
+      before: new InstanceElement('nonExistDeleted', type, { name: 'before' }),
+    })
+    const changes = [
+      modifiedInstance,
+      nonExistModifiedInstance,
+      addedInstance,
+      existingAddedInstance,
+      deletedInstance,
+      nonExistDeletedInstance,
+    ]
+    const pathIndex = new workspace.remoteMap.InMemoryRemoteMap<workspace.pathIndex.Path[]>(
+      changes
+        .filter(isAdditionOrModificationChange)
+        .map(getChangeData)
+        .map(element => ({ key: element.elemID.getFullName(), value: [['test']] }))
+    )
+
+    let patchChanges: fetch.FetchChange[]
+    beforeEach(async () => {
+      const elements = [
+        new InstanceElement('modified', type, { name: 'other', label: 'before' }),
+        new InstanceElement('existingAdded', type, { name: 'other' }),
+        new InstanceElement('deleted', type, { name: 'other' }),
+      ]
+      const ws = mockWorkspace({
+        elements,
+        stateElements: elements,
+        name: 'workspace',
+        accounts: ['salto'],
+        accountToServiceName: { salto: 'salto' },
+      })
+
+      patchChanges = await api.calculatePatchFromChanges({
+        workspace: ws,
+        changes,
+        pathIndex,
+      })
+    })
+
+    it('should calculate changes', () => {
+      expect(patchChanges).toHaveLength(6)
+    })
+    it('should calculate instance modification (conflict)', () => {
+      expect(patchChanges).toEqual(expect.arrayContaining([
+        expect.objectContaining({
+          change: expect.objectContaining({
+            id: new ElemID('salto', 'type', 'instance', 'modified', 'name'),
+            data: { before: 'other', after: 'after' },
+          }),
+          serviceChanges: [expect.objectContaining({
+            id: new ElemID('salto', 'type', 'instance', 'modified', 'name'),
+            data: { before: 'before', after: 'after' },
+          })],
+          pendingChanges: [expect.objectContaining({
+            id: new ElemID('salto', 'type', 'instance', 'modified', 'name'),
+            data: { before: 'before', after: 'other' },
+          })],
+        }),
+      ]))
+    })
+    it('should calculate instance modification', () => {
+      expect(patchChanges).toEqual(expect.arrayContaining([
+        expect.objectContaining({
+          change: expect.objectContaining({
+            id: new ElemID('salto', 'type', 'instance', 'modified', 'label'),
+            data: { before: 'before', after: 'after' },
+          }),
+          serviceChanges: [expect.objectContaining({
+            id: new ElemID('salto', 'type', 'instance', 'modified', 'label'),
+            data: { before: 'before', after: 'after' },
+          })],
+          pendingChanges: [],
+        }),
+      ]))
+    })
+    it('should calculate non existed instance modification (conflict)', () => {
+      expect(patchChanges).toEqual(expect.arrayContaining([
+        expect.objectContaining({
+          change: expect.objectContaining({
+            id: new ElemID('salto', 'type', 'instance', 'nonExistModified'),
+            data: { after: expect.objectContaining({ path: ['test'] }) },
+          }),
+          serviceChanges: [expect.objectContaining({
+            id: new ElemID('salto', 'type', 'instance', 'nonExistModified', 'name'),
+            data: { before: 'before', after: 'after' },
+          })],
+          pendingChanges: [expect.objectContaining({
+            id: new ElemID('salto', 'type', 'instance', 'nonExistModified'),
+            data: { before: expect.any(InstanceElement) },
+          })],
+        }),
+      ]))
+    })
+    it('should calculate existing instance addition (conflict)', () => {
+      expect(patchChanges).toEqual(expect.arrayContaining([
+        expect.objectContaining({
+          change: expect.objectContaining({
+            id: new ElemID('salto', 'type', 'instance', 'existingAdded', 'name'),
+            data: { before: 'other', after: 'after' },
+          }),
+          serviceChanges: [expect.objectContaining({
+            id: new ElemID('salto', 'type', 'instance', 'existingAdded'),
+            data: { after: expect.any(InstanceElement) },
+          })],
+          pendingChanges: [expect.objectContaining({
+            id: new ElemID('salto', 'type', 'instance', 'existingAdded'),
+            data: { after: expect.any(InstanceElement) },
+          })],
+        }),
+      ]))
+    })
+    it('should calculate instance addition', () => {
+      expect(patchChanges).toEqual(expect.arrayContaining([
+        expect.objectContaining({
+          change: expect.objectContaining({
+            id: new ElemID('salto', 'type', 'instance', 'added'),
+            data: { after: expect.objectContaining({ path: ['test'] }) },
+          }),
+          serviceChanges: [expect.objectContaining({
+            id: new ElemID('salto', 'type', 'instance', 'added'),
+            data: { after: expect.any(InstanceElement) },
+          })],
+          pendingChanges: [],
+        }),
+      ]))
+    })
+    it('should calculate instance deletion (conflict)', () => {
+      expect(patchChanges).toEqual(expect.arrayContaining([
+        expect.objectContaining({
+          change: expect.objectContaining({
+            id: new ElemID('salto', 'type', 'instance', 'deleted'),
+            data: { before: expect.any(InstanceElement) },
+          }),
+          serviceChanges: [expect.objectContaining({
+            id: new ElemID('salto', 'type', 'instance', 'deleted'),
+            data: { before: expect.any(InstanceElement) },
+          })],
+          pendingChanges: [expect.objectContaining({
+            id: new ElemID('salto', 'type', 'instance', 'deleted', 'name'),
+            data: { before: 'before', after: 'other' },
+          })],
+        }),
+      ]))
+    })
+  })
+
   describe('calculatePatch', () => {
     const type = new ObjectType({
       elemID: new ElemID('salesforce', 'type'),

packages/netsuite-adapter/src/client/types.ts

@@ -13,7 +13,7 @@
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
-import { Change, ChangeData, ElemID, getChangeData, InstanceElement, isInstanceChange, isObjectType, isObjectTypeChange, ObjectType, Values } from '@salto-io/adapter-api'
+import { Change, ChangeData, ElemID, getChangeData, InstanceElement, isInstanceChange, isObjectType, isObjectTypeChange, ObjectType, TopLevelElement, Values } from '@salto-io/adapter-api'
 import { toCustomRecordTypeInstance } from '../custom_records/custom_record_type'
 import { NetsuiteFilePathsQueryParams, NetsuiteTypesQueryParams, ObjectID } from '../query'
 
@@ -87,7 +87,7 @@ export type ImportObjectsResult = {
 }
 
 export type DataElementsResult = {
-  elements: (ObjectType | InstanceElement)[]
+  elements: TopLevelElement[]
   requestedTypes: string[]
   largeTypesError: string[]
 }

packages/netsuite-adapter/src/reference_dependencies.ts

@@ -17,7 +17,7 @@ import _ from 'lodash'
 import { logger } from '@salto-io/logging'
 import {
   isInstanceElement, isPrimitiveType, ElemID, getFieldType,
-  isReferenceExpression, Value, isServiceId, isObjectType, ChangeDataType, ObjectType, InstanceElement,
+  isReferenceExpression, Value, isServiceId, isObjectType, ChangeDataType, TopLevelElement,
 } from '@salto-io/adapter-api'
 import { transformElement, TransformFunc } from '@salto-io/adapter-utils'
 import { values as lowerDashValues, collections } from '@salto-io/lowerdash'
@@ -30,7 +30,6 @@ const { awu } = collections.asynciterable
 const { isDefined } = lowerDashValues
 const log = logger(module)
 
-type TopLevelElement = ObjectType | InstanceElement
 const isTopLevelElement = (value: unknown): value is TopLevelElement =>
   isObjectType(value) || isInstanceElement(value)
 

packages/netsuite-adapter/src/suiteapp_config_elements.ts

@@ -14,7 +14,7 @@
 * limitations under the License.
 */
 import _ from 'lodash'
-import { BuiltinTypes, ElemID, getChangeData, InstanceElement, isInstanceElement, ModificationChange, ObjectType } from '@salto-io/adapter-api'
+import { BuiltinTypes, ElemID, getChangeData, InstanceElement, isInstanceElement, ModificationChange, ObjectType, TopLevelElement } from '@salto-io/adapter-api'
 import { NETSUITE, SELECT_OPTION, SETTINGS_PATH, TYPES_PATH } from './constants'
 import { SUITEAPP_CONFIG_TYPES_TO_TYPE_NAMES, DeployResult } from './types'
 import { NetsuiteQuery } from './query'
@@ -32,9 +32,7 @@ export const getConfigTypes = (): ObjectType[] => ([new ObjectType({
 export const toConfigElements = (
   configRecords: ConfigRecord[],
   fetchQuery: NetsuiteQuery
-): (
-  ObjectType | InstanceElement
-)[] => {
+): TopLevelElement[] => {
   const elements = configRecords
     .flatMap(configRecord => {
       const typeName = SUITEAPP_CONFIG_TYPES_TO_TYPE_NAMES[configRecord.configType]