fix lineage for embedded properties with milestoning (finos#3177)

goldmansachs · Oct 15, 2024 · 14a6f84 · 14a6f84
1 parent 8b862b1
commit 14a6f84
Show file tree

Hide file tree

Showing 3 changed files with 57 additions and 29 deletions.
diff --git a/...-analytics-lineage-pure/src/main/resources/core_analytics_lineage/tests/lineageTests.pure b/...-analytics-lineage-pure/src/main/resources/core_analytics_lineage/tests/lineageTests.pure
@@ -478,3 +478,18 @@ function <<test.Test>> meta::analytics::lineage::tests::testLineageForComplexAgg
 
   assertEquals($relationTree, meta::pure::lineage::scanRelations::relationTreeAsString($lineage.relationTree));
 }
+
+
+
+function <<meta::pure::profiles::test.Test>> meta::analytics::lineage::tests::relational::testEmbeddedWithMilestonedDuplicateProperties():Boolean[1]
+{
+   let fn= {|meta::relational::tests::milestoning::Product.all(%2020-02-02)->project([p|$p.classification(%2020-02-02).type], ['type']) };
+
+  let mapping = meta::relational::tests::milestoning::milestoningMapWithEmbeddedDuplicateProperty;
+  let runtime = runtime();
+  let lineage = computeLineage($fn, $mapping, $runtime, relationalExtensions());
+  meta::analytics::lineage::assertLineage(['Lambda', 'db_db', 'tb_dbdefaultProductClassificationSystemTable', 'tb_dbdefaultProductTable'],
+                                        ['Lambda', 'meta::relational::tests::milestoning::Product', 'meta::relational::tests::milestoning::ProductClassification', 'pack_meta::relational::tests::milestoning'],
+                                        '[type: [ProductClassificationSystemTable.id <JoinTreeNode>, ProductClassificationSystemTable.name <RelationalOperationElementWithJoin>, ProductTable.classificationSystemId <JoinTreeNode>]]',
+                                          $lineage);
+}
diff --git a/...e-pure/src/main/resources/core_relational/relational/lineage/scanColumns/scanColumns.pure b/...e-pure/src/main/resources/core_relational/relational/lineage/scanColumns/scanColumns.pure
@@ -113,37 +113,35 @@ function <<access.private>> meta::pure::lineage::scanColumns::processNonDataType
                                                                                          state: ColResWithState[1],
                                                                                          debug:DebugContext[1]): ColResWithState[1]
 {
-   let targetSetImplementationId = $pm->match([
-      i: InlineEmbeddedRelationalInstanceSetImplementation[1]|
-      $m->allSubSetImplementationIds($i.inlineSetImplementationId)->removeDuplicates(),
-      s:SemiStructuredRelationalPropertyMapping[1] | [],
-      p:PropertyMapping[1]|
-      let targetSiId = $p.targetSetImplementationId;
-      let nIdToSet = $targetSiId->fold({sid,res|if($res->keys()->contains($sid), |$res ,|$res->keyValues()->concatenate(pair($sid,list($m->classMappingById($sid))))->newMap())}, $state.idToSet);
-      let targetClass = $nIdToSet->get($targetSiId).values.class;
-
-      if ($targetClass->isNotEmpty() && $possiblePropertyTargetClasses->isNotEmpty() && !$possiblePropertyTargetClasses->exists(c|$targetClass->toOne()->_subTypeOf($c)),
-          | if($possiblePropertyTargetClasses->exists(c|$c->_subTypeOf($targetClass->toOne())),
-               |$m->_classMappingByClass($possiblePropertyTargetClasses->filter(c|$c->_subTypeOf($targetClass->toOne()))->toOne())->map(s|$s->resolveOperation($m)).id, |[]),
-      | $targetSiId;);
+
+   let targetSetImplementationId = $pm->match( [ i: InlineEmbeddedRelationalInstanceSetImplementation[1]|$m->allSubSetImplementationIds($i.inlineSetImplementationId)->removeDuplicates(),
+                                                s:SemiStructuredRelationalPropertyMapping[1] | [],
+                                                p:PropertyMapping[1]| let targetSiId = $p.targetSetImplementationId;
+                                                                      let nIdToSet = $targetSiId->fold({sid,res|if($res->keys()->contains($sid), |$res ,|$res->keyValues()->concatenate(pair($sid,list($m->classMappingById($sid))))->newMap())}, $state.idToSet);
+                                                                      let targetClass = $nIdToSet->get($targetSiId).values.class;
+                                                                      if ($targetClass->isNotEmpty() && $possiblePropertyTargetClasses->isNotEmpty() && !$possiblePropertyTargetClasses->exists(c|$targetClass->toOne()->_subTypeOf($c)),
+                                                                          | if($possiblePropertyTargetClasses->exists(c|$c->_subTypeOf($targetClass->toOne())),
+                                                                                |$m->_classMappingByClass($possiblePropertyTargetClasses->filter(c|$c->_subTypeOf($targetClass->toOne()))->toOne())->map(s|$s->resolveOperation($m)).id, |[]),
+                                                                      | $targetSiId;);
                                            ]);
-
-   let columnsFromSemiStructuredMappings = $pm->match([s:SemiStructuredRelationalPropertyMapping[1] | $s->getColumns(), a:Any[*]|[]->cast(@ColumnWithContext)]);
-
-   let potentialColumnsFromJoin = $pm->match([
-      r:RelationalPropertyMapping[1]|
-      if($targetSetImplementationId->contains($r.targetSetImplementationId),
-         | $r->getColumns(),
-         | []),
-      a:Any[1]|[]
-   ]);
-
-   let nIdToSet = $targetSetImplementationId->fold({sid,res|if($res->keys()->contains($sid), |$res ,|$res->keyValues()->concatenate(pair($sid,list($m->classMappingById($sid))))->newMap())}, $state.idToSet);
-
+
+
+   let columnsFromSemiStructuredMappings = $pm->match([s:SemiStructuredRelationalPropertyMapping[1] | $s->getColumns(), 
+                                                      a:Any[*]|[]->cast(@ColumnWithContext)]);
+
+   let potentialColumnsFromJoin = $pm->match([r:RelationalPropertyMapping[1]|if($targetSetImplementationId->contains($r.targetSetImplementationId),
+                                                                                | $r->getColumns(),
+                                                                                | []),
+                                                 a:Any[1]|[]
+                                                 ]);
+
+   let nIdToSet = $targetSetImplementationId->fold({sid,res|if($res->keys()->contains($sid), 
+                                                                |$res,
+                                                                |$res->keyValues()->concatenate(pair($sid,list($m->classMappingById($sid))))->newMap())}, $state.idToSet);
+
+   let targetSetVals =  $targetSetImplementationId->map(id|let vals = $nIdToSet->get($id).values);
    let columnsFromChildren = meta::pure::lineage::scanColumns::manageQualifiers($p.children, [])
-   ->map(c|$c->scanColumns($m, $targetSetImplementationId->map(id|   let vals = $nIdToSet->get($id).values;  
-                                                                      assert($vals->isNotEmpty(),'Expected to find id but did not '+ $id );
-                                                                      $vals->toOne();), $state.classToSets, $nIdToSet, false, $debug));
+                                  ->map(c|  $c->scanColumns($m,$targetSetVals,$state.classToSets, $nIdToSet, false, $debug));
 
    ^ColResWithState(colRes=^ColRes(columns = $state.colRes.columns->concatenate($potentialColumnsFromJoin)->concatenate($columnsFromSemiStructuredMappings)->concatenate($columnsFromChildren.colRes.columns), sets=$targetSetImplementationId->map(id|$nIdToSet->get($id).values)),
                     classToSets=$columnsFromChildren.classToSets->map(n|$n->keyValues())->newMap(),

diff --git a/.../resources/core_relational/relational/milestoning/tests/businessDateMilestoningSetUp.pure b/.../resources/core_relational/relational/milestoning/tests/businessDateMilestoningSetUp.pure
@@ -746,6 +746,21 @@ Mapping meta::relational::tests::milestoning::milestoningMapWithEmbeddedSimple
 
 )
 
+Mapping meta::relational::tests::milestoning::milestoningMapWithEmbeddedDuplicateProperty
+(
+   meta::relational::tests::milestoning::Product : Relational{
+
+        id : [db]ProductTable.id,
+        name : [db]ProductTable.name,
+
+        classification(
+           type : [db]@StockProduct_ClassificationSystem  | ProductClassificationSystemTable.name
+        ),
+      classification : [db]@StockProduct_ClassificationSystem 
+   }
+)
+
+
 Mapping meta::relational::tests::milestoning::isolationFocusedMapping
 (