diff --git a/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/tests/lineageTests.pure b/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/tests/lineageTests.pure index 2a5aaf2e1a..83380aaa84 100644 --- a/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/tests/lineageTests.pure +++ b/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/tests/lineageTests.pure @@ -478,3 +478,18 @@ function <> meta::analytics::lineage::tests::testLineageForComplexAgg assertEquals($relationTree, meta::pure::lineage::scanRelations::relationTreeAsString($lineage.relationTree)); } + + + +function <> meta::analytics::lineage::tests::relational::testEmbeddedWithMilestonedDuplicateProperties():Boolean[1] +{ + let fn= {|meta::relational::tests::milestoning::Product.all(%2020-02-02)->project([p|$p.classification(%2020-02-02).type], ['type']) }; + + let mapping = meta::relational::tests::milestoning::milestoningMapWithEmbeddedDuplicateProperty; + let runtime = runtime(); + let lineage = computeLineage($fn, $mapping, $runtime, relationalExtensions()); + meta::analytics::lineage::assertLineage(['Lambda', 'db_db', 'tb_dbdefaultProductClassificationSystemTable', 'tb_dbdefaultProductTable'], + ['Lambda', 'meta::relational::tests::milestoning::Product', 'meta::relational::tests::milestoning::ProductClassification', 'pack_meta::relational::tests::milestoning'], + '[type: [ProductClassificationSystemTable.id , ProductClassificationSystemTable.name , ProductTable.classificationSystemId ]]', + $lineage); +} diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/lineage/scanColumns/scanColumns.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/lineage/scanColumns/scanColumns.pure index 3c42462e64..4595d559ac 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/lineage/scanColumns/scanColumns.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/lineage/scanColumns/scanColumns.pure @@ -113,37 +113,35 @@ function <> meta::pure::lineage::scanColumns::processNonDataType state: ColResWithState[1], debug:DebugContext[1]): ColResWithState[1] { - let targetSetImplementationId = $pm->match([ - i: InlineEmbeddedRelationalInstanceSetImplementation[1]| - $m->allSubSetImplementationIds($i.inlineSetImplementationId)->removeDuplicates(), - s:SemiStructuredRelationalPropertyMapping[1] | [], - p:PropertyMapping[1]| - let targetSiId = $p.targetSetImplementationId; - let nIdToSet = $targetSiId->fold({sid,res|if($res->keys()->contains($sid), |$res ,|$res->keyValues()->concatenate(pair($sid,list($m->classMappingById($sid))))->newMap())}, $state.idToSet); - let targetClass = $nIdToSet->get($targetSiId).values.class; - - if ($targetClass->isNotEmpty() && $possiblePropertyTargetClasses->isNotEmpty() && !$possiblePropertyTargetClasses->exists(c|$targetClass->toOne()->_subTypeOf($c)), - | if($possiblePropertyTargetClasses->exists(c|$c->_subTypeOf($targetClass->toOne())), - |$m->_classMappingByClass($possiblePropertyTargetClasses->filter(c|$c->_subTypeOf($targetClass->toOne()))->toOne())->map(s|$s->resolveOperation($m)).id, |[]), - | $targetSiId;); + + let targetSetImplementationId = $pm->match( [ i: InlineEmbeddedRelationalInstanceSetImplementation[1]|$m->allSubSetImplementationIds($i.inlineSetImplementationId)->removeDuplicates(), + s:SemiStructuredRelationalPropertyMapping[1] | [], + p:PropertyMapping[1]| let targetSiId = $p.targetSetImplementationId; + let nIdToSet = $targetSiId->fold({sid,res|if($res->keys()->contains($sid), |$res ,|$res->keyValues()->concatenate(pair($sid,list($m->classMappingById($sid))))->newMap())}, $state.idToSet); + let targetClass = $nIdToSet->get($targetSiId).values.class; + if ($targetClass->isNotEmpty() && $possiblePropertyTargetClasses->isNotEmpty() && !$possiblePropertyTargetClasses->exists(c|$targetClass->toOne()->_subTypeOf($c)), + | if($possiblePropertyTargetClasses->exists(c|$c->_subTypeOf($targetClass->toOne())), + |$m->_classMappingByClass($possiblePropertyTargetClasses->filter(c|$c->_subTypeOf($targetClass->toOne()))->toOne())->map(s|$s->resolveOperation($m)).id, |[]), + | $targetSiId;); ]); - - let columnsFromSemiStructuredMappings = $pm->match([s:SemiStructuredRelationalPropertyMapping[1] | $s->getColumns(), a:Any[*]|[]->cast(@ColumnWithContext)]); - - let potentialColumnsFromJoin = $pm->match([ - r:RelationalPropertyMapping[1]| - if($targetSetImplementationId->contains($r.targetSetImplementationId), - | $r->getColumns(), - | []), - a:Any[1]|[] - ]); - - let nIdToSet = $targetSetImplementationId->fold({sid,res|if($res->keys()->contains($sid), |$res ,|$res->keyValues()->concatenate(pair($sid,list($m->classMappingById($sid))))->newMap())}, $state.idToSet); - + + + let columnsFromSemiStructuredMappings = $pm->match([s:SemiStructuredRelationalPropertyMapping[1] | $s->getColumns(), + a:Any[*]|[]->cast(@ColumnWithContext)]); + + let potentialColumnsFromJoin = $pm->match([r:RelationalPropertyMapping[1]|if($targetSetImplementationId->contains($r.targetSetImplementationId), + | $r->getColumns(), + | []), + a:Any[1]|[] + ]); + + let nIdToSet = $targetSetImplementationId->fold({sid,res|if($res->keys()->contains($sid), + |$res, + |$res->keyValues()->concatenate(pair($sid,list($m->classMappingById($sid))))->newMap())}, $state.idToSet); + + let targetSetVals = $targetSetImplementationId->map(id|let vals = $nIdToSet->get($id).values); let columnsFromChildren = meta::pure::lineage::scanColumns::manageQualifiers($p.children, []) - ->map(c|$c->scanColumns($m, $targetSetImplementationId->map(id| let vals = $nIdToSet->get($id).values; - assert($vals->isNotEmpty(),'Expected to find id but did not '+ $id ); - $vals->toOne();), $state.classToSets, $nIdToSet, false, $debug)); + ->map(c| $c->scanColumns($m,$targetSetVals,$state.classToSets, $nIdToSet, false, $debug)); ^ColResWithState(colRes=^ColRes(columns = $state.colRes.columns->concatenate($potentialColumnsFromJoin)->concatenate($columnsFromSemiStructuredMappings)->concatenate($columnsFromChildren.colRes.columns), sets=$targetSetImplementationId->map(id|$nIdToSet->get($id).values)), classToSets=$columnsFromChildren.classToSets->map(n|$n->keyValues())->newMap(), diff --git a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/milestoning/tests/businessDateMilestoningSetUp.pure b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/milestoning/tests/businessDateMilestoningSetUp.pure index 94a00989de..d52e820482 100644 --- a/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/milestoning/tests/businessDateMilestoningSetUp.pure +++ b/legend-engine-xts-relationalStore/legend-engine-xt-relationalStore-generation/legend-engine-xt-relationalStore-pure/legend-engine-xt-relationalStore-core-pure/src/main/resources/core_relational/relational/milestoning/tests/businessDateMilestoningSetUp.pure @@ -746,6 +746,21 @@ Mapping meta::relational::tests::milestoning::milestoningMapWithEmbeddedSimple ) +Mapping meta::relational::tests::milestoning::milestoningMapWithEmbeddedDuplicateProperty +( + meta::relational::tests::milestoning::Product : Relational{ + + id : [db]ProductTable.id, + name : [db]ProductTable.name, + + classification( + type : [db]@StockProduct_ClassificationSystem | ProductClassificationSystemTable.name + ), + classification : [db]@StockProduct_ClassificationSystem + } +) + + Mapping meta::relational::tests::milestoning::isolationFocusedMapping (