From c8d2e8821ae17d7ab7fe2b9094c0bd7e9f484004 Mon Sep 17 00:00:00 2001 From: Aziem Chawdhary <61746398+aziemchawdhary-gs@users.noreply.github.com> Date: Thu, 19 Sep 2024 14:43:43 +0100 Subject: [PATCH] Initial M2M Graphfetch support in lineage computation (#3058) * Initial M2M lineage support --- .../main/resources/core/store/m2m/chain.pure | 10 ++- .../pom.xml | 4 ++ .../core_analytics_lineage/fullAnalytics.pure | 14 ++-- .../tests/m2mAnalytics.pure | 72 +++++++++++++++++++ 4 files changed, 94 insertions(+), 6 deletions(-) create mode 100644 legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/tests/m2mAnalytics.pure diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/store/m2m/chain.pure b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/store/m2m/chain.pure index d302c3708dd..2a4020fef7f 100644 --- a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/store/m2m/chain.pure +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/store/m2m/chain.pure @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +import meta::pure::router::metamodel::clustering::*; import meta::pure::milestoning::*; import meta::pure::executionPlan::*; import meta::pure::router::routing::*; @@ -151,7 +152,9 @@ function meta::pure::mapping::modelToModel::chain::allReprocess(f:FunctionExpres print(if(!$debug.debug, |'', | $debug.space+' Mapping: '+$a.name->toOne()+'\n')); let rez = ^LambdaFunction<{->Any[*]}>(expressionSequence = $b.res->cast(@FunctionExpression))->routeFunction($a, ^Runtime(), $extensions, $debug); print(if(!$debug.debug, |'', | $debug.space+' Routed: '+$rez->at(0)->asString()+'\n')); - let res = $rez.expressionSequence->evaluateAndDeactivate()->cast(@StoreMappingClusteredValueSpecification).val->match( + let res = $rez.expressionSequence->evaluateAndDeactivate()->cast(@ClusteredValueSpecification).val->match( + // let res = $rez.expressionSequence->evaluateAndDeactivate()->cast(@StoreMappingClusteredValueSpecification).val->match( + [ e:StoreMappingRoutedValueSpecification[1]|pair(list($e), $e.value->cast(@FunctionExpression)->toOne()->reprocess($e, $a, $mappings, [], $extensions).newExpression);, f:FunctionExpression[1]|pair(list($r), $f->reprocess($r, $a, $mappings, [], $extensions).newExpression); @@ -223,6 +226,7 @@ function meta::pure::mapping::modelToModel::chain::reprocess(vs:ValueSpecificati );, r:FunctionRoutedValueSpecification[1]|$r.value->reprocess($e, $mapping, $mappings, $r, $extensions);, e:StoreMappingRoutedValueSpecification[1]|$e.value->reprocess($e, $mapping, $mappings, [], $extensions);, + s:StoreMappingClusteredValueSpecification[1] | $s.val->reprocess($e, $mapping, $mappings, [], $extensions);, v:VariableExpression[1]| let newGenericType = if($v.genericType.rawType->toOne()->instanceOf(Class) && $e.sets->isNotEmpty(),| ^GenericType(rawType=$e.sets->at(0)->cast(@PureInstanceSetImplementation).srcClass), | $v.genericType); ^Res( newExpression = ^$v(genericType=$newGenericType), @@ -293,7 +297,9 @@ function meta::pure::mapping::modelToModel::chain::reprocess(vs:ValueSpecificati d:Date[1]|$d, b:Boolean[1]|$b, f:Float[1]|$f, - e:Enumeration[1]|$e + e:Enumeration[1]|$e, + st:StoreMappingClusteredValueSpecification[1]| $st, + gft:meta::pure::graphFetch::GraphFetchTree[1] | $gft ]);); let values = $results->map(r|$r->match([v:Res[1]|$v.newExpression->match([i:InstanceValue[1]|$i.values,a:Any[1]|$a]),k:Any[1]|$k])); diff --git a/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/pom.xml b/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/pom.xml index c593186eb61..c658ba35c93 100644 --- a/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/pom.xml +++ b/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/pom.xml @@ -182,6 +182,10 @@ org.finos.legend.pure legend-pure-m2-dsl-path-pure + + org.finos.legend.pure + legend-pure-m2-dsl-graph-pure + org.finos.legend.pure legend-pure-m2-dsl-mapping-pure diff --git a/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/fullAnalytics.pure b/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/fullAnalytics.pure index ee37620397b..e4dea70c4e8 100644 --- a/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/fullAnalytics.pure +++ b/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/fullAnalytics.pure @@ -84,12 +84,13 @@ function meta::analytics::lineage::computeLineage(f:FunctionDefinition[1], { let mappings = if($r->isEmpty(), |$m, |$m->concatenate(getMappingsFromRuntime($r->toOne()))); let modelToModelMappings = $mappings->init(); + let sourceMapping = $mappings->last()->toOne(); let funcBody = $f.expressionSequence->at(0)->evaluateAndDeactivate(); let updatedFuncBody = $funcBody->meta::pure::lineage::analytics::inlineQualifiedProperties(newMap([]->cast(@Pair), VariableExpression->classPropertyByName('name')->cast(@Property)), $f->openVariableValues(), $extensions); let propertyTrees = $updatedFuncBody->buildMultiLevelPropertyTrees($modelToModelMappings, $extensions); - let reprocessedFuncBody = if($modelToModelMappings->isEmpty(), + let reprocessedFuncBody = if($modelToModelMappings->isEmpty() && $mappings->size() == 1, |$updatedFuncBody, |$updatedFuncBody->cast(@FunctionExpression)->meta::pure::mapping::modelToModel::chain::allReprocess([], $modelToModelMappings, $extensions, noDebug()).res); @@ -98,10 +99,11 @@ function meta::analytics::lineage::computeLineage(f:FunctionDefinition[1], | scanRelations(^LambdaFunction<{->Any[*]}>(expressionSequence = $funcBody), $sourceMapping, $r->toOne(), $vars, noDebug(), $extensions), | scanRelations($propertyTrees->last()->toOne(), $sourceMapping)); + let classLineageMapping = if($modelToModelMappings->isEmpty() && $mappings->size() == 1, | $sourceMapping, | $modelToModelMappings); ^FunctionAnalytics ( databaseLineage = $f->toFlowDatabase($sourceMapping, $propertyTrees->last()->toOne(), $r)->toGraph(), - classLineage = $f->toFlowClass($propertyTrees->at(0), $modelToModelMappings)->toGraph(), + classLineage = $f->toFlowClass($propertyTrees->at(0), $classLineageMapping)->toGraph(), functionTrees = $propertyTrees, relationTree = $relationTree, reportLineage = buildReportLineage($reprocessedFuncBody, $sourceMapping) @@ -189,7 +191,7 @@ function meta::analytics::lineage::flowDatabase::toFlowDatabase(f:FunctionDefini let dbs = $tables->map(r|$r->schema()).database->removeDuplicates(); let maturityTests = maturityTests(); ^Flow( - functions = $f, + functions = if($tables->size() == 0 && $dbs->size() == 0, | [], | $f), databases = $dbs, tables = $tables->cast(@NamedRelation), links = $tables->map(t|let db = $t->map(r|$r->schema()).database->toOne(); @@ -218,7 +220,11 @@ function meta::analytics::lineage::flowDatabase::toFlowDatabase(p:PropertyPathTr sp:Property[1]|let propertyMappings = $wsets->map(s|$s->_propertyMappingsByPropertyName($pr.property.name->toOne());); let isDataTypeProperty = !$pr.property.genericType.rawType->isEmpty() && $pr.property.genericType.rawType->toOne()->instanceOf(DataType); if ($isDataTypeProperty, - | $propertyMappings->cast(@RelationalPropertyMapping)->map(pm|$pm->meta::analytics::lineage::flowDatabase::getTables());, + | $propertyMappings->map(pm |$pm->match([ + rpm: RelationalPropertyMapping[1] | $rpm->meta::analytics::lineage::flowDatabase::getTables(), + ppm: PurePropertyMapping[1] | [], + a: Any[*] | fail('Database lineage not support for given type of mapping: ' + $pm->typeName()); []; + ])), | $propertyMappings->map(pm|processNonDataTypeProperty($p, $pm, $possiblePropertyTargetClasses, $m, $extraChildren)) );, q:QualifiedProperty[1]|if($q->meta::pure::milestoning::hasGeneratedMilestoningPropertyStereotype(), diff --git a/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/tests/m2mAnalytics.pure b/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/tests/m2mAnalytics.pure new file mode 100644 index 00000000000..6195a77145e --- /dev/null +++ b/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/tests/m2mAnalytics.pure @@ -0,0 +1,72 @@ +import meta::analytics::lineage::*; +import meta::pure::extension::*; +import meta::core::runtime::*; +import meta::pure::graphFetch::execution::*; +import meta::analytics::lineage::tests::*; + +function <> meta::analytics::lineage::tests::testSimpleM2M() : Boolean[1] +{ + let result = computeLineage({| TargetClass.all()->graphFetch(#{TargetClass{value1}}#)->serialize(#{TargetClass{value1}}#)}, + mappingForTestPropertyLineage, + ^Runtime(connectionStores = ^meta::core::runtime::ConnectionStore( + connection=^meta::external::store::model::ModelConnection(instances = newMap(pair(TargetClass, list([])))), + element=^meta::external::store::model::ModelStore())), + defaultExtensions() + ); + + let expectedDatabaseNodesId = []; + let expectedDatabaseEdgesId = []; + + let expectedClassNodesId = [ + 'Lambda', + 'meta::analytics::lineage::tests::TargetClass', + 'meta::analytics::lineage::tests::SourceClass', + 'pack_meta::analytics::lineage::tests' + ]; + + let expectedClassEdgesId = [ + 'Lambda -> meta::analytics::lineage::tests::TargetClass', + 'meta::analytics::lineage::tests::TargetClass -> pack_meta::analytics::lineage::tests', + 'meta::analytics::lineage::tests::TargetClass -> meta::analytics::lineage::tests::SourceClass', + 'meta::analytics::lineage::tests::SourceClass -> pack_meta::analytics::lineage::tests' + ]; + + let expectedPropertyTrees = [ + ['root\n', + ' c_TargetClass\n', + ' p_TargetClass.value1\n']->joinStrings() + ]; + + assertSameElements($expectedDatabaseNodesId, $result.databaseLineage.nodes); + assertSameElements($expectedDatabaseEdgesId, $result.databaseLineage.edges); + assertSameElements($expectedClassNodesId, $result.classLineage.nodes.data.id); + assertSameElements($expectedClassEdgesId, $result.classLineage.edges.data->map(m | $m.target.data.id + ' -> ' + $m.source.data.id)); + assertSameElements($expectedPropertyTrees, $result.functionTrees->map(ft | $ft->meta::pure::lineage::scanProperties::propertyTree::printTree(''))); +} + +Class meta::analytics::lineage::tests::TargetClass +{ + value1 : String[1]; +} + +Class meta::analytics::lineage::tests::SourceClass +{ + src1 : SourceClass1[1]; + src2: String[1]; +} + +Class meta::analytics::lineage::tests::SourceClass1 +{ + value : String[1]; +} + +###Mapping +import meta::analytics::lineage::tests::*; +Mapping meta::analytics::lineage::tests::mappingForTestPropertyLineage +( + TargetClass : Pure + { + ~src SourceClass + value1: $src.src1.value + '_' + $src.src2 + } +)