Skip to content

Commit

Permalink
Initial M2M Graphfetch support in lineage computation (finos#3058)
Browse files Browse the repository at this point in the history
* Initial M2M lineage support
  • Loading branch information
aziemchawdhary-gs authored Sep 19, 2024
1 parent 53b11e2 commit c8d2e88
Show file tree
Hide file tree
Showing 4 changed files with 94 additions and 6 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.

import meta::pure::router::metamodel::clustering::*;
import meta::pure::milestoning::*;
import meta::pure::executionPlan::*;
import meta::pure::router::routing::*;
Expand Down Expand Up @@ -151,7 +152,9 @@ function meta::pure::mapping::modelToModel::chain::allReprocess(f:FunctionExpres
print(if(!$debug.debug, |'', | $debug.space+' Mapping: '+$a.name->toOne()+'\n'));
let rez = ^LambdaFunction<{->Any[*]}>(expressionSequence = $b.res->cast(@FunctionExpression))->routeFunction($a, ^Runtime(), $extensions, $debug);
print(if(!$debug.debug, |'', | $debug.space+' Routed: '+$rez->at(0)->asString()+'\n'));
let res = $rez.expressionSequence->evaluateAndDeactivate()->cast(@StoreMappingClusteredValueSpecification).val->match(
let res = $rez.expressionSequence->evaluateAndDeactivate()->cast(@ClusteredValueSpecification).val->match(
// let res = $rez.expressionSequence->evaluateAndDeactivate()->cast(@StoreMappingClusteredValueSpecification).val->match(

[
e:StoreMappingRoutedValueSpecification[1]|pair(list($e), $e.value->cast(@FunctionExpression)->toOne()->reprocess($e, $a, $mappings, [], $extensions).newExpression);,
f:FunctionExpression[1]|pair(list($r), $f->reprocess($r, $a, $mappings, [], $extensions).newExpression);
Expand Down Expand Up @@ -223,6 +226,7 @@ function meta::pure::mapping::modelToModel::chain::reprocess(vs:ValueSpecificati
);,
r:FunctionRoutedValueSpecification[1]|$r.value->reprocess($e, $mapping, $mappings, $r, $extensions);,
e:StoreMappingRoutedValueSpecification[1]|$e.value->reprocess($e, $mapping, $mappings, [], $extensions);,
s:StoreMappingClusteredValueSpecification[1] | $s.val->reprocess($e, $mapping, $mappings, [], $extensions);,
v:VariableExpression[1]| let newGenericType = if($v.genericType.rawType->toOne()->instanceOf(Class) && $e.sets->isNotEmpty(),| ^GenericType(rawType=$e.sets->at(0)->cast(@PureInstanceSetImplementation).srcClass), | $v.genericType);
^Res(
newExpression = ^$v(genericType=$newGenericType),
Expand Down Expand Up @@ -293,7 +297,9 @@ function meta::pure::mapping::modelToModel::chain::reprocess(vs:ValueSpecificati
d:Date[1]|$d,
b:Boolean[1]|$b,
f:Float[1]|$f,
e:Enumeration<Any>[1]|$e
e:Enumeration<Any>[1]|$e,
st:StoreMappingClusteredValueSpecification[1]| $st,
gft:meta::pure::graphFetch::GraphFetchTree[1] | $gft
]););
let values = $results->map(r|$r->match([v:Res[1]|$v.newExpression->match([i:InstanceValue[1]|$i.values,a:Any[1]|$a]),k:Any[1]|$k]));

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,10 @@
<groupId>org.finos.legend.pure</groupId>
<artifactId>legend-pure-m2-dsl-path-pure</artifactId>
</dependency>
<dependency>
<groupId>org.finos.legend.pure</groupId>
<artifactId>legend-pure-m2-dsl-graph-pure</artifactId>
</dependency>
<dependency>
<groupId>org.finos.legend.pure</groupId>
<artifactId>legend-pure-m2-dsl-mapping-pure</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -84,12 +84,13 @@ function meta::analytics::lineage::computeLineage(f:FunctionDefinition<Any>[1],
{
let mappings = if($r->isEmpty(), |$m, |$m->concatenate(getMappingsFromRuntime($r->toOne())));
let modelToModelMappings = $mappings->init();

let sourceMapping = $mappings->last()->toOne();
let funcBody = $f.expressionSequence->at(0)->evaluateAndDeactivate();
let updatedFuncBody = $funcBody->meta::pure::lineage::analytics::inlineQualifiedProperties(newMap([]->cast(@Pair<VariableExpression, ValueSpecification>), VariableExpression->classPropertyByName('name')->cast(@Property<VariableExpression,String|1>)), $f->openVariableValues(), $extensions);
let propertyTrees = $updatedFuncBody->buildMultiLevelPropertyTrees($modelToModelMappings, $extensions);

let reprocessedFuncBody = if($modelToModelMappings->isEmpty(),
let reprocessedFuncBody = if($modelToModelMappings->isEmpty() && $mappings->size() == 1,
|$updatedFuncBody,
|$updatedFuncBody->cast(@FunctionExpression)->meta::pure::mapping::modelToModel::chain::allReprocess([], $modelToModelMappings, $extensions, noDebug()).res);

Expand All @@ -98,10 +99,11 @@ function meta::analytics::lineage::computeLineage(f:FunctionDefinition<Any>[1],
| scanRelations(^LambdaFunction<{->Any[*]}>(expressionSequence = $funcBody), $sourceMapping, $r->toOne(), $vars, noDebug(), $extensions),
| scanRelations($propertyTrees->last()->toOne(), $sourceMapping));

let classLineageMapping = if($modelToModelMappings->isEmpty() && $mappings->size() == 1, | $sourceMapping, | $modelToModelMappings);
^FunctionAnalytics
(
databaseLineage = $f->toFlowDatabase($sourceMapping, $propertyTrees->last()->toOne(), $r)->toGraph(),
classLineage = $f->toFlowClass($propertyTrees->at(0), $modelToModelMappings)->toGraph(),
classLineage = $f->toFlowClass($propertyTrees->at(0), $classLineageMapping)->toGraph(),
functionTrees = $propertyTrees,
relationTree = $relationTree,
reportLineage = buildReportLineage($reprocessedFuncBody, $sourceMapping)
Expand Down Expand Up @@ -189,7 +191,7 @@ function meta::analytics::lineage::flowDatabase::toFlowDatabase(f:FunctionDefini
let dbs = $tables->map(r|$r->schema()).database->removeDuplicates();
let maturityTests = maturityTests();
^Flow(
functions = $f,
functions = if($tables->size() == 0 && $dbs->size() == 0, | [], | $f),
databases = $dbs,
tables = $tables->cast(@NamedRelation),
links = $tables->map(t|let db = $t->map(r|$r->schema()).database->toOne();
Expand Down Expand Up @@ -218,7 +220,11 @@ function meta::analytics::lineage::flowDatabase::toFlowDatabase(p:PropertyPathTr
sp:Property<Nil,Any|*>[1]|let propertyMappings = $wsets->map(s|$s->_propertyMappingsByPropertyName($pr.property.name->toOne()););
let isDataTypeProperty = !$pr.property.genericType.rawType->isEmpty() && $pr.property.genericType.rawType->toOne()->instanceOf(DataType);
if ($isDataTypeProperty,
| $propertyMappings->cast(@RelationalPropertyMapping)->map(pm|$pm->meta::analytics::lineage::flowDatabase::getTables());,
| $propertyMappings->map(pm |$pm->match([
rpm: RelationalPropertyMapping[1] | $rpm->meta::analytics::lineage::flowDatabase::getTables(),
ppm: PurePropertyMapping[1] | [],
a: Any[*] | fail('Database lineage not support for given type of mapping: ' + $pm->typeName()); [];
])),
| $propertyMappings->map(pm|processNonDataTypeProperty($p, $pm, $possiblePropertyTargetClasses, $m, $extraChildren))
);,
q:QualifiedProperty<Any>[1]|if($q->meta::pure::milestoning::hasGeneratedMilestoningPropertyStereotype(),
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
import meta::analytics::lineage::*;
import meta::pure::extension::*;
import meta::core::runtime::*;
import meta::pure::graphFetch::execution::*;
import meta::analytics::lineage::tests::*;

function <<meta::pure::profiles::test.Test>> meta::analytics::lineage::tests::testSimpleM2M() : Boolean[1]
{
let result = computeLineage({| TargetClass.all()->graphFetch(#{TargetClass{value1}}#)->serialize(#{TargetClass{value1}}#)},
mappingForTestPropertyLineage,
^Runtime(connectionStores = ^meta::core::runtime::ConnectionStore(
connection=^meta::external::store::model::ModelConnection(instances = newMap(pair(TargetClass, list([])))),
element=^meta::external::store::model::ModelStore())),
defaultExtensions()
);

let expectedDatabaseNodesId = [];
let expectedDatabaseEdgesId = [];

let expectedClassNodesId = [
'Lambda',
'meta::analytics::lineage::tests::TargetClass',
'meta::analytics::lineage::tests::SourceClass',
'pack_meta::analytics::lineage::tests'
];

let expectedClassEdgesId = [
'Lambda -> meta::analytics::lineage::tests::TargetClass',
'meta::analytics::lineage::tests::TargetClass -> pack_meta::analytics::lineage::tests',
'meta::analytics::lineage::tests::TargetClass -> meta::analytics::lineage::tests::SourceClass',
'meta::analytics::lineage::tests::SourceClass -> pack_meta::analytics::lineage::tests'
];

let expectedPropertyTrees = [
['root\n',
' c_TargetClass\n',
' p_TargetClass.value1\n']->joinStrings()
];

assertSameElements($expectedDatabaseNodesId, $result.databaseLineage.nodes);
assertSameElements($expectedDatabaseEdgesId, $result.databaseLineage.edges);
assertSameElements($expectedClassNodesId, $result.classLineage.nodes.data.id);
assertSameElements($expectedClassEdgesId, $result.classLineage.edges.data->map(m | $m.target.data.id + ' -> ' + $m.source.data.id));
assertSameElements($expectedPropertyTrees, $result.functionTrees->map(ft | $ft->meta::pure::lineage::scanProperties::propertyTree::printTree('')));
}

Class meta::analytics::lineage::tests::TargetClass
{
value1 : String[1];
}

Class meta::analytics::lineage::tests::SourceClass
{
src1 : SourceClass1[1];
src2: String[1];
}

Class meta::analytics::lineage::tests::SourceClass1
{
value : String[1];
}

###Mapping
import meta::analytics::lineage::tests::*;
Mapping meta::analytics::lineage::tests::mappingForTestPropertyLineage
(
TargetClass : Pure
{
~src SourceClass
value1: $src.src1.value + '_' + $src.src2
}
)

0 comments on commit c8d2e88

Please sign in to comment.