From ee03aa7278b7baaf953b988b76087e99f2ad7124 Mon Sep 17 00:00:00 2001 From: Akash <109946032+siaka-Akash@users.noreply.github.com> Date: Mon, 23 Sep 2024 20:40:07 +0530 Subject: [PATCH] Support lineage of relational graph fetch queries (#3103) * add support for lineage in realtional graphfetch queries --- .../pure/graphFetch/graphFetch_routing.pure | 2 +- .../core/pure/lineage/scanProject.pure | 37 ++++ .../pom.xml | 5 + .../core_analytics_lineage/fullAnalytics.pure | 11 + .../tests/lineageTests.pure | 190 +++++++++++++++++- 5 files changed, 243 insertions(+), 2 deletions(-) diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/graphFetch/graphFetch_routing.pure b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/graphFetch/graphFetch_routing.pure index 143db0d86d2..4be336217af 100644 --- a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/graphFetch/graphFetch_routing.pure +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/graphFetch/graphFetch_routing.pure @@ -145,7 +145,7 @@ function meta::pure::graphFetch::routing::createGetAllApplicationForRootGraphFet )->evaluateAndDeactivate(); } -function <> meta::pure::graphFetch::routing::createFunctionApplicationForPropertyGraphFetchTree(prop: PropertyGraphFetchTree[1], functionExpression: ValueSpecification[1]):FunctionExpression[1] +function meta::pure::graphFetch::routing::createFunctionApplicationForPropertyGraphFetchTree(prop: PropertyGraphFetchTree[1], functionExpression: ValueSpecification[1]):FunctionExpression[1] { let mapDummyLambda = {x:Class[1] | $x.name}; let mapExpr = $mapDummyLambda.expressionSequence->evaluateAndDeactivate()->toOne()->cast(@FunctionExpression); diff --git a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/lineage/scanProject.pure b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/lineage/scanProject.pure index a7615d27a49..1fcf6b28dc7 100644 --- a/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/lineage/scanProject.pure +++ b/legend-engine-core/legend-engine-core-pure/legend-engine-pure-code-compiled-core/src/main/resources/core/pure/lineage/scanProject.pure @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +import meta::pure::graphFetch::*; import meta::pure::lineage::scanProject::*; import meta::pure::mapping::*; @@ -69,6 +70,11 @@ function <> meta::pure::lineage::scanProject::scanProjectRecursi columns = zip($names->match([s:String[*]|$s, a:Unknown[*]|range(1, $allFuncs->size(),1)->map(z|'unknown_'+$z->toString())]), $allFuncs->match([s:FunctionDefinition[*]|$s, a:Unknown[*]|[]])) ); + ), + pair('meta::pure::graphFetch::execution::graphFetch_T_MANY__RootGraphFetchTree_1__T_MANY_', + | + let tree = $fe.parametersValues->at(1)->match([s:InstanceValue[1]|$s.values, a:Any[*]|^Unknown()]); + ^Project(projectfuncEntryPoint = $fe, columns = scanRootGraphFetchTree($tree->cast(@RootGraphFetchTree)->toOne())); ) ]; let funcFullPath = $fe.func->toOne()->elementToPath(); @@ -98,6 +104,37 @@ function <> meta::pure::lineage::scanProject::scanProjectRecursi ); } +function <> meta::pure::lineage::scanProject::scanRootGraphFetchTree(root:meta::pure::graphFetch::RootGraphFetchTree[1]):Pair>[*] +{ + let rootGetAllExpression = meta::pure::lineage::scanProject::createGetAllApplicationForRootGraphFetchTree($root); + $root.subTrees->map(x | $x->cast(@PropertyGraphFetchTree)->recurseForPropertyGraphFetchTree($rootGetAllExpression,$root.class.name->toOne())); +} + +function <> meta::pure::lineage::scanProject::recurseForPropertyGraphFetchTree(prop: PropertyGraphFetchTree[1], functionExpression: ValueSpecification[1], name:String[1]):Pair>[*] +{ + let propertyApplicationExpression = meta::pure::graphFetch::routing::createFunctionApplicationForPropertyGraphFetchTree($prop, $functionExpression); + let propertyName = $name + '.' + $prop.property.name->toOne(); + if($prop.subTrees->isEmpty(), + | pair($propertyName, ^FunctionDefinition<{->TabularDataSet[1]}>(expressionSequence = $propertyApplicationExpression));, + | $prop.subTrees->map(x | $x->cast(@PropertyGraphFetchTree)->recurseForPropertyGraphFetchTree($propertyApplicationExpression,$propertyName)); + ); +} + +function <> meta::pure::lineage::scanProject::createGetAllApplicationForRootGraphFetchTree(root: RootGraphFetchTree[1]):FunctionExpression[1] +{ + let getAllExpression = ^SimpleFunctionExpression + ( + func = getAll_Class_1__T_MANY_, + functionName = 'getAll', + importGroup = system::imports::coreImport, + genericType = ^GenericType(rawType = $root.class), + multiplicity = ZeroMany, + parametersValues = ^InstanceValue(values = $root.class, genericType = ^GenericType(rawType = Class, typeArguments = ^GenericType(rawType = $root.class)), multiplicity = PureOne) + )->evaluateAndDeactivate(); +} + + + function <> meta::pure::lineage::scanProject::scanAggregateValue(val:Any[*]):FunctionDefinition[*] { if ($val->isEmpty(), diff --git a/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/pom.xml b/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/pom.xml index c658ba35c93..dc9de24fbd7 100644 --- a/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/pom.xml +++ b/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/pom.xml @@ -204,6 +204,11 @@ legend-engine-pure-functions-standard-pure ${project.version} + + org.finos.legend.pure + legend-pure-m2-dsl-graph-pure + ${legend.pure.version} + diff --git a/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/fullAnalytics.pure b/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/fullAnalytics.pure index e4dea70c4e8..a365ac266c2 100644 --- a/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/fullAnalytics.pure +++ b/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/fullAnalytics.pure @@ -386,3 +386,14 @@ function meta::analytics::lineage::transformColumns(c: meta::analytics::lineage: ) ); } + +//testing purpose +function meta::analytics::lineage::ReportLineageToString(reportLineage:ReportLineage[1]):String[1] +{ + $reportLineage.columns->map(c | $c.name + $c.columns->map(t|$t.column.owner->cast(@Table).name->toOne()+'.'+$t.column.name->toOne()+' <'+$t.context+'>')->removeDuplicates()->sort()->makeString(': [', ', ', ']'))->sort()->joinStrings('[', ', ', ']'); +} + +function meta::analytics::lineage::assertLineage(databaseLineage:String[*], classLineage:String[*], reportLineage:String[1], functionAnalytics:FunctionAnalytics[1]):Boolean[1] +{ + assertSameElements($databaseLineage, $functionAnalytics.databaseLineage.nodes.data.id) && assertSameElements($classLineage, $functionAnalytics.classLineage.nodes.data.id) && assertEquals($reportLineage, ReportLineageToString($functionAnalytics.reportLineage)); +} diff --git a/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/tests/lineageTests.pure b/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/tests/lineageTests.pure index b6c38f5678c..53af3b6ec2f 100644 --- a/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/tests/lineageTests.pure +++ b/legend-engine-xts-analytics/legend-engine-xts-analytics-lineage/legend-engine-xt-analytics-lineage-pure/src/main/resources/core_analytics_lineage/tests/lineageTests.pure @@ -137,4 +137,192 @@ function <> meta::analytics::lineage::tests::re let lineage = computeLineage($fn,simpleRelationalMapping, meta::external::store::relational::tests::testRuntime(), meta::relational::extension::relationalExtensions()); assertSameElements(['Lambda', 'db_db', 'db_dbInc', 'tb_dbIncdefaultaddressTable', 'tb_dbIncdefaultfirmTable', 'tb_dbIncdefaultpersonTable', 'tb_dbdefaultaccountTable', 'tb_dbdefaultorderPnlTable', 'tb_dbdefaultorderTable', 'tb_dbdefaultsalesPersonTable'],$lineage.databaseLineage.nodes.data.id); -} \ No newline at end of file +} + +###Pure +import meta::pure::graphFetch::execution::*; +import meta::relational::tests::model::simple::*; +function <> meta::analytics::lineage::fullAnalyticasTest::testForSimpleRelationalGraphFetch():Boolean[1] +{ + let tree = #{ + Person { + firstName, + lastName + } + }#; + + let query = {|Person.all()->graphFetch($tree)->serialize($tree)}; + let mapping = meta::relational::tests::simpleRelationalMapping; + let runtime = meta::external::store::relational::tests::testRuntime(); + + let lineage = meta::analytics::lineage::computeLineage($query, $mapping, $runtime, meta::relational::extension::relationalExtensions()); + + meta::analytics::lineage::assertLineage(['Lambda', 'db_dbInc', 'tb_dbIncdefaultpersonTable'], + ['Lambda', 'meta::relational::tests::model::simple::Person', 'pack_meta::relational::tests::model::simple'], + '[Person.firstName: [personTable.FIRSTNAME ], Person.lastName: [personTable.LASTNAME ]]', + $lineage); + +} + +function <> meta::analytics::lineage::fullAnalyticasTest::testForComplextRelationalGraphFetch():Boolean[1] +{ + let tree = #{ + Person { + firstName, + lastName, + firm { + legalName + }, + address { + name, + type + } + } + }#; + let query = {|Person.all()->graphFetch($tree)->serialize($tree)}; + let mapping = meta::relational::tests::simpleRelationalMapping; + let runtime = meta::external::store::relational::tests::testRuntime(); + let lineage = meta::analytics::lineage::computeLineage($query, $mapping, $runtime, meta::relational::extension::relationalExtensions()); + + meta::analytics::lineage::assertLineage(['Lambda', 'db_dbInc', 'tb_dbIncdefaultaddressTable', 'tb_dbIncdefaultfirmTable', 'tb_dbIncdefaultpersonTable'], + ['Lambda', 'meta::relational::tests::model::simple::Address', 'meta::relational::tests::model::simple::Firm', 'meta::relational::tests::model::simple::Person', 'pack_meta::relational::tests::model::simple'], + '[Person.address.name: [addressTable.ID , addressTable.NAME , personTable.ADDRESSID ], Person.address.type: [addressTable.ID , addressTable.TYPE , personTable.ADDRESSID ], Person.firm.legalName: [firmTable.ID , firmTable.LEGALNAME , personTable.FIRMID ], Person.firstName: [personTable.FIRSTNAME ], Person.lastName: [personTable.LASTNAME ]]', + $lineage); +} + +###Pure +import meta::relational::tests::milestoning::*; +import meta::pure::graphFetch::execution::*; +function <> meta::analytics::lineage::fullAnalyticasTest::testForRelationalGraphFetchWithMilestoning():Boolean[1] +{ + let tree = #{ + Order { + id, + product(%2015-10-16) { + name, + type, + classificationTypeStr + } + } + }#; + let query = {|Order.all()->graphFetch($tree)->serialize($tree)}; + let mapping = meta::relational::tests::milestoning::milestoningmap; + let runtime = meta::external::store::relational::tests::testRuntime(); + + let lineage = meta::analytics::lineage::computeLineage($query, $mapping, $runtime, meta::relational::extension::relationalExtensions()); + + meta::analytics::lineage::assertLineage(['Lambda', 'db_db', 'tb_dbdefaultOrderTable', 'tb_dbdefaultProductClassificationTable', 'tb_dbdefaultProductTable'], + ['Lambda', 'meta::relational::tests::milestoning::Order', 'meta::relational::tests::milestoning::Product', 'pack_meta::relational::tests::milestoning'], + '[Order.id: [OrderTable.id ], Order.product.classificationTypeStr: [OrderTable.prodFk , ProductClassificationTable.type , ProductClassificationTable.type , ProductTable.id , ProductTable.type ], Order.product.name: [OrderTable.prodFk , ProductTable.id , ProductTable.name ], Order.product.type: [OrderTable.prodFk , ProductTable.id , ProductTable.type ]]', + $lineage); +} + +###Pure +import meta::pure::graphFetch::tests::XStore::inMemoryAndRelational::*; +import meta::pure::graphFetch::execution::*; +function <> meta::analytics::lineage::fullAnalyticasTest::testForRelationalGraphFetchWithCrossStore():Boolean[1] +{ + let tree = #{ + Trade { + tradeId, + product { + productName + } + } + }#; + + let query = {|Trade.all()->graphFetch($tree)->serialize($tree)}; + let mapping = meta::pure::graphFetch::tests::XStore::inMemoryAndRelational::crossMapping1; + let runtime = meta::external::store::relational::tests::testRuntime(); + + let lineage = meta::analytics::lineage::computeLineage($query, $mapping, $runtime, meta::relational::extension::relationalExtensions()); + + meta::analytics::lineage::assertLineage(['Lambda', 'db_db1', 'db_db2', 'tb_db1defaulttradeTable', 'tb_db2defaultproductTable'], + ['Lambda', 'meta::pure::graphFetch::tests::XStore::inMemoryAndRelational::Product', 'meta::pure::graphFetch::tests::XStore::inMemoryAndRelational::Trade', 'pack_meta::pure::graphFetch::tests::XStore::inMemoryAndRelational'], + '[Trade.product.productName: [productTable.productName ], Trade.tradeId: [tradeTable.tradeId_source ]]', + $lineage); +} + +###Pure +import meta::pure::graphFetch::execution::*; +import meta::relational::graphFetch::tests::union::rootLevel::*; +function <> meta::analytics::lineage::fullAnalyticasTest::testForRelationalGraphFetchWithRootUnionSameStore():Boolean[1] +{ + let tree = #{ + Trade { + tradeId, + traderKerb, + quantity, + product { + productId, + productName, + description + } + } + }#; + let query = {|Trade.all()->graphFetch($tree)->serialize($tree)}; + let mapping = meta::relational::graphFetch::tests::union::rootLevel::SameStoreMapping; + + let runtime = meta::external::store::relational::tests::testRuntime(); + + let lineage = meta::analytics::lineage::computeLineage($query, $mapping, $runtime, meta::relational::extension::relationalExtensions()); + + meta::analytics::lineage::assertLineage(['Lambda', 'db_dbUnion', 'tb_dbUniondefaultproductTable', 'tb_dbUniondefaultproductTable2', 'tb_dbUniondefaulttradeTable', 'tb_dbUniondefaulttradeTable2'], + ['Lambda', 'meta::relational::graphFetch::tests::union::rootLevel::Product', 'meta::relational::graphFetch::tests::union::rootLevel::Trade', 'pack_meta::relational::graphFetch::tests::union::rootLevel'], + '[Trade.product.description: [productTable.description , productTable.productId , productTable2.description , productTable2.productId , tradeTable.productId , tradeTable2.productId ], Trade.product.productId: [productTable.productId , productTable.productId , productTable2.productId , productTable2.productId , tradeTable.productId , tradeTable2.productId ], Trade.product.productName: [productTable.productId , productTable.productName , productTable2.productId , productTable2.productName , tradeTable.productId , tradeTable2.productId ], Trade.quantity: [tradeTable.quantity , tradeTable2.quantity ], Trade.tradeId: [tradeTable.tradeId , tradeTable2.tradeId ], Trade.traderKerb: [tradeTable.traderKerb , tradeTable2.traderKerb ]]', + $lineage); +} + +function <> meta::analytics::lineage::fullAnalyticasTest::testForRelationalGraphFetchWithRootUnionCrossStore():Boolean[1] +{ + let tree = #{ + Trade { + tradeId, + traderKerb, + quantity, + product { + productId, + productName, + description + } + } + }#; + let query = {|Trade.all()->graphFetch($tree)->serialize($tree)}; + let mapping = meta::relational::graphFetch::tests::union::rootLevel::CrossStoreMapping; + + let runtime = meta::external::store::relational::tests::testRuntime();let lineage = meta::analytics::lineage::computeLineage($query, $mapping, $runtime, meta::relational::extension::relationalExtensions()); + + meta::analytics::lineage::assertLineage(['Lambda', 'db_dbUnion', 'db_dbUnion_XStore', 'tb_dbUnion_XStoredefaultproductTable_XStore', 'tb_dbUnion_XStoredefaulttradeTable_XStore', 'tb_dbUniondefaultproductTable', 'tb_dbUniondefaulttradeTable'], + ['Lambda', 'meta::relational::graphFetch::tests::union::rootLevel::Product', 'meta::relational::graphFetch::tests::union::rootLevel::Trade', 'pack_meta::relational::graphFetch::tests::union::rootLevel'], + '[Trade.product.description: [productTable.description , productTable.productId , productTable_XStore.description , productTable_XStore.productId , tradeTable.productId , tradeTable_XStore.productId ], Trade.product.productId: [productTable.productId , productTable.productId , productTable_XStore.productId , productTable_XStore.productId , tradeTable.productId , tradeTable_XStore.productId ], Trade.product.productName: [productTable.productId , productTable.productName , productTable_XStore.productId , productTable_XStore.productName , tradeTable.productId , tradeTable_XStore.productId ], Trade.quantity: [tradeTable.quantity , tradeTable_XStore.quantity ], Trade.tradeId: [tradeTable.tradeId , tradeTable_XStore.tradeId ], Trade.traderKerb: [tradeTable.traderKerb , tradeTable_XStore.traderKerb ]]', + $lineage); +} + + +###Pure +import meta::relational::tests::model::simple::*; +import meta::pure::graphFetch::execution::*; +function <> meta::analytics::lineage::fullAnalyticasTest::testForRelationalGraphFetchWithPropertyUnion():Boolean[1] +{ + let tree = #{ + Firm{ + legalName, + employees{ + lastName, + address{ + name + } + } + } + }#; + let query = {|Firm.all()->graphFetch($tree)->serialize($tree)}; + + let mapping = meta::relational::graphFetch::tests::union::propertyLevel::Mapping6; + + let runtime = meta::external::store::relational::tests::testRuntime();let lineage = meta::analytics::lineage::computeLineage($query, $mapping, $runtime, meta::relational::extension::relationalExtensions()); + + meta::analytics::lineage::assertLineage(['Lambda', 'db_myDB', 'tb_myDBdefaultAddressSet1', 'tb_myDBdefaultAddressSet2', 'tb_myDBdefaultAddressSet3', 'tb_myDBdefaultFirmSet1', 'tb_myDBdefaultFirmSet2', 'tb_myDBdefaultFirmSet3', 'tb_myDBdefaultPersonSet1', 'tb_myDBdefaultPersonSet2', 'tb_myDBdefaultPersonSet3'], + ['Lambda', 'meta::relational::tests::model::simple::Address', 'meta::relational::tests::model::simple::Firm', 'meta::relational::tests::model::simple::Person', 'pack_meta::relational::tests::model::simple'], + '[Firm.employees.address.name: [AddressSet1.ID , AddressSet1.name , AddressSet2.ID , AddressSet2.name , AddressSet3.ID , AddressSet3.name , FirmSet1.ID , FirmSet2.ID , FirmSet3.ID , PersonSet1.ADDRESSID , PersonSet1.FirmID , PersonSet2.ADDRESSID , PersonSet2.FirmID , PersonSet3.ADDRESSID , PersonSet3.FirmID ], Firm.employees.lastName: [FirmSet1.ID , FirmSet2.ID , FirmSet3.ID , PersonSet1.FirmID , PersonSet1.lastName_s1 , PersonSet2.FirmID , PersonSet2.lastName_s2 , PersonSet3.FirmID , PersonSet3.lastName_s3 ], Firm.legalName: [FirmSet1.name , FirmSet2.name , FirmSet3.name ]]', + $lineage); +}