From a9292a2e8c98b12dee163f306a4e5dd0c4d0d9d6 Mon Sep 17 00:00:00 2001 From: Vijay Garg Date: Wed, 20 Nov 2024 14:49:05 +0530 Subject: [PATCH] Dataquality - support for relation validations (#3257) * [maven-release-plugin] prepare release legend-engine-4.66.0 * [maven-release-plugin] prepare for next development iteration * Dataquality - support for relation validations --------- Co-authored-by: FINOS Administrator <37706051+finos-admin@users.noreply.github.com> Co-authored-by: Mauricio Uyaguari --- .../dataquality/api/DataQualityExecute.java | 6 +- .../api/DataQualityExecuteTrialInput.java | 1 + .../DataQualityCompilerExtension.java | 140 ++- ...TestDataQualityCompilationFromGrammar.java | 27 + .../DataQualityLambdaGenerator.java | 37 +- ...ValidationArtifactGenerationExtension.java | 2 +- .../TestDataQualityLambdaGenerator.java | 2 +- .../from/antlr4/DataQualityLexerGrammar.g4 | 9 + .../from/antlr4/DataQualityParserGrammar.g4 | 41 +- .../DataQualityGrammarParserExtension.java | 2 +- .../grammar/from/DataQualityTreeWalker.java | 87 +- .../DataQualityGrammarComposerExtension.java | 50 +- .../grammar/from/TestDataQualityParsing.java | 74 ++ .../grammar/to/TestDataQualityRoundtrip.java | 17 + .../DataQualityProtocolExtension.java | 6 +- .../DataqualityRelationValidation.java | 38 + .../metamodel/RelationValidation.java | 29 + .../legend-engine-xt-dataquality-pure/pom.xml | 21 +- .../core_dataquality.definition.json | 1 + .../generation/dataquality.pure | 1039 +++++++++-------- .../core_dataquality/metamodel/metamodel.pure | 13 + 21 files changed, 1085 insertions(+), 557 deletions(-) create mode 100644 legend-engine-xts-dataquality/legend-engine-xt-dataquality-protocol/src/main/java/org/finos/legend/engine/protocol/dataquality/metamodel/DataqualityRelationValidation.java create mode 100644 legend-engine-xts-dataquality/legend-engine-xt-dataquality-protocol/src/main/java/org/finos/legend/engine/protocol/dataquality/metamodel/RelationValidation.java diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/main/java/org/finos/legend/engine/language/dataquality/api/DataQualityExecute.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/main/java/org/finos/legend/engine/language/dataquality/api/DataQualityExecute.java index 2030175ce9d..9cb29522240 100644 --- a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/main/java/org/finos/legend/engine/language/dataquality/api/DataQualityExecute.java +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/main/java/org/finos/legend/engine/language/dataquality/api/DataQualityExecute.java @@ -118,7 +118,7 @@ public Response generatePlan(DataQualityExecuteTrialInput dataQualityExecuteInpu // 1. load pure model from PureModelContext PureModel pureModel = this.modelManager.loadModel(dataQualityExecuteInput.model, dataQualityExecuteInput.clientVersion, identity, null); // 2. call DQ PURE func to generate lambda - LambdaFunction dqLambdaFunction = DataQualityLambdaGenerator.generateLambda(pureModel, dataQualityExecuteInput.packagePath, this.extensions); + LambdaFunction dqLambdaFunction = DataQualityLambdaGenerator.generateLambda(pureModel, dataQualityExecuteInput.packagePath, dataQualityExecuteInput.validationName); // 3. Generate Plan from the lambda generated in the previous step SingleExecutionPlan singleExecutionPlan = PlanGenerator.generateExecutionPlan(dqLambdaFunction, null, null, null, pureModel, dataQualityExecuteInput.clientVersion, PlanPlatform.JAVA, null, this.extensions.apply(pureModel), this.transformers);// since lambda has from function we dont need mapping, runtime and context LOGGER.info(new LogInfo(identity.getName(), DataQualityExecutionLoggingEventType.DATAQUALITY_GENERATE_PLAN_END, System.currentTimeMillis() - start).toString()); @@ -146,7 +146,7 @@ public Response execute(@Context HttpServletRequest request, DataQualityExecuteT // 1. load pure model from PureModelContext PureModel pureModel = this.modelManager.loadModel(dataQualityExecuteInput.model, dataQualityExecuteInput.clientVersion, identity, null); // 2. call DQ PURE func to generate lambda - LambdaFunction dqLambdaFunction = DataQualityLambdaGenerator.generateLambdaForTrial(pureModel, dataQualityExecuteInput.packagePath, dataQualityExecuteInput.queryLimit, this.extensions); + LambdaFunction dqLambdaFunction = DataQualityLambdaGenerator.generateLambdaForTrial(pureModel, dataQualityExecuteInput.packagePath, dataQualityExecuteInput.queryLimit, dataQualityExecuteInput.validationName); // 3. Generate Plan from the lambda generated in the previous step SingleExecutionPlan singleExecutionPlan = PlanGenerator.generateExecutionPlan(dqLambdaFunction, null, null, null, pureModel, dataQualityExecuteInput.clientVersion, PlanPlatform.JAVA, null, this.extensions.apply(pureModel), this.transformers);// since lambda has from function we dont need mapping, runtime and context // 4. execute plan @@ -176,7 +176,7 @@ public Response lambda(@Context HttpServletRequest request, DataQualityExecuteTr // 1. load pure model from PureModelContext PureModel pureModel = this.modelManager.loadModel(dataQualityExecuteInput.model, dataQualityExecuteInput.clientVersion, identity, null); // 2. call DQ PURE func to generate lambda - LambdaFunction dqLambdaFunction = DataQualityLambdaGenerator.generateLambdaForTrial(pureModel, dataQualityExecuteInput.packagePath, dataQualityExecuteInput.queryLimit, this.extensions); + LambdaFunction dqLambdaFunction = DataQualityLambdaGenerator.generateLambdaForTrial(pureModel, dataQualityExecuteInput.packagePath, dataQualityExecuteInput.queryLimit, dataQualityExecuteInput.validationName); Lambda lambda = DataQualityLambdaGenerator.transformLambda(dqLambdaFunction, pureModel, this.extensions); return ManageConstantResult.manageResult(identity.getName(), lambda, objectMapper); } diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/main/java/org/finos/legend/engine/language/dataquality/api/DataQualityExecuteTrialInput.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/main/java/org/finos/legend/engine/language/dataquality/api/DataQualityExecuteTrialInput.java index 88d98aa31e2..04f7dcd375c 100644 --- a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/main/java/org/finos/legend/engine/language/dataquality/api/DataQualityExecuteTrialInput.java +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-api/src/main/java/org/finos/legend/engine/language/dataquality/api/DataQualityExecuteTrialInput.java @@ -29,4 +29,5 @@ public class DataQualityExecuteTrialInput public String packagePath; public Integer queryLimit; public List lambdaParameterValues; + public String validationName; } diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/DataQualityCompilerExtension.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/DataQualityCompilerExtension.java index 92aba1ae064..acc0ba53954 100644 --- a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/DataQualityCompilerExtension.java +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-compiler/src/main/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/DataQualityCompilerExtension.java @@ -14,31 +14,38 @@ package org.finos.legend.engine.language.pure.compiler.toPureGraph; +import org.eclipse.collections.api.RichIterable; import org.eclipse.collections.api.factory.Lists; import org.eclipse.collections.api.list.ListIterable; import org.eclipse.collections.api.list.MutableList; import org.eclipse.collections.impl.list.mutable.FastList; import org.eclipse.collections.impl.utility.ListIterate; +import org.finos.legend.engine.language.pure.compiler.Compiler; import org.finos.legend.engine.language.pure.compiler.toPureGraph.extension.CompilerExtension; import org.finos.legend.engine.language.pure.compiler.toPureGraph.extension.Processor; import org.finos.legend.engine.protocol.dataquality.metamodel.DataQuality; import org.finos.legend.engine.protocol.dataquality.metamodel.DataQualityPropertyGraphFetchTree; import org.finos.legend.engine.protocol.dataquality.metamodel.DataQualityRootGraphFetchTree; import org.finos.legend.engine.protocol.dataquality.metamodel.DataSpaceDataQualityExecutionContext; +import org.finos.legend.engine.protocol.dataquality.metamodel.DataqualityRelationValidation; import org.finos.legend.engine.protocol.dataquality.metamodel.MappingAndRuntimeDataQualityExecutionContext; +import org.finos.legend.engine.protocol.dataquality.metamodel.RelationValidation; import org.finos.legend.engine.protocol.pure.v1.model.context.EngineErrorType; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.dataSpace.DataSpace; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.domain.Multiplicity; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mapping.Mapping; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.runtime.PackageableRuntime; import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.ValueSpecification; import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.Variable; import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.classInstance.graph.PropertyGraphFetchTree; import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.classInstance.graph.SubTypeGraphFetchTree; +import org.finos.legend.engine.shared.core.operational.Assert; import org.finos.legend.engine.shared.core.operational.errorManagement.EngineException; import org.finos.legend.pure.generated.Root_meta_external_dataquality_DataQuality; import org.finos.legend.pure.generated.Root_meta_external_dataquality_DataQualityExecutionContext; import org.finos.legend.pure.generated.Root_meta_external_dataquality_DataQualityPropertyGraphFetchTree; import org.finos.legend.pure.generated.Root_meta_external_dataquality_DataQualityPropertyGraphFetchTree_Impl; +import org.finos.legend.pure.generated.Root_meta_external_dataquality_DataQualityRelationValidation_Impl; import org.finos.legend.pure.generated.Root_meta_external_dataquality_DataQualityRootGraphFetchTree; import org.finos.legend.pure.generated.Root_meta_external_dataquality_DataQualityRootGraphFetchTree_Impl; import org.finos.legend.pure.generated.Root_meta_external_dataquality_DataQuality_Impl; @@ -46,6 +53,8 @@ import org.finos.legend.pure.generated.Root_meta_external_dataquality_DataSpaceDataQualityExecutionContext_Impl; import org.finos.legend.pure.generated.Root_meta_external_dataquality_MappingAndRuntimeDataQualityExecutionContext; import org.finos.legend.pure.generated.Root_meta_external_dataquality_MappingAndRuntimeDataQualityExecutionContext_Impl; +import org.finos.legend.pure.generated.Root_meta_external_dataquality_RelationValidation; +import org.finos.legend.pure.generated.Root_meta_external_dataquality_RelationValidation_Impl; import org.finos.legend.pure.generated.Root_meta_pure_metamodel_constraint_Constraint_Impl; import org.finos.legend.pure.generated.Root_meta_pure_metamodel_type_Class_Impl; import org.finos.legend.pure.generated.Root_meta_pure_metamodel_type_generics_GenericType_Impl; @@ -53,16 +62,21 @@ import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.constraint.Constraint; import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.LambdaFunction; import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.function.property.AbstractProperty; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.relation.RelationType; import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.Class; +import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.FunctionType; import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.Type; import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.type.generics.GenericType; import org.finos.legend.pure.m3.navigation._class._Class; import org.finos.legend.pure.m4.coreinstance.CoreInstance; +import org.finos.legend.pure.m4.coreinstance.SourceInformation; import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; public class DataQualityCompilerExtension implements CompilerExtension { @@ -93,34 +107,110 @@ private GenericType genericType(CompileContext context, String _class) public Iterable> getExtraProcessors() { return Lists.fixedSize.of( - Processor.newProcessor( - DataQuality.class, - org.eclipse.collections.impl.factory.Lists.fixedSize.with(PackageableRuntime.class, org.finos.legend.engine.protocol.pure.v1.model.packageableElement.mapping.Mapping.class, org.finos.legend.engine.protocol.pure.v1.model.packageableElement.domain.Class.class, DataSpace.class), - (dataquality, compileContext) -> - { - Root_meta_external_dataquality_DataQuality metamodel = new Root_meta_external_dataquality_DataQuality_Impl<>( - dataquality.name, - SourceInformationHelper.toM3SourceInformation(dataquality.sourceInformation), - compileContext.pureModel.getClass("meta::external::dataquality::DataQuality") - )._classifierGenericType(genericType(compileContext, dataquality.dataQualityRootGraphFetchTree._class)); - return metamodel; - }, - (dataquality, compileContext) -> - { - - }, - (dataquality, compileContext) -> - { - Root_meta_external_dataquality_DataQuality metamodel = (Root_meta_external_dataquality_DataQuality) compileContext.pureModel.getPackageableElement(compileContext.pureModel.buildPackageString(dataquality._package, dataquality.name)); - metamodel._context(buildDataQualityExecutionContext(dataquality, compileContext)) - ._filter(getFilterLambda(dataquality, compileContext)) - ._validationTree(buildRootGraphFetchTree(dataquality.dataQualityRootGraphFetchTree, compileContext, compileContext.pureModel.getClass(dataquality.dataQualityRootGraphFetchTree._class), null, new ProcessingContext("DataQuality"))); - metamodel._validate(true, SourceInformationHelper.toM3SourceInformation(dataquality.sourceInformation), compileContext.getExecutionSupport()); - } - ) + getDataQualityModelContraintProcessor(), + getDataQualityRelationValidationProcessor() ); } + private Processor getDataQualityModelContraintProcessor() + { + return Processor.newProcessor( + DataQuality.class, + org.eclipse.collections.impl.factory.Lists.fixedSize.with(PackageableRuntime.class, Mapping.class, org.finos.legend.engine.protocol.pure.v1.model.packageableElement.domain.Class.class, DataSpace.class), + (dataquality, compileContext) -> + { + Root_meta_external_dataquality_DataQuality metamodel = new Root_meta_external_dataquality_DataQuality_Impl<>( + dataquality.name, + SourceInformationHelper.toM3SourceInformation(dataquality.sourceInformation), + compileContext.pureModel.getClass("meta::external::dataquality::DataQuality") + )._classifierGenericType(genericType(compileContext, dataquality.dataQualityRootGraphFetchTree._class)); + return metamodel; + }, + (dataquality, compileContext) -> + { + + }, + (dataquality, compileContext) -> + { + Root_meta_external_dataquality_DataQuality metamodel = (Root_meta_external_dataquality_DataQuality) compileContext.pureModel.getPackageableElement(compileContext.pureModel.buildPackageString(dataquality._package, dataquality.name)); + metamodel._context(buildDataQualityExecutionContext(dataquality, compileContext)) + ._filter(getFilterLambda(dataquality, compileContext)) + ._validationTree(buildRootGraphFetchTree(dataquality.dataQualityRootGraphFetchTree, compileContext, compileContext.pureModel.getClass(dataquality.dataQualityRootGraphFetchTree._class), null, new ProcessingContext("DataQuality"))); + metamodel._validate(true, SourceInformationHelper.toM3SourceInformation(dataquality.sourceInformation), compileContext.getExecutionSupport()); + } + ); + } + + private Processor getDataQualityRelationValidationProcessor() + { + return Processor.newProcessor( + DataqualityRelationValidation.class, + org.eclipse.collections.impl.factory.Lists.fixedSize.with(PackageableRuntime.class, Mapping.class, org.finos.legend.engine.protocol.pure.v1.model.packageableElement.domain.Class.class, DataSpace.class), + (dataqualityRelationValidation, compileContext) -> + { + Root_meta_external_dataquality_DataQualityRelationValidation_Impl metamodel = new Root_meta_external_dataquality_DataQualityRelationValidation_Impl( + dataqualityRelationValidation.name, + SourceInformationHelper.toM3SourceInformation(dataqualityRelationValidation.sourceInformation), + compileContext.pureModel.getClass("meta::external::dataquality::DataQualityRelationValidation") + ); + return metamodel; + }, + (dataqualityRelationValidation, compileContext) -> + { + + }, + (dataqualityRelationValidation, compileContext) -> + { + Root_meta_external_dataquality_DataQualityRelationValidation_Impl metamodel = (Root_meta_external_dataquality_DataQualityRelationValidation_Impl) compileContext.pureModel.getPackageableElement(compileContext.pureModel.buildPackageString(dataqualityRelationValidation._package, dataqualityRelationValidation.name)); + LambdaFunction relationValidationQuery = buildDataqualityRelationValidationQuery(dataqualityRelationValidation, compileContext); + metamodel._query(relationValidationQuery); + metamodel._validations(buildDataQualityRelationValidations(dataqualityRelationValidation.validations, relationValidationQuery, SourceInformationHelper.toM3SourceInformation(dataqualityRelationValidation.sourceInformation), compileContext)); + metamodel._validate(true, SourceInformationHelper.toM3SourceInformation(dataqualityRelationValidation.sourceInformation), compileContext.getExecutionSupport()); + } + ); + } + + private LambdaFunction buildDataqualityRelationValidationQuery(DataqualityRelationValidation dataqualityRelationValidation, CompileContext compileContext) + { + LambdaFunction relationQuery = HelperValueSpecificationBuilder.buildLambda(dataqualityRelationValidation.query, compileContext); + String relationQueryReturnType = Compiler.getLambdaReturnType(dataqualityRelationValidation.query, compileContext.pureModel); + Assert.assertTrue("meta::pure::metamodel::relation::Relation".equals(relationQueryReturnType), () -> "Relation expected from lambda"); + return relationQuery; + } + + private RichIterable buildDataQualityRelationValidations(List relationValidations, LambdaFunction relationQuery, SourceInformation sourceInformation, CompileContext compileContext) + { + Set allValidationNames = new HashSet<>(); + return Lists.mutable.withAll(relationValidations.stream() + .map(relationValidation -> buildRelationValidation(relationValidation, relationQuery, allValidationNames, sourceInformation, compileContext)) + .collect(Collectors.toList())); + } + + private Root_meta_external_dataquality_RelationValidation_Impl buildRelationValidation(RelationValidation relationalValidation, LambdaFunction relationQuery, Set allValidationNames, SourceInformation sourceInformation, CompileContext compileContext) + { + Assert.assertTrue(relationalValidation.assertion.parameters.size() == 1, () -> "Assertion should have one input param!"); + Root_meta_external_dataquality_RelationValidation_Impl relationValidation = new Root_meta_external_dataquality_RelationValidation_Impl("", sourceInformation, compileContext.pureModel.getClass("meta::external::dataquality::RelationValidation")); + + if (!allValidationNames.add(relationalValidation.name)) + { + throw new EngineException("Duplicated validation '" + relationalValidation.name + "'", EngineErrorType.COMPILATION); + } + relationValidation._name(relationalValidation.name); + relationValidation._description(relationalValidation.description); + Variable assertionInputParam = relationalValidation.assertion.parameters.get(0); + assertionInputParam.multiplicity = Multiplicity.PURE_ONE; + assertionInputParam.genericType = new org.finos.legend.engine.protocol.pure.v1.model.type.GenericType(RelationTypeHelper.convert(fetchQueryRelationType(relationQuery))); + + LambdaFunction assertion = HelperValueSpecificationBuilder.buildLambda(relationalValidation.assertion, compileContext); + relationValidation._assertion(assertion); + return relationValidation; + } + + private RelationType fetchQueryRelationType(LambdaFunction relationQuery) + { + FunctionType functionType = (FunctionType) relationQuery._classifierGenericType()._typeArguments().getLast()._rawType(); + return (RelationType) functionType._returnType()._typeArguments().getLast()._rawType(); + } private LambdaFunction getFilterLambda(DataQuality app, CompileContext compileContext) { diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-compiler/src/test/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/TestDataQualityCompilationFromGrammar.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-compiler/src/test/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/TestDataQualityCompilationFromGrammar.java index 4bc9e60de42..6f2f5616e15 100644 --- a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-compiler/src/test/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/TestDataQualityCompilationFromGrammar.java +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-compiler/src/test/java/org/finos/legend/engine/language/pure/compiler/toPureGraph/TestDataQualityCompilationFromGrammar.java @@ -126,6 +126,23 @@ public void testRootClassEmptyTree() } + @Test + public void testRelationValidation() + { + TestCompilationFromGrammar.TestCompilationFromGrammarTestSuite.test(RELATION_COMPILATION_PREREQUISITE_CODE + + "###DataQualityValidation\n" + + "DataQualityRelationValidation meta::external::dataquality::testvalidation\n" + + "{\n" + + " query: #>{my::Store.myTable}#->select(~name);\n" + + " validations: [\n" + + " {\n" + + " name: 'testValidation';\n" + + " description: 'test validation';\n" + + " assertion: row|$row.name != 'error';\n" + + " }\n" + + " ];\n" + + "}"); + } private static final String COMPILATION_PREREQUISITE_CODE = "###Connection\n" + @@ -231,4 +248,14 @@ public void testRootClassEmptyTree() " defaultExecutionContext: 'default';\n" + "}\n"; + private static final String RELATION_COMPILATION_PREREQUISITE_CODE = "###Relational\n" + + "Database my::Store" + + "(" + + " Table myTable" + + " (" + + " id INT," + + " name VARCHAR(200)" + + " )" + + ")\n"; + } diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/src/main/java/org/finos/legend/engine/generation/dataquality/DataQualityLambdaGenerator.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/src/main/java/org/finos/legend/engine/generation/dataquality/DataQualityLambdaGenerator.java index e96fb0815de..c5203cab09b 100644 --- a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/src/main/java/org/finos/legend/engine/generation/dataquality/DataQualityLambdaGenerator.java +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/src/main/java/org/finos/legend/engine/generation/dataquality/DataQualityLambdaGenerator.java @@ -22,7 +22,9 @@ import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.Lambda; import org.finos.legend.engine.shared.core.ObjectMapperFactory; import org.finos.legend.engine.shared.core.operational.errorManagement.EngineException; +import org.finos.legend.engine.shared.core.operational.errorManagement.ExceptionCategory; import org.finos.legend.pure.generated.Root_meta_external_dataquality_DataQuality; +import org.finos.legend.pure.generated.Root_meta_external_dataquality_DataQualityRelationValidation; import org.finos.legend.pure.generated.Root_meta_pure_extension_Extension; import org.finos.legend.pure.generated.core_dataquality_generation_dataquality; import org.finos.legend.pure.generated.core_external_format_json_toJSON; @@ -39,27 +41,42 @@ public class DataQualityLambdaGenerator { public static final int DEFAULT_QUERY_LIMIT = 100; - public static LambdaFunction generateLambda(PureModel pureModel, String qualifiedPath, Function> extensions) + public static LambdaFunction generateLambda(PureModel pureModel, String qualifiedPath, String validationName) { PackageableElement packageableElement = pureModel.getPackageableElement(qualifiedPath); - return generateLambda(pureModel, packageableElement, extensions); + return generateLambda(pureModel, packageableElement, validationName); } - public static LambdaFunction generateLambda(PureModel pureModel, PackageableElement packageableElement, Function> extensions) + public static LambdaFunction generateLambda(PureModel pureModel, PackageableElement packageableElement, String validationName) { - return core_dataquality_generation_dataquality.Root_meta_external_dataquality_generateDataQualityQuery_DataQuality_1__Integer_MANY__LambdaFunction_1_((Root_meta_external_dataquality_DataQuality) packageableElement, Lists.immutable.empty(), pureModel.getExecutionSupport()); + if (packageableElement instanceof Root_meta_external_dataquality_DataQuality) + { + return core_dataquality_generation_dataquality.Root_meta_external_dataquality_generateDataQualityQuery_DataQuality_1__Integer_MANY__LambdaFunction_1_((Root_meta_external_dataquality_DataQuality)packageableElement, Lists.immutable.empty(), pureModel.getExecutionSupport()); + } + else if (packageableElement instanceof Root_meta_external_dataquality_DataQualityRelationValidation) + { + return (LambdaFunction) core_dataquality_generation_dataquality.Root_meta_external_dataquality_generateDataqualityRelationValidationLambda_DataQualityRelationValidation_1__String_1__LambdaFunction_1_((Root_meta_external_dataquality_DataQualityRelationValidation)packageableElement, validationName, pureModel.getExecutionSupport()); + } + throw new EngineException("Unsupported Dataquality element! " + packageableElement.getClass().getSimpleName(), ExceptionCategory.USER_EXECUTION_ERROR); } - public static LambdaFunction generateLambdaForTrial(PureModel pureModel, String qualifiedPath, Integer queryLimit, Function> extensions) + public static LambdaFunction generateLambdaForTrial(PureModel pureModel, String qualifiedPath, Integer queryLimit, String validationName) { PackageableElement packageableElement = pureModel.getPackageableElement(qualifiedPath); - int trialQueryLimit = DEFAULT_QUERY_LIMIT; - if (Objects.nonNull(queryLimit)) + if (packageableElement instanceof Root_meta_external_dataquality_DataQuality) + { + int trialQueryLimit = DEFAULT_QUERY_LIMIT; + if (Objects.nonNull(queryLimit)) + { + trialQueryLimit = queryLimit; + } + return core_dataquality_generation_dataquality.Root_meta_external_dataquality_generateDataQualityQuery_DataQuality_1__Integer_MANY__LambdaFunction_1_((Root_meta_external_dataquality_DataQuality)packageableElement, Lists.immutable.of((long)trialQueryLimit), pureModel.getExecutionSupport()); + } + else if (packageableElement instanceof Root_meta_external_dataquality_DataQualityRelationValidation) { - trialQueryLimit = queryLimit; + return (LambdaFunction) core_dataquality_generation_dataquality.Root_meta_external_dataquality_generateDataqualityRelationValidationLambda_DataQualityRelationValidation_1__String_1__LambdaFunction_1_((Root_meta_external_dataquality_DataQualityRelationValidation)packageableElement, validationName, pureModel.getExecutionSupport()); } - - return core_dataquality_generation_dataquality.Root_meta_external_dataquality_generateDataQualityQuery_DataQuality_1__Integer_MANY__LambdaFunction_1_((Root_meta_external_dataquality_DataQuality)packageableElement, Lists.immutable.of((long)trialQueryLimit), pureModel.getExecutionSupport()); + throw new EngineException("Unsupported Dataquality element! " + packageableElement.getClass().getSimpleName(), ExceptionCategory.USER_EXECUTION_ERROR); } public static Lambda transformLambda(LambdaFunction lambda, PureModel pureModel, Function> extensions) diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/src/main/java/org/finos/legend/engine/generation/dataquality/DataQualityValidationArtifactGenerationExtension.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/src/main/java/org/finos/legend/engine/generation/dataquality/DataQualityValidationArtifactGenerationExtension.java index 6d9e87aa439..14348a67082 100644 --- a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/src/main/java/org/finos/legend/engine/generation/dataquality/DataQualityValidationArtifactGenerationExtension.java +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/src/main/java/org/finos/legend/engine/generation/dataquality/DataQualityValidationArtifactGenerationExtension.java @@ -80,7 +80,7 @@ public List generate(PackageableElement packageableElement, PureModel Function> routerExtensions = (PureModel p) -> PureCoreExtensionLoader.extensions().flatCollect(e -> e.extraPureCoreExtensions(p.getExecutionSupport())); // 1. call DQ PURE func to generate lambda - LambdaFunction dqLambdaFunction = DataQualityLambdaGenerator.generateLambda(pureModel, packageableElement, routerExtensions); + LambdaFunction dqLambdaFunction = DataQualityLambdaGenerator.generateLambda(pureModel, packageableElement, null); // 2. Generate Plan from the lambda generated in the previous step SingleExecutionPlan singleExecutionPlan = PlanGenerator.generateExecutionPlan(dqLambdaFunction, null, null, null, pureModel, clientVersion, PlanPlatform.JAVA, null, routerExtensions.apply(pureModel), LegendPlanTransformers.transformers);// since lambda has from function we dont need mapping, runtime and context diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/src/test/java/org/finos/legend/engine/generation/dataquality/TestDataQualityLambdaGenerator.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/src/test/java/org/finos/legend/engine/generation/dataquality/TestDataQualityLambdaGenerator.java index 3973d45560c..625f4028d3b 100644 --- a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/src/test/java/org/finos/legend/engine/generation/dataquality/TestDataQualityLambdaGenerator.java +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-generation/src/test/java/org/finos/legend/engine/generation/dataquality/TestDataQualityLambdaGenerator.java @@ -92,7 +92,7 @@ private String generateLambda(String modelString) PureModelContextData modelData = loadWithModel(modelString); PureModel model = Compiler.compile(modelData, DeploymentMode.TEST_IGNORE_FUNCTION_MATCH, Identity.getAnonymousIdentity().getName()); Function> routerExtensions = extensions(); - LambdaFunction lambdaFunction = DataQualityLambdaGenerator.generateLambda(model, "meta::dataquality::Validation", routerExtensions); + LambdaFunction lambdaFunction = DataQualityLambdaGenerator.generateLambda(model, "meta::dataquality::Validation", null); return DataQualityLambdaGenerator.transformLambdaAsJson(lambdaFunction, model, routerExtensions); } diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/DataQualityLexerGrammar.g4 b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/DataQualityLexerGrammar.g4 index b557cbbc04f..005d8d692cc 100644 --- a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/DataQualityLexerGrammar.g4 +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/DataQualityLexerGrammar.g4 @@ -16,3 +16,12 @@ GRAPH_END: ']$'; // ---------------------------------- BUILDING BLOCK -------------------------------------- SUBTYPE_START: '->subType(@'; + + +//----------------------- RELATION VALIDATION -------------------------------------------------- +DATAQUALITYRELATIONVALIDATION: 'DataQualityRelationValidation'; +RELATION_FUNCTION: 'query'; +VALIDATIONS: 'validations'; +VALIDATION_NAME: 'name'; +VALIDATION_DESCRIPTION: 'description'; +VALIDATION_ASSERTION: 'assertion'; \ No newline at end of file diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/DataQualityParserGrammar.g4 b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/DataQualityParserGrammar.g4 index afbbc724583..c6b35250e7f 100644 --- a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/DataQualityParserGrammar.g4 +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/main/antlr4/org/finos/legend/engine/language/pure/grammar/from/antlr4/DataQualityParserGrammar.g4 @@ -17,6 +17,12 @@ identifier: VALID_STRING | STRING | FROM_MAPPING_AND_RUNTIME | DQCONTEXT | FILTER + | DATAQUALITYRELATIONVALIDATION + | RELATION_FUNCTION + | VALIDATIONS + | VALIDATION_NAME + | VALIDATION_DESCRIPTION + | VALIDATION_ASSERTION ; @@ -25,7 +31,11 @@ identifier: VALID_STRING | STRING definition: (validationDefinition)* EOF ; -validationDefinition: DATAQUALITYVALIDATION stereotypes? taggedValues? qualifiedName +validationDefinition: classValidationDefinition + | relationValidationDefinition +; + +classValidationDefinition: DATAQUALITYVALIDATION stereotypes? taggedValues? qualifiedName BRACE_OPEN ( dqContext @@ -97,4 +107,33 @@ constraintList: LESS_THAN GREATER_THAN ; dqConstraintName: identifier +; + +// --------------------------- Relation Definition ---------------------------------------------------------- +relationValidationDefinition: DATAQUALITYRELATIONVALIDATION stereotypes? taggedValues? qualifiedName + BRACE_OPEN + ( + relationFunc + | validations + )* + BRACE_CLOSE +; + +relationFunc: RELATION_FUNCTION COLON combinedExpression SEMI_COLON +; +validations: VALIDATIONS COLON BRACKET_OPEN validation(COMMA validation)* BRACKET_CLOSE SEMI_COLON +; +validation: BRACE_OPEN + ( + validationName + | validationDesc + | validationAssertion + )* + BRACE_CLOSE +; +validationName: VALIDATION_NAME COLON STRING SEMI_COLON +; +validationDesc: VALIDATION_DESCRIPTION COLON STRING SEMI_COLON +; +validationAssertion: VALIDATION_ASSERTION COLON combinedExpression SEMI_COLON ; \ No newline at end of file diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/main/java/org/finos/legend/engine/language/dataquality/grammar/from/DataQualityGrammarParserExtension.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/main/java/org/finos/legend/engine/language/dataquality/grammar/from/DataQualityGrammarParserExtension.java index 97daa25377c..b67b1e84ff9 100644 --- a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/main/java/org/finos/legend/engine/language/dataquality/grammar/from/DataQualityGrammarParserExtension.java +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/main/java/org/finos/legend/engine/language/dataquality/grammar/from/DataQualityGrammarParserExtension.java @@ -56,7 +56,7 @@ private static Section parseSection(SectionSourceCode sectionSourceCode, Consume section.parserName = sectionSourceCode.sectionType; section.sourceInformation = parserInfo.sourceInformation; - DataQualityTreeWalker walker = new DataQualityTreeWalker(parserInfo.input, parserInfo.walkerSourceInformation, elementConsumer, section); + DataQualityTreeWalker walker = new DataQualityTreeWalker(parserInfo.input, parserInfo.walkerSourceInformation, elementConsumer, section, context); walker.visit((DataQualityParserGrammar.DefinitionContext) parserInfo.rootContext); return section; diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/main/java/org/finos/legend/engine/language/dataquality/grammar/from/DataQualityTreeWalker.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/main/java/org/finos/legend/engine/language/dataquality/grammar/from/DataQualityTreeWalker.java index 962f237d994..0fd2e5a1a20 100644 --- a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/main/java/org/finos/legend/engine/language/dataquality/grammar/from/DataQualityTreeWalker.java +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/main/java/org/finos/legend/engine/language/dataquality/grammar/from/DataQualityTreeWalker.java @@ -19,6 +19,7 @@ import org.eclipse.collections.impl.factory.Lists; import org.eclipse.collections.impl.utility.ListIterate; import org.finos.legend.engine.language.pure.grammar.from.ParseTreeWalkerSourceInformation; +import org.finos.legend.engine.language.pure.grammar.from.PureGrammarParserContext; import org.finos.legend.engine.language.pure.grammar.from.PureGrammarParserUtility; import org.finos.legend.engine.language.pure.grammar.from.antlr4.DataQualityParserGrammar; import org.finos.legend.engine.language.pure.grammar.from.domain.DomainParser; @@ -27,8 +28,11 @@ import org.finos.legend.engine.protocol.dataquality.metamodel.DataQualityPropertyGraphFetchTree; import org.finos.legend.engine.protocol.dataquality.metamodel.DataQualityRootGraphFetchTree; import org.finos.legend.engine.protocol.dataquality.metamodel.DataSpaceDataQualityExecutionContext; +import org.finos.legend.engine.protocol.dataquality.metamodel.DataqualityRelationValidation; import org.finos.legend.engine.protocol.dataquality.metamodel.MappingAndRuntimeDataQualityExecutionContext; +import org.finos.legend.engine.protocol.dataquality.metamodel.RelationValidation; import org.finos.legend.engine.protocol.pure.v1.model.SourceInformation; +import org.finos.legend.engine.protocol.pure.v1.model.context.EngineErrorType; import org.finos.legend.engine.protocol.pure.v1.model.context.PackageableElementPointer; import org.finos.legend.engine.protocol.pure.v1.model.context.PackageableElementType; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.PackageableElement; @@ -41,12 +45,14 @@ import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.classInstance.graph.GraphFetchTree; import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.classInstance.graph.PropertyGraphFetchTree; import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.classInstance.graph.SubTypeGraphFetchTree; +import org.finos.legend.engine.shared.core.operational.errorManagement.EngineException; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.function.Consumer; +import java.util.stream.Collectors; public class DataQualityTreeWalker { @@ -54,13 +60,15 @@ public class DataQualityTreeWalker private final ParseTreeWalkerSourceInformation walkerSourceInformation; private final Consumer elementConsumer; private final DefaultCodeSection section; + private final PureGrammarParserContext parserContext; - public DataQualityTreeWalker(CharStream input, ParseTreeWalkerSourceInformation walkerSourceInformation, Consumer elementConsumer, DefaultCodeSection section) + public DataQualityTreeWalker(CharStream input, ParseTreeWalkerSourceInformation walkerSourceInformation, Consumer elementConsumer, DefaultCodeSection section, PureGrammarParserContext parserContext) { this.input = input; this.walkerSourceInformation = walkerSourceInformation; this.elementConsumer = elementConsumer; this.section = section; + this.parserContext = parserContext; } public void visit(DataQualityParserGrammar.DefinitionContext definitionContext) @@ -71,7 +79,20 @@ public void visit(DataQualityParserGrammar.DefinitionContext definitionContext) } } - private DataQuality visitDataQualityValidation(DataQualityParserGrammar.ValidationDefinitionContext validationDefinitionContext) + private PackageableElement visitDataQualityValidation(DataQualityParserGrammar.ValidationDefinitionContext validationDefinitionContext) + { + if (validationDefinitionContext.classValidationDefinition() != null) + { + return visitClassValidation(validationDefinitionContext.classValidationDefinition()); + } + if (validationDefinitionContext.relationValidationDefinition() != null) + { + return visitRelationValidation(validationDefinitionContext.relationValidationDefinition()); + } + throw new EngineException("Unsupported syntax", this.walkerSourceInformation.getSourceInformation(validationDefinitionContext), EngineErrorType.PARSER); + } + + private DataQuality visitClassValidation(DataQualityParserGrammar.ClassValidationDefinitionContext validationDefinitionContext) { DataQuality dataQuality = new DataQuality(); dataQuality.name = PureGrammarParserUtility.fromIdentifier(validationDefinitionContext.qualifiedName().identifier()); @@ -262,7 +283,7 @@ private Lambda visitLambda(DataQualityParserGrammar.CombinedExpressionContext co int columnOffset = (startLine == 1 ? walkerSourceInformation.getColumnOffset() : 0) + combinedExpressionContext.getStart().getCharPositionInLine(); ParseTreeWalkerSourceInformation combineExpressionSourceInformation = new ParseTreeWalkerSourceInformation.Builder(walkerSourceInformation.getSourceId(), lineOffset, columnOffset).withReturnSourceInfo(this.walkerSourceInformation.getReturnSourceInfo()).build(); String lambdaString = this.input.getText(new Interval(combinedExpressionContext.start.getStartIndex(), combinedExpressionContext.stop.getStopIndex())); - ValueSpecification valueSpecification = parser.parseCombinedExpression(lambdaString, combineExpressionSourceInformation, null); + ValueSpecification valueSpecification = parser.parseCombinedExpression(lambdaString, combineExpressionSourceInformation, this.parserContext); if (valueSpecification instanceof Lambda) { return (Lambda) valueSpecification; @@ -275,4 +296,64 @@ private Lambda visitLambda(DataQualityParserGrammar.CombinedExpressionContext co lambda.parameters = new ArrayList<>(); return lambda; } + + private DataqualityRelationValidation visitRelationValidation(DataQualityParserGrammar.RelationValidationDefinitionContext relationValidationDefinitionContext) + { + DataqualityRelationValidation dataqualityRelationValidation = new DataqualityRelationValidation(); + dataqualityRelationValidation.name = PureGrammarParserUtility.fromIdentifier(relationValidationDefinitionContext.qualifiedName().identifier()); + dataqualityRelationValidation._package = relationValidationDefinitionContext.qualifiedName().packagePath() == null ? "" : PureGrammarParserUtility.fromPath(relationValidationDefinitionContext.qualifiedName().packagePath().identifier()); + dataqualityRelationValidation.sourceInformation = walkerSourceInformation.getSourceInformation(relationValidationDefinitionContext); + dataqualityRelationValidation.stereotypes = relationValidationDefinitionContext.stereotypes() == null ? Lists.mutable.empty() : this.visitStereotypes(relationValidationDefinitionContext.stereotypes()); + dataqualityRelationValidation.taggedValues = relationValidationDefinitionContext.taggedValues() == null ? Lists.mutable.empty() : this.visitTaggedValues(relationValidationDefinitionContext.taggedValues()); + dataqualityRelationValidation.sourceInformation = walkerSourceInformation.getSourceInformation(relationValidationDefinitionContext); + + // query + DataQualityParserGrammar.RelationFuncContext relationFuncContext = PureGrammarParserUtility.validateAndExtractRequiredField(relationValidationDefinitionContext.relationFunc(), + "query", + dataqualityRelationValidation.sourceInformation); + dataqualityRelationValidation.query = visitLambda(relationFuncContext.combinedExpression()); + + // relation validations + DataQualityParserGrammar.ValidationsContext validationsContext = PureGrammarParserUtility.validateAndExtractRequiredField(relationValidationDefinitionContext.validations(), + "validations", + dataqualityRelationValidation.sourceInformation); + dataqualityRelationValidation.validations = visitValidations(validationsContext, dataqualityRelationValidation.sourceInformation); + + return dataqualityRelationValidation; + } + + private List visitValidations(DataQualityParserGrammar.ValidationsContext validationContexts, SourceInformation sourceInformation) + { + if (Objects.isNull(validationContexts.validation())) + { + return Collections.emptyList(); + } + return validationContexts.validation().stream() + .map(validationContext -> this.visitValidation(validationContext, sourceInformation)) + .collect(Collectors.toList()); + } + + private RelationValidation visitValidation(DataQualityParserGrammar.ValidationContext validationContext, SourceInformation sourceInformation) + { + RelationValidation relationValidation = new RelationValidation(); + DataQualityParserGrammar.ValidationNameContext validationNameContext = PureGrammarParserUtility.validateAndExtractRequiredField(validationContext.validationName(), + "name", + sourceInformation); + relationValidation.name = PureGrammarParserUtility.fromGrammarString(validationNameContext.STRING().getText(), true); + + DataQualityParserGrammar.ValidationDescContext validationDescContext = PureGrammarParserUtility.validateAndExtractOptionalField(validationContext.validationDesc(), + "description", + sourceInformation); + if (Objects.nonNull(validationDescContext)) + { + relationValidation.description = PureGrammarParserUtility.fromGrammarString(validationDescContext.STRING().getText(), true); + } + + DataQualityParserGrammar.ValidationAssertionContext validationAssertionContext = PureGrammarParserUtility.validateAndExtractRequiredField(validationContext.validationAssertion(), + "assertion", + sourceInformation); + relationValidation.assertion = this.visitLambda(validationAssertionContext.combinedExpression()); + + return relationValidation; + } } diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/main/java/org/finos/legend/engine/language/dataquality/grammar/to/DataQualityGrammarComposerExtension.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/main/java/org/finos/legend/engine/language/dataquality/grammar/to/DataQualityGrammarComposerExtension.java index 51e18bf71d7..56d48bfec68 100644 --- a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/main/java/org/finos/legend/engine/language/dataquality/grammar/to/DataQualityGrammarComposerExtension.java +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/main/java/org/finos/legend/engine/language/dataquality/grammar/to/DataQualityGrammarComposerExtension.java @@ -29,7 +29,9 @@ import org.finos.legend.engine.protocol.dataquality.metamodel.DataQualityPropertyGraphFetchTree; import org.finos.legend.engine.protocol.dataquality.metamodel.DataQualityRootGraphFetchTree; import org.finos.legend.engine.protocol.dataquality.metamodel.DataSpaceDataQualityExecutionContext; +import org.finos.legend.engine.protocol.dataquality.metamodel.DataqualityRelationValidation; import org.finos.legend.engine.protocol.dataquality.metamodel.MappingAndRuntimeDataQualityExecutionContext; +import org.finos.legend.engine.protocol.dataquality.metamodel.RelationValidation; import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.PackageableElement; import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.classInstance.graph.SubTypeGraphFetchTree; import org.finos.legend.engine.shared.core.api.grammar.RenderStyle; @@ -61,6 +63,10 @@ private static String renderElement(PackageableElement element, PureGrammarCompo { return renderDataQuality((DataQuality) element, context); } + if (element instanceof DataqualityRelationValidation) + { + return renderDataQualityRelationValidation((DataqualityRelationValidation) element, context); + } return "/* Can't transform element '" + element.getPath() + "' in this section */"; } @@ -76,6 +82,16 @@ private static String renderDataQuality(DataQuality dataQuality, PureGrammarComp "}"; } + private static String renderDataQualityRelationValidation(DataqualityRelationValidation dataqualityRelationValidation, PureGrammarComposerContext context) + { + String packageName = dataqualityRelationValidation._package == null || dataqualityRelationValidation._package.isEmpty() ? dataqualityRelationValidation.name : dataqualityRelationValidation._package + "::" + dataqualityRelationValidation.name; + return "DataQualityRelationValidation " + renderAnnotations(dataqualityRelationValidation.stereotypes, dataqualityRelationValidation.taggedValues) + packageName + "\n" + + "{\n" + + " query: " + renderRelationQuery(dataqualityRelationValidation, context) + + " validations: " + renderValidations(dataqualityRelationValidation.validations, context) + + "}"; + } + private static String renderLambda(DataQuality dataQuality, PureGrammarComposerContext context) { if (Objects.isNull(dataQuality.filter)) @@ -222,6 +238,10 @@ public List, PureGrammarComposerContext, Stri { return renderDataQuality((DataQuality) element, context); } + if (element instanceof DataqualityRelationValidation) + { + return renderDataQualityRelationValidation((DataqualityRelationValidation) element, context); + } return "/* Can't transform element '" + element.getPath() + "' in this section */"; }).makeString("\n\n"); }); @@ -232,10 +252,38 @@ public List, PureGrammarComposerContext, List { return Collections.singletonList((elements, context, composedSections) -> // TODO: use context for render style etc - dont hardcode { - MutableList composableElements = Iterate.select(elements, e -> (e instanceof DataQuality), Lists.mutable.empty()); + MutableList composableElements = Iterate.select(elements, e -> (e instanceof DataQuality || e instanceof DataqualityRelationValidation), Lists.mutable.empty()); return composableElements.isEmpty() ? null : new PureFreeSectionGrammarComposerResult(composableElements.asLazy().collect(element -> renderElement(element, context)).makeString("###" + DataQualityGrammarParserExtension.NAME + "\n", "\n\n", ""), composableElements); }); } + + private static String renderRelationQuery(DataqualityRelationValidation dataqualityRelationValidation, PureGrammarComposerContext context) + { + return dataqualityRelationValidation.query.accept(DEPRECATED_PureGrammarComposerCore.Builder.newInstance(context).build()) + ";\n"; + } + + private static String renderValidations(List relationValidations, PureGrammarComposerContext context) + { + return "[\n" + + relationValidations.stream().map(val -> renderValidation(val, context)).collect(Collectors.joining(",\n", "", "\n")) + + " ];\n"; + } + + private static String renderValidation(RelationValidation relationValidation, PureGrammarComposerContext context) + { + return + " {\n" + + " name: '" + relationValidation.name + "';\n" + + (Objects.nonNull(relationValidation.description) ? + " description: '" + relationValidation.description + "';\n" : "") + + " assertion: " + renderAssertion(relationValidation, context) + ";\n" + + " }"; + } + + private static String renderAssertion(RelationValidation relationValidation, PureGrammarComposerContext context) + { + return relationValidation.assertion.accept(DEPRECATED_PureGrammarComposerCore.Builder.newInstance(context).build()); + } } diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/test/java/org/finos/legend/engine/language/dataquality/grammar/from/TestDataQualityParsing.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/test/java/org/finos/legend/engine/language/dataquality/grammar/from/TestDataQualityParsing.java index 5b8e2f2c35a..496be5ac0b6 100644 --- a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/test/java/org/finos/legend/engine/language/dataquality/grammar/from/TestDataQualityParsing.java +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/test/java/org/finos/legend/engine/language/dataquality/grammar/from/TestDataQualityParsing.java @@ -198,6 +198,80 @@ public void testEdgeScenarios() } + @Test + public void testParserForValidRelationValidationGrammar() + { + test("###DataQualityValidation\n" + + "DataQualityRelationValidation meta::external::dataquality::testvalidation\n" + + "{\n" + + " query: #>{my::Store.myTable}#->filter(c|$c.name == 'ok');\n" + + " validations: [\n" + + " {\n" + + " name: 'testValidation';\n" + + " description: 'test validation';\n" + + " assertion: row|$row.name != 'error';\n" + + " }\n" + + " ];\n" + + "}"); + + // with description as optional + test("###DataQualityValidation\n" + + "DataQualityRelationValidation meta::external::dataquality::testvalidation\n" + + "{\n" + + " query: #>{my::Store.myTable}#->filter(c|$c.name == 'ok');\n" + + " validations: [\n" + + " {\n" + + " name: 'testValidation';\n" + + " assertion: row|$row.name != 'error';\n" + + " }\n" + + " ];\n" + + "}"); + } + + @Test + public void testParserErrorForMandatoryFields_relationalValidations() + { + test("###DataQualityValidation\n" + + "DataQualityRelationValidation meta::external::dataquality::testvalidation\n" + + "{\n" + + " validations: [\n" + + " {\n" + + " name: 'testValidation';\n" + + " description: 'test validation';\n" + + " assertion: row|$row.name != 'error';\n" + + " }\n" + + " ];\n" + + "}", "PARSER error at [2:1-11:1]: Field 'query' is required"); + + test("###DataQualityValidation\n" + + "DataQualityRelationValidation meta::external::dataquality::testvalidation\n" + + "{\n" + + " query: #>{my::Store.myTable}#->filter(c|$c.name == 'ok');\n" + + "}", "PARSER error at [2:1-5:1]: Field 'validations' is required"); + + test("###DataQualityValidation\n" + + "DataQualityRelationValidation meta::external::dataquality::testvalidation\n" + + "{\n" + + " query: #>{my::Store.myTable}#->filter(c|$c.name == 'ok');\n" + + " validations: [\n" + + " {\n" + + " description: 'test validation';\n" + + " assertion: row|$row.name != 'error';\n" + + " }\n" + + " ];\n" + + "}", "PARSER error at [2:1-11:1]: Field 'name' is required"); + + test("###DataQualityValidation\n" + + "DataQualityRelationValidation meta::external::dataquality::testvalidation\n" + + "{\n" + + " query: #>{my::Store.myTable}#->filter(c|$c.name == 'ok');\n" + + " validations: [\n" + + " {\n" + + " name: 'testValidation';\n" + + " }\n" + + " ];\n" + + "}", "PARSER error at [2:1-10:1]: Field 'assertion' is required"); + } } diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/test/java/org/finos/legend/engine/language/dataquality/grammar/to/TestDataQualityRoundtrip.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/test/java/org/finos/legend/engine/language/dataquality/grammar/to/TestDataQualityRoundtrip.java index 614c585ffd4..fe1dd5e3944 100644 --- a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/test/java/org/finos/legend/engine/language/dataquality/grammar/to/TestDataQualityRoundtrip.java +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-grammar/src/test/java/org/finos/legend/engine/language/dataquality/grammar/to/TestDataQualityRoundtrip.java @@ -119,4 +119,21 @@ public void testParserForValidGrammar_whiteSpaceCharsInConstraintNames() "}\n"); } + @Test + public void testRelationalValidation() + { + test("###DataQualityValidation\n" + + "DataQualityRelationValidation meta::external::dataquality::testvalidation\n" + + "{\n" + + " query: |#>{my::Store.myTable}#->filter(c|$c.name == 'ok');\n" + + " validations: [\n" + + " {\n" + + " name: 'testValidation';\n" + + " description: 'test validation';\n" + + " assertion: row|$row.name != 'error';\n" + + " }\n" + + " ];\n" + + "}\n"); + } + } diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-protocol/src/main/java/org/finos/legend/engine/protocol/dataquality/metamodel/DataQualityProtocolExtension.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-protocol/src/main/java/org/finos/legend/engine/protocol/dataquality/metamodel/DataQualityProtocolExtension.java index da899dabb21..3578d95a396 100644 --- a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-protocol/src/main/java/org/finos/legend/engine/protocol/dataquality/metamodel/DataQualityProtocolExtension.java +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-protocol/src/main/java/org/finos/legend/engine/protocol/dataquality/metamodel/DataQualityProtocolExtension.java @@ -49,6 +49,9 @@ public List>>> getExtraProtocolSubTypeInfo .build(), ProtocolSubTypeInfo.newBuilder(PropertyGraphFetchTree.class) .withSubtype(DataQualityPropertyGraphFetchTree.class, "dataQualityPropertyGraphFetchTree") + .build(), + ProtocolSubTypeInfo.newBuilder(PackageableElement.class) + .withSubtype(DataqualityRelationValidation.class, "dataqualityRelationValidation") .build() )); } @@ -56,6 +59,7 @@ public List>>> getExtraProtocolSubTypeInfo @Override public Map, String> getExtraProtocolToClassifierPathMap() { - return Maps.mutable.with(DataQuality.class, "meta::external::dataquality::DataQuality"); // TODO: check classifier path + return Maps.mutable.with(DataQuality.class, "meta::external::dataquality::DataQuality", + DataqualityRelationValidation.class, "meta::external::dataquality::DataQualityRelationValidation"); } } diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-protocol/src/main/java/org/finos/legend/engine/protocol/dataquality/metamodel/DataqualityRelationValidation.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-protocol/src/main/java/org/finos/legend/engine/protocol/dataquality/metamodel/DataqualityRelationValidation.java new file mode 100644 index 00000000000..94d7ee083f9 --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-protocol/src/main/java/org/finos/legend/engine/protocol/dataquality/metamodel/DataqualityRelationValidation.java @@ -0,0 +1,38 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.protocol.dataquality.metamodel; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.PackageableElement; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.domain.StereotypePtr; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.domain.TaggedValue; +import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.Lambda; + +import java.util.Collections; +import java.util.List; + + +//------------------------------------------------------------ +// Should be generated out of the Pure protocol specification +//------------------------------------------------------------ +@JsonIgnoreProperties(ignoreUnknown = true) +public class DataqualityRelationValidation extends PackageableElement +{ + public List stereotypes = Collections.emptyList(); + public List taggedValues = Collections.emptyList(); + + public Lambda query; + public List validations = Collections.emptyList(); +} diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-protocol/src/main/java/org/finos/legend/engine/protocol/dataquality/metamodel/RelationValidation.java b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-protocol/src/main/java/org/finos/legend/engine/protocol/dataquality/metamodel/RelationValidation.java new file mode 100644 index 00000000000..77623e0c546 --- /dev/null +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-protocol/src/main/java/org/finos/legend/engine/protocol/dataquality/metamodel/RelationValidation.java @@ -0,0 +1,29 @@ +// Copyright 2024 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.protocol.dataquality.metamodel; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.finos.legend.engine.protocol.pure.v1.model.valueSpecification.raw.Lambda; + +//------------------------------------------------------------ +// Should be generated out of the Pure protocol specification +//------------------------------------------------------------ +@JsonIgnoreProperties(ignoreUnknown = true) +public class RelationValidation +{ + public String name; + public String description; + public Lambda assertion; +} diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/pom.xml b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/pom.xml index 239db0daed4..4cc8b91cb99 100644 --- a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/pom.xml +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/pom.xml @@ -75,6 +75,11 @@ legend-engine-xt-relationalStore-core-pure ${project.version} + + org.finos.legend.engine + legend-engine-pure-functions-relation-pure + ${project.version} + @@ -138,6 +143,11 @@ legend-engine-xt-relationalStore-core-pure ${project.version} + + org.finos.legend.engine + legend-engine-pure-functions-relation-pure + ${project.version} + @@ -216,11 +226,12 @@ org.finos.legend.engine legend-engine-pure-platform-java - - - - - + + org.finos.legend.engine + legend-engine-pure-functions-relation-pure + ${project.version} + runtime + com.fasterxml.jackson.core diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/src/main/resources/core_dataquality.definition.json b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/src/main/resources/core_dataquality.definition.json index 76b939b78f1..c9a4a99ba87 100644 --- a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/src/main/resources/core_dataquality.definition.json +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/src/main/resources/core_dataquality.definition.json @@ -6,6 +6,7 @@ "core_relational", "core_data_space_metamodel", "core_functions_unclassified", + "core_functions_relation", "platform", "platform_dsl_store", "platform_dsl_graph", diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/src/main/resources/core_dataquality/generation/dataquality.pure b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/src/main/resources/core_dataquality/generation/dataquality.pure index 0683e1e5014..6138ae59fd5 100644 --- a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/src/main/resources/core_dataquality/generation/dataquality.pure +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/src/main/resources/core_dataquality/generation/dataquality.pure @@ -1,505 +1,534 @@ -import meta::pure::dataQuality::*; -import meta::pure::metamodel::serialization::grammar::*; -import meta::pure::lineage::scanProperties::*; -import meta::external::dataquality::*; -import meta::pure::graphFetch::*; -import meta::pure::functions::collection::*; -import meta::pure::functions::meta::*; -import meta::pure::graphFetch::execution::*; -import meta::core::runtime::*; -import meta::pure::executionPlan::*; -import meta::pure::mapping::*; -import meta::pure::functions::boolean::*; -import meta::pure::metamodel::constraint::*; -import meta::pure::lineage::scanProperties::propertyTree::*; - - -Class meta::external::dataquality::MilestoningContext -{ - rootBusinessTemporal: Boolean[1]; - rootProcessingTemporal: Boolean[1]; - nonRootBusinessTemporal: Boolean[1]; - nonRootProcessingTemporal: Boolean[1]; - - businessTemporal(){ - $this.rootBusinessTemporal || $this.nonRootBusinessTemporal - }:Boolean[1]; - - processingTemporal(){ - $this.rootProcessingTemporal || $this.nonRootProcessingTemporal - }:Boolean[1]; -} - -function meta::external::dataquality::generateDataQualityQuery(dataquality:meta::external::dataquality::DataQuality[1], limit: Integer[*]): LambdaFunction[1] -{ - generateDataQualityQuery($dataquality, $limit, true); -} - - -function meta::external::dataquality::getMilestoningContext(tree:RootGraphFetchTree[1]): MilestoningContext[1] -{ - let properties = getAllProperties($tree); - let propertyTypes = $properties.genericType.rawType; - - ^MilestoningContext( - rootBusinessTemporal = $tree.class->meta::pure::milestoning::isBusinessTemporal(), - rootProcessingTemporal = $tree.class->meta::pure::milestoning::isProcessingTemporal(), - nonRootBusinessTemporal = $propertyTypes->exists(t | $t->meta::pure::milestoning::isBusinessTemporal()), - nonRootProcessingTemporal = $propertyTypes->exists(t | $t->meta::pure::milestoning::isProcessingTemporal()) - ); -} - -function meta::external::dataquality::addMilestoningParameters(tree:GraphFetchTree[1]):GraphFetchTree[1] -{ - $tree->match([ - p:PropertyGraphFetchTree[1] | - let parameters = if ($p.parameters->isEmpty(), - | let isBusinessTemporal = $p.property.genericType.rawType->toOne()->meta::pure::milestoning::isBusinessTemporal(); - let isProcessingTemporal = $p.property.genericType.rawType->toOne()->meta::pure::milestoning::isProcessingTemporal(); - getTemporalParameters($isProcessingTemporal, $isBusinessTemporal);, - | $p.parameters - ); - - ^$p( - subTrees = $p.subTrees->map(t | addMilestoningParameters($t)), - parameters = $parameters, - subTypeTrees = $p.subTypeTrees->map(t | addMilestoningParameters($t))->cast(@SubTypeGraphFetchTree) - );, - g:GraphFetchTree[1] | - ^$g( - subTrees = $g.subTrees->map(t | addMilestoningParameters($t)), - subTypeTrees = $g.subTypeTrees->map(t | addMilestoningParameters($t))->cast(@SubTypeGraphFetchTree) - ) - ]) -} - -function meta::external::dataquality::generateDataQualityQuery(dataquality:meta::external::dataquality::DataQuality[1], limit: Integer[*], useFrom:Boolean[1]): LambdaFunction[1] -{ - $dataquality.validationTree->validateTreeForNestedConstraints(true); - - let milestonedTree = $dataquality.validationTree->addMilestoningParameters(); - - // 1. enrich tree with selected constraint properties - let enrichedTree = $milestonedTree->ensureFunctionRequirementsForDataQuality($dataquality.validationTree.constraints, $dataquality.validationTree.class, [], true)->cast(@RootGraphFetchTree); - - let milestoningContext = getMilestoningContext($enrichedTree); - - // 2. build query - let getAll = $dataquality.validationTree.class->createGetAll($milestoningContext.rootProcessingTemporal, $milestoningContext.rootBusinessTemporal); - - let getExpr = if ($dataquality.filter->isNotEmpty(), - | $dataquality.validationTree.class->generateFilterQuery($getAll, $dataquality.filter->toOne());, - | $getAll); - let dqRootConstraints = $dataquality.validationTree.constraints; - let constraintQueryExpr = $dataquality.validationTree.class->meta::external::dataquality::generateConstraintsNegatedORQuery($getExpr, ^List(values=$dqRootConstraints)); - let limitQueryExpr = if ($limit->isNotEmpty(), - | ^SimpleFunctionExpression(func=take_T_MANY__Integer_1__T_MANY_, - parametersValues=[$constraintQueryExpr, ^InstanceValue(values=$limit->toOne(), genericType=^GenericType(rawType=Integer), multiplicity=PureOne)], - functionName=take_T_MANY__Integer_1__T_MANY_.name, // todo: function ref+name - genericType=^GenericType(rawType=$dataquality.validationTree.class), - multiplicity = ZeroMany, - importGroup=system::imports::coreImport), - | $constraintQueryExpr); - let graphFetchChecked = ^SimpleFunctionExpression(func=graphFetchChecked_T_MANY__RootGraphFetchTree_1__Checked_MANY_ , - parametersValues=[$limitQueryExpr, ^InstanceValue(values=$enrichedTree, genericType=^GenericType(rawType=RootGraphFetchTree), multiplicity=PureOne)], - functionName=graphFetchChecked_T_MANY__RootGraphFetchTree_1__Checked_MANY_.name, - genericType=^GenericType(rawType=Checked, typeArguments=^GenericType(rawType=$dataquality.validationTree.class)), - multiplicity = ZeroMany, - importGroup=system::imports::coreImport); - // 2.1 - let serialized = ^SimpleFunctionExpression(func=serialize_Checked_MANY__RootGraphFetchTree_1__String_1_, - functionName=serialize_Checked_MANY__RootGraphFetchTree_1__String_1_.name, - importGroup=system::imports::coreImport, - genericType=^GenericType(rawType=String), - multiplicity=ZeroMany, - parametersValues=[$graphFetchChecked, ^InstanceValue(values=$enrichedTree, genericType=^GenericType(rawType=RootGraphFetchTree), multiplicity=PureOne)])->evaluateAndDeactivate(); - - // 2.2 extract mapping and runtime here - let mappingAndRuntime = $dataquality.context->getMappingAndRuntime(); - let deactivatedMapping = ^InstanceValue(values=$mappingAndRuntime.first->evaluateAndDeactivate(), genericType=^GenericType(rawType=Mapping), multiplicity=PureOne) ; - let deactivatedRuntime = ^InstanceValue(values=$mappingAndRuntime.second->evaluateAndDeactivate(), genericType=^GenericType(rawType=NonExecutableValueSpecification), multiplicity=PureOne); - - let from = if ($useFrom, | ^SimpleFunctionExpression(func=from_T_m__Mapping_1__Runtime_1__T_m_, - functionName=from_T_m__Mapping_1__Runtime_1__T_m_.name, - importGroup=system::imports::coreImport, - genericType=^GenericType(rawType=String), - multiplicity=ZeroMany, - parametersValues=[$serialized, $deactivatedMapping, $deactivatedRuntime])->evaluateAndDeactivate(), | $serialized); - - // 3. build lambda - createLambda($from, $milestoningContext.processingTemporal, $milestoningContext.businessTemporal); -} - -function meta::external::dataquality::validateTreeForNestedConstraints(node:GraphFetchTree[1], isRoot:Boolean[1]): Boolean[1] -{ - if ($isRoot->isFalse() && $node->cast(@DataQualityPropertyGraphFetchTree).constraints->isNotEmpty(), - | fail('Nested constraints are not currently supported!'), - | '' - ); - $node.subTrees->fold({subtree, isValid | $isValid && $subtree->validateTreeForNestedConstraints(false)}, true); -} - -function meta::external::dataquality::ensureFunctionRequirementsForDataQuality(node:GraphFetchTree[1], constraints:Constraint[*], class:Class[1], processed:Class[*], ensureConstraintsForSubTrees:Boolean[1]): GraphFetchTree[1] -{ - let constraintResult = pathsForConstraintFunctions($class, $constraints.functionDefinition->concatenate($constraints.messageFunction)); - let qualifiedPropertyPaths = $constraintResult->filter(path| $path.values->exists(x| $x.property->instanceOf(QualifiedProperty))); //QualifiedProperty/derived property - methods within a class - first try with inline properties, milestoning properties - used in loans usecase - let inlinedPropertyTree = $constraintResult->meta::pure::lineage::scanProperties::propertyTree::buildPropertyTree()->meta::pure::lineage::scanProperties::inlineQualifiedPropertyNodes(); - let inlinedGraphTree = $inlinedPropertyTree->propertyTreeToGraphFetchTree($class); - let inlinedPropertyGraphTrees = $inlinedGraphTree.subTrees->cast(@PropertyGraphFetchTree); - let withFoundProperties = $node->addSubTrees($inlinedPropertyGraphTrees); - let updatedForClass = $qualifiedPropertyPaths->fold({path, gt| $gt->meta::external::dataquality::recordQualifiedProperties($path)}, $withFoundProperties); - let updatedProcessed = $processed->add($class); - - if($ensureConstraintsForSubTrees, - {| - let newSubTrees = $updatedForClass.subTrees->map({st| - let returns = if($st->cast(@PropertyGraphFetchTree).subType->isEmpty(), - | $st->cast(@PropertyGraphFetchTree).property->functionReturnType().rawType->toOne(), - | $st->cast(@PropertyGraphFetchTree).subType->toOne() - ); - if($returns->instanceOf(Class) && !$updatedProcessed->contains($returns), - | $st->ensureFunctionRequirementsForDataQuality($constraints, $returns->cast(@Class), $updatedProcessed, $ensureConstraintsForSubTrees), - | $st - ); - }); - - ^$updatedForClass(subTrees=$newSubTrees); - }, - | $updatedForClass - ); -} - -function <> meta::external::dataquality::createGetAll(c: Class[1]):FunctionExpression[1] -{ - let getAllExpression = ^SimpleFunctionExpression - ( - func = getAll_Class_1__T_MANY_, - functionName = getAll_Class_1__T_MANY_.name, - importGroup = system::imports::coreImport, - genericType = ^GenericType(rawType = $c), - multiplicity = ZeroMany, - parametersValues = ^InstanceValue( genericType = ^GenericType(rawType = Class, typeArguments = ^GenericType(rawType = $c)), - multiplicity = PureOne, - values = $c - ))->evaluateAndDeactivate(); - let classifierGenericType = ^GenericType(rawType = LambdaFunction, typeArguments = ^GenericType(rawType = ^FunctionType(returnMultiplicity = ZeroMany, returnType = ^GenericType(rawType = $c)))); - let lambda = {|[]}; - ^$lambda(classifierGenericType=$classifierGenericType, expressionSequence = $getAllExpression).expressionSequence->at(0)->cast(@FunctionExpression); -} - -function <> meta::external::dataquality::getAllProperties(tree:GraphFetchTree[1]):AbstractProperty[*] -{ - let properties = $tree->match([ - p:PropertyGraphFetchTree[1] | $p.property, - g:GraphFetchTree[1] | [] - ]); - - $tree.subTrees->concatenate($tree.subTypeTrees) - ->map(t | $t->getAllProperties()) - ->concatenate($properties); -} - -function meta::external::dataquality::createLambda(body:ValueSpecification[1], processingTemporal:Boolean[1], businessTemporal:Boolean[1]):LambdaFunction[1] -{ - let parameters = getTemporalParameters($processingTemporal, $businessTemporal); - let functionType = ^FunctionType(returnMultiplicity = $body.multiplicity, returnType = $body.genericType, parameters = $parameters); - - let lambda = newLambdaFunction($functionType); - ^$lambda(expressionSequence = $body); -} - -function <> meta::external::dataquality::createGetAll(c: Class[1], processingTemporal:Boolean[1], businessTemporal:Boolean[1]):FunctionExpression[1] -{ - let func = [ - pair($processingTemporal && $businessTemporal, getAll_Class_1__Date_1__Date_1__T_MANY_), - pair($processingTemporal || $businessTemporal, getAll_Class_1__Date_1__T_MANY_), - pair(true, getAll_Class_1__T_MANY_) - ]->filter(t | $t.first == true)->at(0).second; - - let parameters = getTemporalParameters($processingTemporal, $businessTemporal); - - ^SimpleFunctionExpression - ( - func = $func, - functionName = $func.name, - importGroup = system::imports::coreImport, - genericType = ^GenericType(rawType = $c), - multiplicity = ZeroMany, - parametersValues = ^InstanceValue( genericType = ^GenericType(rawType = Class, typeArguments = ^GenericType(rawType = $c)), - multiplicity = PureOne, - values = $c - )->concatenate($parameters) - )->evaluateAndDeactivate(); -} - -function <> meta::external::dataquality::getTemporalParameters(processingTemporal:Boolean[1], businessTemporal:Boolean[1]):VariableExpression[*] -{ - - [ - pair($processingTemporal, ^VariableExpression(name='processingDate', genericType=^GenericType(rawType=Date), multiplicity=PureOne)->evaluateAndDeactivate()), - pair($businessTemporal, ^VariableExpression(name='businessDate', genericType=^GenericType(rawType=Date), multiplicity=PureOne)->evaluateAndDeactivate()) - ]->filter(f | $f.first).second->evaluateAndDeactivate(); -} - -function meta::external::dataquality::generateFilterQuery(c:Class[1], f: FunctionExpression[1], filter:LambdaFunction<{T[1]->Boolean[1]}>[1]):FunctionExpression[1] { - let dummyLambda = {|'ok'}; - ^SimpleFunctionExpression - ( - func = filter_T_MANY__Function_1__T_MANY_, - multiplicity = ZeroMany, - genericType = ^GenericType(rawType = $c), - importGroup = system::imports::coreImport, - parametersValues = - [ $f, - ^InstanceValue ( - genericType = ^GenericType(rawType=LambdaFunction, typeArguments=^GenericType(rawType=^FunctionType(parameters=^VariableExpression(name='x', genericType=^GenericType(rawType = $c), multiplicity=PureOne), returnMultiplicity=PureOne, returnType=^GenericType(rawType=Boolean)))), - multiplicity = PureOne, - values = lambda(functionType('x', $c, PureOne, Boolean, ZeroMany), $filter.expressionSequence->evaluateAndDeactivate()->replaceVariableWithVariable('this', 'x')) - ) - ] - )->evaluateAndDeactivate(); -} - - -function meta::external::dataquality::lambda(functionType:FunctionType[1], expressionSequence:ValueSpecification[*]):LambdaFunction[1] -{ - let lambda = meta::pure::functions::meta::newLambdaFunction($functionType); - ^$lambda(expressionSequence = $expressionSequence->toOneMany()); -} - -function meta::external::dataquality::functionType(name:String[1], type:Type[1], multiplicity:Multiplicity[1], returnType:Type[1], returnMultiplicity:Multiplicity[1]):FunctionType[1] -{ - functionType($name, ^GenericType(rawType = $type), $multiplicity, ^GenericType(rawType = $returnType), $returnMultiplicity); -} - -function meta::external::dataquality::functionType(name:String[1], type:GenericType[1], multiplicity:Multiplicity[1], returnType:GenericType[1], returnMultiplicity:Multiplicity[1]):FunctionType[1] -{ - ^FunctionType(parameters = ^VariableExpression(genericType = $type, name = $name, multiplicity = $multiplicity), returnMultiplicity = $returnMultiplicity, returnType = $returnType); -} - -function meta::external::dataquality::functionType(parameters:VariableExpression[*], returnType:GenericType[1], returnMultiplicity:Multiplicity[1]):FunctionType[1] -{ - ^FunctionType(parameters = $parameters, returnMultiplicity = $returnMultiplicity, returnType = $returnType); -} - - -function meta::external::dataquality::generateConstraintsNegatedORQuery(c:Class[1], f: FunctionExpression[1], constraints: List[1]):FunctionExpression[1] { - if ($constraints.values->isEmpty(), - | $f , - | if ($constraints.values->size() == 1, - | $c->generateConstraintNegatedQuery($f, $constraints.values->at(0)) , - | $c->generateORNegatedQuery($f, $constraints) - ); - ); -} - - -function meta::external::dataquality::generateORNegatedQuery(c:Class[1], f: FunctionExpression[1], constraints: List[1]):FunctionExpression[1] -{ - let c1 = $constraints.values->at(0); - let c2 = $constraints.values->at(1); - let c1_negatedExprSequence = $c1.functionDefinition.expressionSequence->evaluateAndDeactivate()->toOne()->negatedFunctionExpression(); - let c2_negatedExprSequence = $c2.functionDefinition.expressionSequence->evaluateAndDeactivate()->toOne()->negatedFunctionExpression(); - let c1_c2_or_expr = $c1_negatedExprSequence->orFunctionExpression($c2_negatedExprSequence); - - let final_expr = $constraints.values->drop(2)->fold({c3, curr_or_exp | - let c3_negatedExprSequence = $c3.functionDefinition.expressionSequence->evaluateAndDeactivate()->toOne()->negatedFunctionExpression(); - $c3_negatedExprSequence->orFunctionExpression($curr_or_exp);}, $c1_c2_or_expr); - - ^SimpleFunctionExpression - ( - func = filter_T_MANY__Function_1__T_MANY_, - multiplicity = ZeroMany, - genericType = ^GenericType(rawType = $c), - importGroup = system::imports::coreImport, - parametersValues = - [ $f, - ^InstanceValue ( - genericType = $c1.functionDefinition.classifierGenericType->toOne(), - multiplicity = PureOne, - values = lambda(functionType('x', $c, PureOne, Boolean, ZeroMany), $final_expr->evaluateAndDeactivate()->replaceVariableWithVariable('this', 'x')) - ) - ] - )->evaluateAndDeactivate(); - -} - -function meta::external::dataquality::replaceVariableWithVariable(v:ValueSpecification[1..*], from:String[1], to:String[1]):ValueSpecification[1..*] -{ - $v->map(a | $a->replaceVariableWithVariable($from, $to)) -} - -function meta::external::dataquality::replaceVariableWithVariable(v:ValueSpecification[1], from:String[1], to:String[1]):ValueSpecification[1] -{ - $v->match([ - v:VariableExpression[1] | if ($v.name == $from, | ^$v(name = $to), | $v), - s:SimpleFunctionExpression[1] | ^$s(parametersValues = $s.parametersValues->map(p | replaceVariableWithVariable($p, $from, $to))), - i:InstanceValue[1] | ^$i(values = $i.values->map(v | $v->replaceVariableWithVariable($from, $to))) - ]) -} - -function meta::external::dataquality::replaceVariableWithVariable(l:FunctionDefinition[1], from:String[1], to:String[1]):FunctionDefinition[1] -{ - ^$l(expressionSequence = $l.expressionSequence->replaceVariableWithVariable($from, $to)) -} - -function meta::external::dataquality::replaceVariableWithVariable(a:Any[1], from:String[1], to:String[1]):Any[1] -{ - $a->match([ - v:ValueSpecification[1] | $v->replaceVariableWithVariable($from, $to), - l:LambdaFunction[1] | $l->replaceVariableWithVariable($from, $to), - a:Any[1] | $a - ]) -} - - -function meta::external::dataquality::generateConstraintNegatedQuery(c:Class[1], f: FunctionExpression[1], constraint: Constraint[1]):FunctionExpression[1] -{ - let constraintExprSequence = $constraint.functionDefinition.expressionSequence->evaluateAndDeactivate()->toOne(); - let negatedExprSequence = $constraintExprSequence->negatedFunctionExpression(); - - ^SimpleFunctionExpression - ( - func = filter_T_MANY__Function_1__T_MANY_, - multiplicity = ZeroMany, - genericType = ^GenericType(rawType = $c), - importGroup = system::imports::coreImport, - parametersValues = - [ $f, - ^InstanceValue ( - genericType = $constraint.functionDefinition.classifierGenericType->toOne(), - multiplicity = PureOne, - values = lambda(functionType('x', $c, PureOne, Boolean, ZeroMany), $negatedExprSequence->evaluateAndDeactivate()->replaceVariableWithVariable('this', 'x')) - ) - ] - )->evaluateAndDeactivate(); -} - -function <> {doc.doc = 'Genrates a "!$f" query'} -meta::external::dataquality::negatedFunctionExpression(f: ValueSpecification[1]):ValueSpecification[1] -{ - if($f->instanceOf(SimpleFunctionExpression) && $f->cast(@SimpleFunctionExpression).func->evaluateAndDeactivate() == not_Boolean_1__Boolean_1_, - | $f->cast(@SimpleFunctionExpression).parametersValues->evaluateAndDeactivate()->at(0), - | ^SimpleFunctionExpression - ( - func = not_Boolean_1__Boolean_1_, //native function meta::pure::functions::boolean::not(bool:Boolean[1]):Boolean[1]; - multiplicity = PureOne, - genericType = ^GenericType(rawType = Boolean), - importGroup = system::imports::coreImport, - parametersValues = $f - )->evaluateAndDeactivate() - ) -} - -function {doc.doc = 'Generates a OR query'} -meta::external::dataquality::orFunctionExpression(f1: ValueSpecification[1], f2: ValueSpecification[1]):ValueSpecification[1] -{ -^SimpleFunctionExpression - ( - func = meta::pure::functions::boolean::or_Boolean_1__Boolean_1__Boolean_1_ , // meta::pure::functions::boolean::or **use OR** - multiplicity = PureOne, - genericType = ^GenericType(rawType = Boolean), // check types - importGroup = system::imports::coreImport, - parametersValues = [$f1, $f2] - )->evaluateAndDeactivate(); -} - -function {doc.doc = 'Generates a AND query'} -meta::external::dataquality::andFunctionExpression(f1: ValueSpecification[1], f2: ValueSpecification[1]):ValueSpecification[1] -{ -^SimpleFunctionExpression - ( - func = meta::pure::functions::boolean::and_Boolean_1__Boolean_1__Boolean_1_ , - multiplicity = PureOne, - genericType = ^GenericType(rawType = Boolean), - importGroup = system::imports::coreImport, - parametersValues = [$f1, $f2] - )->evaluateAndDeactivate(); -} - -function meta::external::dataquality::getMappingAndRuntime(context: DataQualityExecutionContext[1]):Pair[1] { - if($context->instanceOf(MappingAndRuntimeDataQualityExecutionContext), - | let mappingAndRuntimeContext = $context->cast(@MappingAndRuntimeDataQualityExecutionContext); - ^Pair(first=$mappingAndRuntimeContext.mapping, second=$mappingAndRuntimeContext.runtime);, - | let dataSpaceDataQualityExecutionContext = $context->cast(@DataSpaceDataQualityExecutionContext); - let dataSpaceExecutionContext = $dataSpaceDataQualityExecutionContext.dataSpace.executionContexts->filter(execContext| $execContext.name == $dataSpaceDataQualityExecutionContext.contextName)->toOne(); - ^Pair(first=$dataSpaceExecutionContext.mapping, second=$dataSpaceExecutionContext.defaultRuntime.runtimeValue); - ); -} - -function meta::external::dataquality::recordQualifiedProperties(tree:GraphFetchTree[1], path:List[1]): GraphFetchTree[1] -{ - if($path.values->isEmpty(), - | $tree, - {| - let head = $path.values->at(0); - let tail = if($head.property->instanceOf(QualifiedProperty), - | list($head.nestedQualifierReturn.values->tail()->concatenate($path.values->tail())), - | list($path.values->tail()) - ); - let nextProperty = if($head.property->instanceOf(QualifiedProperty), - | $head.nestedQualifierReturn.values->first().property, - | $head.property - ); - let withQp = if($head.property->instanceOf(QualifiedProperty), - {| - let qp = $head.property->cast(@QualifiedProperty); - $tree->match([ - ergft: ExtendedRootGraphFetchTree[1] | ^$ergft(requiredQualifiedProperties=$ergft.requiredQualifiedProperties->concatenate($qp)->removeDuplicates()), - epgft: ExtendedPropertyGraphFetchTree[1] | ^$epgft(requiredQualifiedProperties=$epgft.requiredQualifiedProperties->concatenate($qp)->removeDuplicates()), - rgft : RootGraphFetchTree[1] | ^ExtendedRootGraphFetchTree(requiredQualifiedProperties=$qp, class=$rgft.class, subTrees=$rgft.subTrees), - pgft : PropertyGraphFetchTree[1] | ^ExtendedPropertyGraphFetchTree(requiredQualifiedProperties=$qp, property=$pgft.property, subTrees=$pgft.subTrees) - ]); - }, - | $tree; - ); - - ^$withQp(subTrees=$tree.subTrees->cast(@PropertyGraphFetchTree)->map(st|if($st.property == $nextProperty, |$st->meta::external::dataquality::recordQualifiedProperties($tail), |$st))); - } - ); -} - -function meta::external::dataquality::getEnrichedTreeForStructuralValidations(validationTree:meta::external::dataquality::DataQualityRootGraphFetchTree[1]): RootGraphFetchTree[1] -{ - // 1. enrich tree with selected constraint properties - $validationTree->ensureFunctionRequirementsForDataQuality($validationTree.constraints, $validationTree.class, [], true)->cast(@RootGraphFetchTree); -} - -function meta::external::dataquality::generateDQMetaDataForDQValidation(dataquality:meta::external::dataquality::DataQuality[1]): DataQualityRule[*] -{ - let enrichedTree = ensureFunctionRequirementsForDataQuality($dataquality.validationTree, $dataquality.validationTree.constraints, $dataquality.validationTree.class, [], true)->cast(@RootGraphFetchTree); - $dataquality.validationTree->nodeToDqRule(true, '')->concatenate($enrichedTree->nodeToDqRule(false, '')); -} - -function meta::external::dataquality::nodeToDqRule(node:GraphFetchTree[1], processOnlyConstraints:Boolean[1], path:String[*]):DataQualityRule[*] -{ - let dqRules = $node->match([ - dr: DataQualityRootGraphFetchTree[1] | $dr->rootNodeToDqRule($processOnlyConstraints), - dp: DataQualityPropertyGraphFetchTree[1] | $dp->propertyNodeToDqRule($processOnlyConstraints, $path), - r : RootGraphFetchTree[1] | $r->rootNodeToDqRule($processOnlyConstraints), - p : PropertyGraphFetchTree[1] | $p->propertyNodeToDqRule($processOnlyConstraints, $path) - ]); - - $dqRules->concatenate($node.subTrees->map(st|$st->nodeToDqRule($processOnlyConstraints, if ($dqRules->isEmpty(), | [], | $dqRules->at(0).propertyPath)))); -} - -function <> meta::external::dataquality::rootNodeToDqRule(node:RootGraphFetchTree[1], processOnlyConstraints: Boolean[1]):DataQualityRule[*] -{ - if ( $processOnlyConstraints, - | $node->cast(@DataQualityRootGraphFetchTree).constraints->map(c|$c->constraintToDqRule($node.class.name->toOne())), - | ^DataQualityRule(constraintName=$node.class.name->toOne(), constraintGrammar='Class', constraintType='Alloy_Class_Validation', propertyPath=$node.class.name->toOne()) - ); -} - -function <> meta::external::dataquality::propertyNodeToDqRule(node:PropertyGraphFetchTree[1], processOnlyConstraints: Boolean[1], path:String[*]):DataQualityRule[*] -{ - if ( $processOnlyConstraints, - | $node->cast(@DataQualityPropertyGraphFetchTree).constraints->map(c|$c->constraintToDqRule($path->concatenate($node.property.name->toOne())->joinStrings('::'))), - | ^DataQualityRule(constraintName=$node.property.name->toOne(), constraintGrammar=$node.property.multiplicity->printMultiplicity(), constraintType='Alloy_Structural_Validation', propertyPath=$path->concatenate($node.property.name->toOne())->joinStrings('::')) - ); -} - -function <> meta::external::dataquality::constraintToDqRule(constraint:Constraint[1], path:String[1]):DataQualityRule[1] -{ - ^DataQualityRule(constraintName=$constraint.name->toOne(), constraintGrammar=$constraint.functionDefinition->replaceVariableWithVariable('this', 'x')->meta::pure::metamodel::serialization::grammar::printFunctionDefinitionExpressions('')->toOne(), constraintType='Alloy_Constraint_Validation', propertyPath=$path); -} \ No newline at end of file +import meta::pure::metamodel::relation::*; +import meta::pure::dataQuality::*; +import meta::pure::metamodel::serialization::grammar::*; +import meta::pure::lineage::scanProperties::*; +import meta::external::dataquality::*; +import meta::pure::graphFetch::*; +import meta::pure::functions::collection::*; +import meta::pure::functions::meta::*; +import meta::pure::graphFetch::execution::*; +import meta::core::runtime::*; +import meta::pure::executionPlan::*; +import meta::pure::mapping::*; +import meta::pure::functions::boolean::*; +import meta::pure::metamodel::constraint::*; +import meta::pure::lineage::scanProperties::propertyTree::*; + + +Class meta::external::dataquality::MilestoningContext +{ + rootBusinessTemporal: Boolean[1]; + rootProcessingTemporal: Boolean[1]; + nonRootBusinessTemporal: Boolean[1]; + nonRootProcessingTemporal: Boolean[1]; + + businessTemporal(){ + $this.rootBusinessTemporal || $this.nonRootBusinessTemporal + }:Boolean[1]; + + processingTemporal(){ + $this.rootProcessingTemporal || $this.nonRootProcessingTemporal + }:Boolean[1]; +} + +function meta::external::dataquality::generateDataQualityQuery(dataquality:meta::external::dataquality::DataQuality[1], limit: Integer[*]): LambdaFunction[1] +{ + generateDataQualityQuery($dataquality, $limit, true); +} + + +function meta::external::dataquality::getMilestoningContext(tree:RootGraphFetchTree[1]): MilestoningContext[1] +{ + let properties = getAllProperties($tree); + let propertyTypes = $properties.genericType.rawType; + + ^MilestoningContext( + rootBusinessTemporal = $tree.class->meta::pure::milestoning::isBusinessTemporal(), + rootProcessingTemporal = $tree.class->meta::pure::milestoning::isProcessingTemporal(), + nonRootBusinessTemporal = $propertyTypes->exists(t | $t->meta::pure::milestoning::isBusinessTemporal()), + nonRootProcessingTemporal = $propertyTypes->exists(t | $t->meta::pure::milestoning::isProcessingTemporal()) + ); +} + +function meta::external::dataquality::addMilestoningParameters(tree:GraphFetchTree[1]):GraphFetchTree[1] +{ + $tree->match([ + p:PropertyGraphFetchTree[1] | + let parameters = if ($p.parameters->isEmpty(), + | let isBusinessTemporal = $p.property.genericType.rawType->toOne()->meta::pure::milestoning::isBusinessTemporal(); + let isProcessingTemporal = $p.property.genericType.rawType->toOne()->meta::pure::milestoning::isProcessingTemporal(); + getTemporalParameters($isProcessingTemporal, $isBusinessTemporal);, + | $p.parameters + ); + + ^$p( + subTrees = $p.subTrees->map(t | addMilestoningParameters($t)), + parameters = $parameters, + subTypeTrees = $p.subTypeTrees->map(t | addMilestoningParameters($t))->cast(@SubTypeGraphFetchTree) + );, + g:GraphFetchTree[1] | + ^$g( + subTrees = $g.subTrees->map(t | addMilestoningParameters($t)), + subTypeTrees = $g.subTypeTrees->map(t | addMilestoningParameters($t))->cast(@SubTypeGraphFetchTree) + ) + ]) +} + +function meta::external::dataquality::generateDataQualityQuery(dataquality:meta::external::dataquality::DataQuality[1], limit: Integer[*], useFrom:Boolean[1]): LambdaFunction[1] +{ + $dataquality.validationTree->validateTreeForNestedConstraints(true); + + let milestonedTree = $dataquality.validationTree->addMilestoningParameters(); + + // 1. enrich tree with selected constraint properties + let enrichedTree = $milestonedTree->ensureFunctionRequirementsForDataQuality($dataquality.validationTree.constraints, $dataquality.validationTree.class, [], true)->cast(@RootGraphFetchTree); + + let milestoningContext = getMilestoningContext($enrichedTree); + + // 2. build query + let getAll = $dataquality.validationTree.class->createGetAll($milestoningContext.rootProcessingTemporal, $milestoningContext.rootBusinessTemporal); + + let getExpr = if ($dataquality.filter->isNotEmpty(), + | $dataquality.validationTree.class->generateFilterQuery($getAll, $dataquality.filter->toOne());, + | $getAll); + let dqRootConstraints = $dataquality.validationTree.constraints; + let constraintQueryExpr = $dataquality.validationTree.class->meta::external::dataquality::generateConstraintsNegatedORQuery($getExpr, ^List(values=$dqRootConstraints)); + let limitQueryExpr = if ($limit->isNotEmpty(), + | ^SimpleFunctionExpression(func=take_T_MANY__Integer_1__T_MANY_, + parametersValues=[$constraintQueryExpr, ^InstanceValue(values=$limit->toOne(), genericType=^GenericType(rawType=Integer), multiplicity=PureOne)], + functionName=take_T_MANY__Integer_1__T_MANY_.name, // todo: function ref+name + genericType=^GenericType(rawType=$dataquality.validationTree.class), + multiplicity = ZeroMany, + importGroup=system::imports::coreImport), + | $constraintQueryExpr); + let graphFetchChecked = ^SimpleFunctionExpression(func=graphFetchChecked_T_MANY__RootGraphFetchTree_1__Checked_MANY_ , + parametersValues=[$limitQueryExpr, ^InstanceValue(values=$enrichedTree, genericType=^GenericType(rawType=RootGraphFetchTree), multiplicity=PureOne)], + functionName=graphFetchChecked_T_MANY__RootGraphFetchTree_1__Checked_MANY_.name, + genericType=^GenericType(rawType=Checked, typeArguments=^GenericType(rawType=$dataquality.validationTree.class)), + multiplicity = ZeroMany, + importGroup=system::imports::coreImport); + // 2.1 + let serialized = ^SimpleFunctionExpression(func=serialize_Checked_MANY__RootGraphFetchTree_1__String_1_, + functionName=serialize_Checked_MANY__RootGraphFetchTree_1__String_1_.name, + importGroup=system::imports::coreImport, + genericType=^GenericType(rawType=String), + multiplicity=ZeroMany, + parametersValues=[$graphFetchChecked, ^InstanceValue(values=$enrichedTree, genericType=^GenericType(rawType=RootGraphFetchTree), multiplicity=PureOne)])->evaluateAndDeactivate(); + + // 2.2 extract mapping and runtime here + let mappingAndRuntime = $dataquality.context->getMappingAndRuntime(); + let deactivatedMapping = ^InstanceValue(values=$mappingAndRuntime.first->evaluateAndDeactivate(), genericType=^GenericType(rawType=Mapping), multiplicity=PureOne) ; + let deactivatedRuntime = ^InstanceValue(values=$mappingAndRuntime.second->evaluateAndDeactivate(), genericType=^GenericType(rawType=NonExecutableValueSpecification), multiplicity=PureOne); + + let from = if ($useFrom, | ^SimpleFunctionExpression(func=from_T_m__Mapping_1__Runtime_1__T_m_, + functionName=from_T_m__Mapping_1__Runtime_1__T_m_.name, + importGroup=system::imports::coreImport, + genericType=^GenericType(rawType=String), + multiplicity=ZeroMany, + parametersValues=[$serialized, $deactivatedMapping, $deactivatedRuntime])->evaluateAndDeactivate(), | $serialized); + + // 3. build lambda + createLambda($from, $milestoningContext.processingTemporal, $milestoningContext.businessTemporal); +} + +function meta::external::dataquality::validateTreeForNestedConstraints(node:GraphFetchTree[1], isRoot:Boolean[1]): Boolean[1] +{ + if ($isRoot->isFalse() && $node->cast(@DataQualityPropertyGraphFetchTree).constraints->isNotEmpty(), + | fail('Nested constraints are not currently supported!'), + | '' + ); + $node.subTrees->fold({subtree, isValid | $isValid && $subtree->validateTreeForNestedConstraints(false)}, true); +} + +function meta::external::dataquality::ensureFunctionRequirementsForDataQuality(node:GraphFetchTree[1], constraints:Constraint[*], class:Class[1], processed:Class[*], ensureConstraintsForSubTrees:Boolean[1]): GraphFetchTree[1] +{ + let constraintResult = pathsForConstraintFunctions($class, $constraints.functionDefinition->concatenate($constraints.messageFunction)); + let qualifiedPropertyPaths = $constraintResult->filter(path| $path.values->exists(x| $x.property->instanceOf(QualifiedProperty))); //QualifiedProperty/derived property - methods within a class - first try with inline properties, milestoning properties - used in loans usecase + let inlinedPropertyTree = $constraintResult->meta::pure::lineage::scanProperties::propertyTree::buildPropertyTree()->meta::pure::lineage::scanProperties::inlineQualifiedPropertyNodes(); + let inlinedGraphTree = $inlinedPropertyTree->propertyTreeToGraphFetchTree($class); + let inlinedPropertyGraphTrees = $inlinedGraphTree.subTrees->cast(@PropertyGraphFetchTree); + let withFoundProperties = $node->addSubTrees($inlinedPropertyGraphTrees); + let updatedForClass = $qualifiedPropertyPaths->fold({path, gt| $gt->meta::external::dataquality::recordQualifiedProperties($path)}, $withFoundProperties); + let updatedProcessed = $processed->add($class); + + if($ensureConstraintsForSubTrees, + {| + let newSubTrees = $updatedForClass.subTrees->map({st| + let returns = if($st->cast(@PropertyGraphFetchTree).subType->isEmpty(), + | $st->cast(@PropertyGraphFetchTree).property->functionReturnType().rawType->toOne(), + | $st->cast(@PropertyGraphFetchTree).subType->toOne() + ); + if($returns->instanceOf(Class) && !$updatedProcessed->contains($returns), + | $st->ensureFunctionRequirementsForDataQuality($constraints, $returns->cast(@Class), $updatedProcessed, $ensureConstraintsForSubTrees), + | $st + ); + }); + + ^$updatedForClass(subTrees=$newSubTrees); + }, + | $updatedForClass + ); +} + +function <> meta::external::dataquality::createGetAll(c: Class[1]):FunctionExpression[1] +{ + let getAllExpression = ^SimpleFunctionExpression + ( + func = getAll_Class_1__T_MANY_, + functionName = getAll_Class_1__T_MANY_.name, + importGroup = system::imports::coreImport, + genericType = ^GenericType(rawType = $c), + multiplicity = ZeroMany, + parametersValues = ^InstanceValue( genericType = ^GenericType(rawType = Class, typeArguments = ^GenericType(rawType = $c)), + multiplicity = PureOne, + values = $c + ))->evaluateAndDeactivate(); + let classifierGenericType = ^GenericType(rawType = LambdaFunction, typeArguments = ^GenericType(rawType = ^FunctionType(returnMultiplicity = ZeroMany, returnType = ^GenericType(rawType = $c)))); + let lambda = {|[]}; + ^$lambda(classifierGenericType=$classifierGenericType, expressionSequence = $getAllExpression).expressionSequence->at(0)->cast(@FunctionExpression); +} + +function <> meta::external::dataquality::getAllProperties(tree:GraphFetchTree[1]):AbstractProperty[*] +{ + let properties = $tree->match([ + p:PropertyGraphFetchTree[1] | $p.property, + g:GraphFetchTree[1] | [] + ]); + + $tree.subTrees->concatenate($tree.subTypeTrees) + ->map(t | $t->getAllProperties()) + ->concatenate($properties); +} + +function meta::external::dataquality::createLambda(body:ValueSpecification[1], processingTemporal:Boolean[1], businessTemporal:Boolean[1]):LambdaFunction[1] +{ + let parameters = getTemporalParameters($processingTemporal, $businessTemporal); + let functionType = ^FunctionType(returnMultiplicity = $body.multiplicity, returnType = $body.genericType, parameters = $parameters); + + let lambda = newLambdaFunction($functionType); + ^$lambda(expressionSequence = $body); +} + +function <> meta::external::dataquality::createGetAll(c: Class[1], processingTemporal:Boolean[1], businessTemporal:Boolean[1]):FunctionExpression[1] +{ + let func = [ + pair($processingTemporal && $businessTemporal, getAll_Class_1__Date_1__Date_1__T_MANY_), + pair($processingTemporal || $businessTemporal, getAll_Class_1__Date_1__T_MANY_), + pair(true, getAll_Class_1__T_MANY_) + ]->filter(t | $t.first == true)->at(0).second; + + let parameters = getTemporalParameters($processingTemporal, $businessTemporal); + + ^SimpleFunctionExpression + ( + func = $func, + functionName = $func.name, + importGroup = system::imports::coreImport, + genericType = ^GenericType(rawType = $c), + multiplicity = ZeroMany, + parametersValues = ^InstanceValue( genericType = ^GenericType(rawType = Class, typeArguments = ^GenericType(rawType = $c)), + multiplicity = PureOne, + values = $c + )->concatenate($parameters) + )->evaluateAndDeactivate(); +} + +function <> meta::external::dataquality::getTemporalParameters(processingTemporal:Boolean[1], businessTemporal:Boolean[1]):VariableExpression[*] +{ + + [ + pair($processingTemporal, ^VariableExpression(name='processingDate', genericType=^GenericType(rawType=Date), multiplicity=PureOne)->evaluateAndDeactivate()), + pair($businessTemporal, ^VariableExpression(name='businessDate', genericType=^GenericType(rawType=Date), multiplicity=PureOne)->evaluateAndDeactivate()) + ]->filter(f | $f.first).second->evaluateAndDeactivate(); +} + +function meta::external::dataquality::generateFilterQuery(c:Class[1], f: FunctionExpression[1], filter:LambdaFunction<{T[1]->Boolean[1]}>[1]):FunctionExpression[1] { + let dummyLambda = {|'ok'}; + ^SimpleFunctionExpression + ( + func = filter_T_MANY__Function_1__T_MANY_, + multiplicity = ZeroMany, + genericType = ^GenericType(rawType = $c), + importGroup = system::imports::coreImport, + parametersValues = + [ $f, + ^InstanceValue ( + genericType = ^GenericType(rawType=LambdaFunction, typeArguments=^GenericType(rawType=^FunctionType(parameters=^VariableExpression(name='x', genericType=^GenericType(rawType = $c), multiplicity=PureOne), returnMultiplicity=PureOne, returnType=^GenericType(rawType=Boolean)))), + multiplicity = PureOne, + values = lambda(functionType('x', $c, PureOne, Boolean, ZeroMany), $filter.expressionSequence->evaluateAndDeactivate()->replaceVariableWithVariable('this', 'x')) + ) + ] + )->evaluateAndDeactivate(); +} + + +function meta::external::dataquality::lambda(functionType:FunctionType[1], expressionSequence:ValueSpecification[*]):LambdaFunction[1] +{ + let lambda = meta::pure::functions::meta::newLambdaFunction($functionType); + ^$lambda(expressionSequence = $expressionSequence->toOneMany()); +} + +function meta::external::dataquality::functionType(name:String[1], type:Type[1], multiplicity:Multiplicity[1], returnType:Type[1], returnMultiplicity:Multiplicity[1]):FunctionType[1] +{ + functionType($name, ^GenericType(rawType = $type), $multiplicity, ^GenericType(rawType = $returnType), $returnMultiplicity); +} + +function meta::external::dataquality::functionType(name:String[1], type:GenericType[1], multiplicity:Multiplicity[1], returnType:GenericType[1], returnMultiplicity:Multiplicity[1]):FunctionType[1] +{ + ^FunctionType(parameters = ^VariableExpression(genericType = $type, name = $name, multiplicity = $multiplicity), returnMultiplicity = $returnMultiplicity, returnType = $returnType); +} + +function meta::external::dataquality::functionType(parameters:VariableExpression[*], returnType:GenericType[1], returnMultiplicity:Multiplicity[1]):FunctionType[1] +{ + ^FunctionType(parameters = $parameters, returnMultiplicity = $returnMultiplicity, returnType = $returnType); +} + + +function meta::external::dataquality::generateConstraintsNegatedORQuery(c:Class[1], f: FunctionExpression[1], constraints: List[1]):FunctionExpression[1] { + if ($constraints.values->isEmpty(), + | $f , + | if ($constraints.values->size() == 1, + | $c->generateConstraintNegatedQuery($f, $constraints.values->at(0)) , + | $c->generateORNegatedQuery($f, $constraints) + ); + ); +} + + +function meta::external::dataquality::generateORNegatedQuery(c:Class[1], f: FunctionExpression[1], constraints: List[1]):FunctionExpression[1] +{ + let c1 = $constraints.values->at(0); + let c2 = $constraints.values->at(1); + let c1_negatedExprSequence = $c1.functionDefinition.expressionSequence->evaluateAndDeactivate()->toOne()->negatedFunctionExpression(); + let c2_negatedExprSequence = $c2.functionDefinition.expressionSequence->evaluateAndDeactivate()->toOne()->negatedFunctionExpression(); + let c1_c2_or_expr = $c1_negatedExprSequence->orFunctionExpression($c2_negatedExprSequence); + + let final_expr = $constraints.values->drop(2)->fold({c3, curr_or_exp | + let c3_negatedExprSequence = $c3.functionDefinition.expressionSequence->evaluateAndDeactivate()->toOne()->negatedFunctionExpression(); + $c3_negatedExprSequence->orFunctionExpression($curr_or_exp);}, $c1_c2_or_expr); + + ^SimpleFunctionExpression + ( + func = filter_T_MANY__Function_1__T_MANY_, + multiplicity = ZeroMany, + genericType = ^GenericType(rawType = $c), + importGroup = system::imports::coreImport, + parametersValues = + [ $f, + ^InstanceValue ( + genericType = $c1.functionDefinition.classifierGenericType->toOne(), + multiplicity = PureOne, + values = lambda(functionType('x', $c, PureOne, Boolean, ZeroMany), $final_expr->evaluateAndDeactivate()->replaceVariableWithVariable('this', 'x')) + ) + ] + )->evaluateAndDeactivate(); + +} + +function meta::external::dataquality::replaceVariableWithVariable(v:ValueSpecification[1..*], from:String[1], to:String[1]):ValueSpecification[1..*] +{ + $v->map(a | $a->replaceVariableWithVariable($from, $to)) +} + +function meta::external::dataquality::replaceVariableWithVariable(v:ValueSpecification[1], from:String[1], to:String[1]):ValueSpecification[1] +{ + $v->match([ + v:VariableExpression[1] | if ($v.name == $from, | ^$v(name = $to), | $v), + s:SimpleFunctionExpression[1] | ^$s(parametersValues = $s.parametersValues->map(p | replaceVariableWithVariable($p, $from, $to))), + i:InstanceValue[1] | ^$i(values = $i.values->map(v | $v->replaceVariableWithVariable($from, $to))) + ]) +} + +function meta::external::dataquality::replaceVariableWithVariable(l:FunctionDefinition[1], from:String[1], to:String[1]):FunctionDefinition[1] +{ + ^$l(expressionSequence = $l.expressionSequence->replaceVariableWithVariable($from, $to)) +} + +function meta::external::dataquality::replaceVariableWithVariable(a:Any[1], from:String[1], to:String[1]):Any[1] +{ + $a->match([ + v:ValueSpecification[1] | $v->replaceVariableWithVariable($from, $to), + l:LambdaFunction[1] | $l->replaceVariableWithVariable($from, $to), + a:Any[1] | $a + ]) +} + + +function meta::external::dataquality::generateConstraintNegatedQuery(c:Class[1], f: FunctionExpression[1], constraint: Constraint[1]):FunctionExpression[1] +{ + let constraintExprSequence = $constraint.functionDefinition.expressionSequence->evaluateAndDeactivate()->toOne(); + let negatedExprSequence = $constraintExprSequence->negatedFunctionExpression(); + + ^SimpleFunctionExpression + ( + func = filter_T_MANY__Function_1__T_MANY_, + multiplicity = ZeroMany, + genericType = ^GenericType(rawType = $c), + importGroup = system::imports::coreImport, + parametersValues = + [ $f, + ^InstanceValue ( + genericType = $constraint.functionDefinition.classifierGenericType->toOne(), + multiplicity = PureOne, + values = lambda(functionType('x', $c, PureOne, Boolean, ZeroMany), $negatedExprSequence->evaluateAndDeactivate()->replaceVariableWithVariable('this', 'x')) + ) + ] + )->evaluateAndDeactivate(); +} + +function <> {doc.doc = 'Genrates a "!$f" query'} +meta::external::dataquality::negatedFunctionExpression(f: ValueSpecification[1]):ValueSpecification[1] +{ + if($f->instanceOf(SimpleFunctionExpression) && $f->cast(@SimpleFunctionExpression).func->evaluateAndDeactivate() == not_Boolean_1__Boolean_1_, + | $f->cast(@SimpleFunctionExpression).parametersValues->evaluateAndDeactivate()->at(0), + | ^SimpleFunctionExpression + ( + func = not_Boolean_1__Boolean_1_, //native function meta::pure::functions::boolean::not(bool:Boolean[1]):Boolean[1]; + multiplicity = PureOne, + genericType = ^GenericType(rawType = Boolean), + importGroup = system::imports::coreImport, + parametersValues = $f + )->evaluateAndDeactivate() + ) +} + +function {doc.doc = 'Generates a OR query'} +meta::external::dataquality::orFunctionExpression(f1: ValueSpecification[1], f2: ValueSpecification[1]):ValueSpecification[1] +{ +^SimpleFunctionExpression + ( + func = meta::pure::functions::boolean::or_Boolean_1__Boolean_1__Boolean_1_ , // meta::pure::functions::boolean::or **use OR** + multiplicity = PureOne, + genericType = ^GenericType(rawType = Boolean), // check types + importGroup = system::imports::coreImport, + parametersValues = [$f1, $f2] + )->evaluateAndDeactivate(); +} + +function {doc.doc = 'Generates a AND query'} +meta::external::dataquality::andFunctionExpression(f1: ValueSpecification[1], f2: ValueSpecification[1]):ValueSpecification[1] +{ +^SimpleFunctionExpression + ( + func = meta::pure::functions::boolean::and_Boolean_1__Boolean_1__Boolean_1_ , + multiplicity = PureOne, + genericType = ^GenericType(rawType = Boolean), + importGroup = system::imports::coreImport, + parametersValues = [$f1, $f2] + )->evaluateAndDeactivate(); +} + +function meta::external::dataquality::getMappingAndRuntime(context: DataQualityExecutionContext[1]):Pair[1] { + if($context->instanceOf(MappingAndRuntimeDataQualityExecutionContext), + | let mappingAndRuntimeContext = $context->cast(@MappingAndRuntimeDataQualityExecutionContext); + ^Pair(first=$mappingAndRuntimeContext.mapping, second=$mappingAndRuntimeContext.runtime);, + | let dataSpaceDataQualityExecutionContext = $context->cast(@DataSpaceDataQualityExecutionContext); + let dataSpaceExecutionContext = $dataSpaceDataQualityExecutionContext.dataSpace.executionContexts->filter(execContext| $execContext.name == $dataSpaceDataQualityExecutionContext.contextName)->toOne(); + ^Pair(first=$dataSpaceExecutionContext.mapping, second=$dataSpaceExecutionContext.defaultRuntime.runtimeValue); + ); +} + +function meta::external::dataquality::recordQualifiedProperties(tree:GraphFetchTree[1], path:List[1]): GraphFetchTree[1] +{ + if($path.values->isEmpty(), + | $tree, + {| + let head = $path.values->at(0); + let tail = if($head.property->instanceOf(QualifiedProperty), + | list($head.nestedQualifierReturn.values->tail()->concatenate($path.values->tail())), + | list($path.values->tail()) + ); + let nextProperty = if($head.property->instanceOf(QualifiedProperty), + | $head.nestedQualifierReturn.values->first().property, + | $head.property + ); + let withQp = if($head.property->instanceOf(QualifiedProperty), + {| + let qp = $head.property->cast(@QualifiedProperty); + $tree->match([ + ergft: ExtendedRootGraphFetchTree[1] | ^$ergft(requiredQualifiedProperties=$ergft.requiredQualifiedProperties->concatenate($qp)->removeDuplicates()), + epgft: ExtendedPropertyGraphFetchTree[1] | ^$epgft(requiredQualifiedProperties=$epgft.requiredQualifiedProperties->concatenate($qp)->removeDuplicates()), + rgft : RootGraphFetchTree[1] | ^ExtendedRootGraphFetchTree(requiredQualifiedProperties=$qp, class=$rgft.class, subTrees=$rgft.subTrees), + pgft : PropertyGraphFetchTree[1] | ^ExtendedPropertyGraphFetchTree(requiredQualifiedProperties=$qp, property=$pgft.property, subTrees=$pgft.subTrees) + ]); + }, + | $tree; + ); + + ^$withQp(subTrees=$tree.subTrees->cast(@PropertyGraphFetchTree)->map(st|if($st.property == $nextProperty, |$st->meta::external::dataquality::recordQualifiedProperties($tail), |$st))); + } + ); +} + +function meta::external::dataquality::getEnrichedTreeForStructuralValidations(validationTree:meta::external::dataquality::DataQualityRootGraphFetchTree[1]): RootGraphFetchTree[1] +{ + // 1. enrich tree with selected constraint properties + $validationTree->ensureFunctionRequirementsForDataQuality($validationTree.constraints, $validationTree.class, [], true)->cast(@RootGraphFetchTree); +} + +function meta::external::dataquality::generateDQMetaDataForDQValidation(dataquality:meta::external::dataquality::DataQuality[1]): DataQualityRule[*] +{ + let enrichedTree = ensureFunctionRequirementsForDataQuality($dataquality.validationTree, $dataquality.validationTree.constraints, $dataquality.validationTree.class, [], true)->cast(@RootGraphFetchTree); + $dataquality.validationTree->nodeToDqRule(true, '')->concatenate($enrichedTree->nodeToDqRule(false, '')); +} + +function meta::external::dataquality::nodeToDqRule(node:GraphFetchTree[1], processOnlyConstraints:Boolean[1], path:String[*]):DataQualityRule[*] +{ + let dqRules = $node->match([ + dr: DataQualityRootGraphFetchTree[1] | $dr->rootNodeToDqRule($processOnlyConstraints), + dp: DataQualityPropertyGraphFetchTree[1] | $dp->propertyNodeToDqRule($processOnlyConstraints, $path), + r : RootGraphFetchTree[1] | $r->rootNodeToDqRule($processOnlyConstraints), + p : PropertyGraphFetchTree[1] | $p->propertyNodeToDqRule($processOnlyConstraints, $path) + ]); + + $dqRules->concatenate($node.subTrees->map(st|$st->nodeToDqRule($processOnlyConstraints, if ($dqRules->isEmpty(), | [], | $dqRules->at(0).propertyPath)))); +} + +function <> meta::external::dataquality::rootNodeToDqRule(node:RootGraphFetchTree[1], processOnlyConstraints: Boolean[1]):DataQualityRule[*] +{ + if ( $processOnlyConstraints, + | $node->cast(@DataQualityRootGraphFetchTree).constraints->map(c|$c->constraintToDqRule($node.class.name->toOne())), + | ^DataQualityRule(constraintName=$node.class.name->toOne(), constraintGrammar='Class', constraintType='Alloy_Class_Validation', propertyPath=$node.class.name->toOne()) + ); +} + +function <> meta::external::dataquality::propertyNodeToDqRule(node:PropertyGraphFetchTree[1], processOnlyConstraints: Boolean[1], path:String[*]):DataQualityRule[*] +{ + if ( $processOnlyConstraints, + | $node->cast(@DataQualityPropertyGraphFetchTree).constraints->map(c|$c->constraintToDqRule($path->concatenate($node.property.name->toOne())->joinStrings('::'))), + | ^DataQualityRule(constraintName=$node.property.name->toOne(), constraintGrammar=$node.property.multiplicity->printMultiplicity(), constraintType='Alloy_Structural_Validation', propertyPath=$path->concatenate($node.property.name->toOne())->joinStrings('::')) + ); +} + +function <> meta::external::dataquality::constraintToDqRule(constraint:Constraint[1], path:String[1]):DataQualityRule[1] +{ + ^DataQualityRule(constraintName=$constraint.name->toOne(), constraintGrammar=$constraint.functionDefinition->replaceVariableWithVariable('this', 'x')->meta::pure::metamodel::serialization::grammar::printFunctionDefinitionExpressions('')->toOne(), constraintType='Alloy_Constraint_Validation', propertyPath=$path); +} + +function meta::external::dataquality::generateDataqualityRelationValidationLambda(dqRelationValidation: meta::external::dataquality::DataQualityRelationValidation[1], validationName: String[1]): LambdaFunction[1] +{ + let lambda = $dqRelationValidation.query; + ^$lambda(expressionSequence = [$dqRelationValidation->buildRelationFilterExpression($validationName)]); +} + +function meta::external::dataquality::buildRelationFilterExpression(dqRelationValidation: meta::external::dataquality::DataQualityRelationValidation[1], validationName: String[1]):FunctionExpression[1] +{ + let relationType = $dqRelationValidation.query->evaluateAndDeactivate()->functionReturnType().typeArguments.rawType->toOne(); + + ^SimpleFunctionExpression + ( + func = filter_Relation_1__Function_1__Relation_1_, + multiplicity = PureOne, + genericType = ^GenericType(rawType=Relation, typeArguments = ^GenericType(rawType=$relationType)), + importGroup = system::imports::coreImport, + parametersValues = + [ $dqRelationValidation.query->evaluateAndDeactivate()->toOne().expressionSequence->toOne(), + ^InstanceValue ( + genericType = ^GenericType(rawType=LambdaFunction, typeArguments=^GenericType(rawType=^FunctionType(parameters=^VariableExpression(name='validationFilter', genericType=^GenericType(rawType=$relationType), multiplicity=PureOne), returnMultiplicity=PureOne, returnType=^GenericType(rawType=Boolean)))), + multiplicity = PureOne, + values = meta::external::dataquality::lambda(^FunctionType(returnMultiplicity = PureOne, returnType = ^GenericType(rawType = Boolean), parameters = [^VariableExpression(multiplicity=PureOne,genericType=^GenericType(rawType=$relationType),name='assertion')]), + $dqRelationValidation.validations->filter(val| $val.name == $validationName)->toOne().assertion.expressionSequence->evaluateAndDeactivate()->toOne()->negatedFunctionExpression()) + ) + ] + )->evaluateAndDeactivate(); +} diff --git a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/src/main/resources/core_dataquality/metamodel/metamodel.pure b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/src/main/resources/core_dataquality/metamodel/metamodel.pure index 17aca437cc0..3a110ce06dc 100644 --- a/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/src/main/resources/core_dataquality/metamodel/metamodel.pure +++ b/legend-engine-xts-dataquality/legend-engine-xt-dataquality-pure/src/main/resources/core_dataquality/metamodel/metamodel.pure @@ -48,4 +48,17 @@ Class meta::external::dataquality::DataQualityRule constraintType : String[1]; constraintGrammar: String[1]; propertyPath: String[1]; +} + +Class meta::external::dataquality::DataQualityRelationValidation extends PackageableElement +{ + query: LambdaFunction[1]; // should return a relation - enforced in compiler + validations: meta::external::dataquality::RelationValidation[*]; +} + +Class meta::external::dataquality::RelationValidation +{ + name: String[1]; + description: String[0..1]; + assertion: LambdaFunction<{Nil[1]->Boolean[1]}>[1]; } \ No newline at end of file