diff --git a/legend-engine-extensions-collection-generation/src/test/java/org/finos/legend/engine/extensions/collection/generation/TestExtensions.java b/legend-engine-extensions-collection-generation/src/test/java/org/finos/legend/engine/extensions/collection/generation/TestExtensions.java index 26218d53f22..3356434f902 100644 --- a/legend-engine-extensions-collection-generation/src/test/java/org/finos/legend/engine/extensions/collection/generation/TestExtensions.java +++ b/legend-engine-extensions-collection-generation/src/test/java/org/finos/legend/engine/extensions/collection/generation/TestExtensions.java @@ -14,6 +14,7 @@ package org.finos.legend.engine.extensions.collection.generation; +import java.util.Set; import org.eclipse.collections.api.factory.Lists; import org.eclipse.collections.api.factory.Sets; import org.eclipse.collections.api.list.MutableList; @@ -25,6 +26,7 @@ import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel; import org.finos.legend.engine.language.pure.compiler.toPureGraph.extension.CompilerExtension; import org.finos.legend.engine.language.pure.dsl.generation.extension.ArtifactGenerationExtension; +import org.finos.legend.engine.language.pure.dsl.generation.extension.ArtifactGenerationExtensionLoader; import org.finos.legend.engine.language.pure.grammar.from.DataSpaceParserExtension; import org.finos.legend.engine.language.pure.grammar.from.DiagramParserExtension; import org.finos.legend.engine.language.pure.grammar.from.TextParserExtension; @@ -93,7 +95,7 @@ public void testExpectedExternalFormatExtensionsArePresent() @Test public void testExpectedArtifactGenerationExtensionsArePresent() { - assertHasExtensions(getExpectedArtifactGenerationExtensions(), ArtifactGenerationExtension.class); + assertHasExtensions(ArtifactGenerationExtensionLoader.extensions(), getExpectedArtifactGenerationExtensions(), ArtifactGenerationExtension.class); } @Test diff --git a/legend-engine-language-pure-dsl-generation/src/main/java/org/finos/legend/engine/language/pure/dsl/generation/extension/ArtifactGenerationExtension.java b/legend-engine-language-pure-dsl-generation/src/main/java/org/finos/legend/engine/language/pure/dsl/generation/extension/ArtifactGenerationExtension.java index f60b77e2201..302ab47db4e 100644 --- a/legend-engine-language-pure-dsl-generation/src/main/java/org/finos/legend/engine/language/pure/dsl/generation/extension/ArtifactGenerationExtension.java +++ b/legend-engine-language-pure-dsl-generation/src/main/java/org/finos/legend/engine/language/pure/dsl/generation/extension/ArtifactGenerationExtension.java @@ -25,24 +25,19 @@ */ public interface ArtifactGenerationExtension { - - /** - * Determines whether the extension can generate artifacts based on the element. + * Gives the key for the extension * - * @return boolean flag indicating if the extension can generate artifacts + * @return string */ - boolean canGenerate(PackageableElement element); - + String getKey(); /** - * Gets root path where all artifacts will be stored + * Determines whether a packageable element can generate artifacts * - * @return root path + * @return boolean flag indicating if the extension can generate artifacts */ - - String getArtifactsRootPath(); - + boolean canGenerate(PackageableElement element); /** * Generates artifacts given a packageable element. Methods assumes element can generate diff --git a/legend-engine-language-pure-dsl-generation/src/main/java/org/finos/legend/engine/language/pure/dsl/generation/extension/ArtifactGenerationExtensionLoader.java b/legend-engine-language-pure-dsl-generation/src/main/java/org/finos/legend/engine/language/pure/dsl/generation/extension/ArtifactGenerationExtensionLoader.java new file mode 100644 index 00000000000..390ff70c47e --- /dev/null +++ b/legend-engine-language-pure-dsl-generation/src/main/java/org/finos/legend/engine/language/pure/dsl/generation/extension/ArtifactGenerationExtensionLoader.java @@ -0,0 +1,50 @@ +// Copyright 2022 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.language.pure.dsl.generation.extension; + +import java.util.List; +import java.util.ServiceLoader; +import java.util.Set; +import java.util.stream.Collectors; +import org.eclipse.collections.api.factory.Lists; +import org.eclipse.collections.api.factory.Sets; +import org.eclipse.collections.impl.utility.ListIterate; +import org.finos.legend.engine.shared.core.operational.errorManagement.EngineException; + +public class ArtifactGenerationExtensionLoader +{ + public static final String EXTENSION_KEY_REGEX = "^[a-zA-Z_\\-]+$"; + + public static List extensions() + { + List extensions = Lists.mutable.withAll(ServiceLoader.load(ArtifactGenerationExtension.class)); + Set extensionKeys = Sets.mutable.empty(); + for (ArtifactGenerationExtension extension : extensions) + { + if (!extensionKeys.add(extension.getKey())) + { + String extensionsWithSameKey = ListIterate.collect(extensions.stream().filter(e -> e.getKey().equals(extension.getKey())).collect(Collectors.toList()), e -> e.getClass().getName()) + .makeString(","); + throw new EngineException("Artifact extension keys must be unique. Found duplicate key: '" + extension.getKey() + "' on extensions: " + extensionsWithSameKey); + } + if (!extension.getKey().matches(EXTENSION_KEY_REGEX)) + { + throw new EngineException("Artifact extension keys can't have spaces or special characters. Found invalid key: '" + extension.getKey() + "'."); + } + } + return extensions; + } + +} diff --git a/legend-engine-xt-data-space-generation/pom.xml b/legend-engine-xt-data-space-generation/pom.xml index fd7b98c50aa..c9ed9c8ab04 100644 --- a/legend-engine-xt-data-space-generation/pom.xml +++ b/legend-engine-xt-data-space-generation/pom.xml @@ -28,11 +28,14 @@ Legend Engine - XT - Data Space - Generation - + - org.finos.legend.pure - legend-pure-runtime-java-engine-compiled + org.eclipse.collections + eclipse-collections-api + + + org.finos.legend.pure legend-pure-m3-core @@ -76,5 +79,34 @@ jackson-databind + + + + junit + junit + test + + + org.finos.legend.engine + legend-engine-xt-diagram-grammar + test + + + org.finos.legend.engine + legend-engine-xt-data-space-grammar + test + + + org.finos.legend.engine + legend-engine-language-pure-grammar + test + + + org.finos.legend.engine + legend-engine-language-pure-compiler + test-jar + test + + diff --git a/legend-engine-xt-data-space-generation/src/main/java/org/finos/legend/engine/generation/DataSpaceAnalyticsArtifactGenerationExtension.java b/legend-engine-xt-data-space-generation/src/main/java/org/finos/legend/engine/generation/DataSpaceAnalyticsArtifactGenerationExtension.java index f799a22e315..79cd34a8a0d 100644 --- a/legend-engine-xt-data-space-generation/src/main/java/org/finos/legend/engine/generation/DataSpaceAnalyticsArtifactGenerationExtension.java +++ b/legend-engine-xt-data-space-generation/src/main/java/org/finos/legend/engine/generation/DataSpaceAnalyticsArtifactGenerationExtension.java @@ -14,7 +14,11 @@ package org.finos.legend.engine.generation; +import static org.finos.legend.engine.language.pure.compiler.toPureGraph.HelperModelBuilder.getElementFullPath; + import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.Collections; +import java.util.List; import org.finos.legend.engine.analytics.DataSpaceAnalyticsHelper; import org.finos.legend.engine.analytics.model.DataSpaceAnalysisResult; import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel; @@ -24,36 +28,32 @@ import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.dataSpace.DataSpace; import org.finos.legend.engine.shared.core.ObjectMapperFactory; import org.finos.legend.engine.shared.core.operational.Assert; +import org.finos.legend.engine.shared.core.operational.errorManagement.EngineException; import org.finos.legend.pure.generated.Root_meta_pure_metamodel_dataSpace_DataSpace; import org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.PackageableElement; -import java.util.Collections; -import java.util.List; - -import static org.finos.legend.engine.language.pure.compiler.toPureGraph.HelperModelBuilder.getElementFullPath; -import static org.finos.legend.pure.generated.platform_pure_corefunctions_meta.Root_meta_pure_functions_meta_elementToPath_PackageableElement_1__String_1__String_1_; - public class DataSpaceAnalyticsArtifactGenerationExtension implements ArtifactGenerationExtension { + public final String ROOT_PATH = "dataSpace-analytics"; - public final String ROOT_PATH = "data-space-analytics"; public static ObjectMapper objectMapper = ObjectMapperFactory.getNewStandardObjectMapperWithPureProtocolExtensionSupports(); @Override - public boolean canGenerate(PackageableElement element) + public String getKey() { - return element instanceof Root_meta_pure_metamodel_dataSpace_DataSpace; + return ROOT_PATH; } @Override - public String getArtifactsRootPath() + public boolean canGenerate(PackageableElement element) { - return ROOT_PATH; + return element instanceof Root_meta_pure_metamodel_dataSpace_DataSpace; } @Override public List generate(PackageableElement element, PureModel pureModel, PureModelContextData data, String clientVersion) { + String fileName = "AnalyticsResult.json"; String dataSpacePath = getElementFullPath(element, pureModel.getExecutionSupport()); Assert.assertTrue(this.canGenerate(element), () -> "DataSpace analytics only supports dataSpace elements"); Root_meta_pure_metamodel_dataSpace_DataSpace dataSpace = (Root_meta_pure_metamodel_dataSpace_DataSpace) element; @@ -63,15 +63,12 @@ public List generate(PackageableElement element, PureModel pureModel, try { String stringResult = objectMapper.writeValueAsString(result); - String filePath = Root_meta_pure_functions_meta_elementToPath_PackageableElement_1__String_1__String_1_(element, "/", pureModel.getExecutionSupport()) + ".json"; - Artifact output = new Artifact(stringResult, filePath, "json"); + Artifact output = new Artifact(stringResult, fileName, "json"); return Collections.singletonList(output); - } - catch (Exception ignored) + catch (Exception exception) { - // ignore + throw new EngineException("Can't serialize data space analysis result", exception); } - return null; } } diff --git a/legend-engine-xt-data-space-generation/src/test/java/org/finos/legend/engine/generation/TestDataSpaceAnalyticsArtifactGenerationExtension.java b/legend-engine-xt-data-space-generation/src/test/java/org/finos/legend/engine/generation/TestDataSpaceAnalyticsArtifactGenerationExtension.java new file mode 100644 index 00000000000..fde85ad3a7b --- /dev/null +++ b/legend-engine-xt-data-space-generation/src/test/java/org/finos/legend/engine/generation/TestDataSpaceAnalyticsArtifactGenerationExtension.java @@ -0,0 +1,81 @@ +// Copyright 2022 Goldman Sachs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.finos.legend.engine.generation; + +import java.net.URL; +import java.util.List; +import java.util.Map; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.eclipse.collections.api.factory.Maps; +import org.finos.legend.engine.analytics.model.DataSpaceAnalysisResult; +import org.finos.legend.engine.language.pure.compiler.Compiler; +import org.finos.legend.engine.language.pure.compiler.toPureGraph.PureModel; +import org.finos.legend.engine.language.pure.dsl.generation.extension.Artifact; +import org.finos.legend.engine.language.pure.grammar.from.PureGrammarParser; +import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData; +import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.dataSpace.DataSpace; +import org.finos.legend.engine.shared.core.ObjectMapperFactory; +import org.finos.legend.engine.shared.core.deployment.DeploymentMode; +import org.finos.legend.engine.shared.core.deployment.DeploymentStateAndVersions; +import org.finos.legend.pure.generated.Root_meta_pure_metamodel_dataSpace_DataSpace; +import org.junit.Assert; +import org.junit.Test; + + +public class TestDataSpaceAnalyticsArtifactGenerationExtension +{ + private static final ObjectMapper objectMapper = ObjectMapperFactory.getNewStandardObjectMapperWithPureProtocolExtensionSupports(); + + private String getResourceAsString(String path) + { + try + { + URL infoURL = DeploymentStateAndVersions.class.getClassLoader().getResource(path); + if (infoURL != null) + { + java.util.Scanner scanner = new java.util.Scanner(infoURL.openStream()).useDelimiter("\\A"); + return scanner.hasNext() ? scanner.next() : null; + } + return null; + } + catch (Exception e) + { + throw new RuntimeException(e); + } + } + + @Test + public void testDataSpaceAnalyticsArtifactGenerationExtension() throws Exception + { + String pureModelString = getResourceAsString("models/DataspaceModel.pure"); + PureModelContextData pureModelContextData = PureGrammarParser.newInstance().parseModel(pureModelString); + PureModel pureModel = Compiler.compile(pureModelContextData, DeploymentMode.TEST, null); + Map> results = Maps.mutable.empty(); + DataSpaceAnalyticsArtifactGenerationExtension extension = new DataSpaceAnalyticsArtifactGenerationExtension(); + org.finos.legend.pure.m3.coreinstance.meta.pure.metamodel.PackageableElement packageableElement = pureModel.getPackageableElement("dataSpace::_FirmDataSpace"); + Assert.assertTrue(packageableElement instanceof Root_meta_pure_metamodel_dataSpace_DataSpace); + Root_meta_pure_metamodel_dataSpace_DataSpace metamodelDatasapce = (Root_meta_pure_metamodel_dataSpace_DataSpace) packageableElement; + Assert.assertTrue(extension.canGenerate(metamodelDatasapce)); + List outputs = extension.generate(packageableElement, pureModel, pureModelContextData, "vX_X_X"); + Assert.assertEquals(1, outputs.size()); + Artifact dataSpaceAnalyticsResult = outputs.get(0); + Assert.assertEquals(dataSpaceAnalyticsResult.format, "json"); + Assert.assertEquals(dataSpaceAnalyticsResult.path, "AnalyticsResult.json"); + Assert.assertNotNull(dataSpaceAnalyticsResult.content); + // assert the result is JSON of data space analysis result + objectMapper.readValue(dataSpaceAnalyticsResult.content, DataSpaceAnalysisResult.class); + } +} diff --git a/legend-engine-xt-data-space-generation/src/test/resources/models/DataspaceModel.pure b/legend-engine-xt-data-space-generation/src/test/resources/models/DataspaceModel.pure new file mode 100644 index 00000000000..899234cbf00 --- /dev/null +++ b/legend-engine-xt-data-space-generation/src/test/resources/models/DataspaceModel.pure @@ -0,0 +1,124 @@ +###Diagram +Diagram diagram::MyDiagram +{ + classView a6b8266b-395e-4308-a08a-8f9b44af6d53 + { + class: model::target::_Firm; + position: (547.0,270.0); + rectangle: (159.0,72.0); + } + classView c41deda8-19be-4374-9260-8be50c32d3cd + { + class: model::target::_Person; + position: (1128.0,308.0); + rectangle: (122.0,44.0); + } + propertyView + { + property: model::target::_Firm.employees; + source: a6b8266b-395e-4308-a08a-8f9b44af6d53; + target: c41deda8-19be-4374-9260-8be50c32d3cd; + points: [(626.5,306.0),(1189.0,330.0)]; + } +} + + +###DataSpace +DataSpace dataSpace::_FirmDataSpace +{ + executionContexts: + [ + { + name: 'dummyContext'; + mapping: mapping::ModelToModelMapping; + defaultRuntime: mapping::ModelToModelRuntime; + } + ]; + defaultExecutionContext: 'dummyContext'; + title: 'Firm Dataspace'; +} + + +###Pure +Enum model::target::IncType +{ + LLC, + CORP +} + +Class model::Person +{ + firstName: String[1]; + lastName: String[1]; +} + +Class model::Firm +{ + employees: model::Person[1..*]; + legalName: String[1]; + type: String[1]; +} + +Class model::target::_Person +{ + fullName: String[1]; +} + +Class model::target::_Firm +{ + employees: model::target::_Person[1..*]; + type: model::target::IncType[1]; + name: String[1]; +} + +Class model::NotRelevant +{ +} + + +###Mapping +Mapping mapping::ModelToModelMapping +( + *model::target::_Person: Pure + { + ~src model::Person + fullName: $src.firstName + ' ' + $src.lastName + } + *model::target::_Firm: Pure + { + ~src model::Firm + employees[model_target__Person]: $src.employees, + type: EnumerationMapping model_target_IncType: $src.type, + name: $src.legalName + } + + model::target::IncType: EnumerationMapping + { + LLC: ['llc'], + CORP: ['corp'] + } +) + + +###Runtime +Runtime mapping::ModelToModelRuntime +{ + mappings: + [ + mapping::ModelToModelMapping + ]; + connections: + [ + ModelStore: + [ + connection_1: + #{ + JsonModelConnection + { + class: model::Firm; + url: 'data:application/json,%7B%7D'; + } + }# + ] + ]; +}