Skip to content

Commit

Permalink
Initial version of DB generation from a connection
Browse files Browse the repository at this point in the history
  • Loading branch information
@kevin-m-knight-gs
kevin-m-knight-gs committed Dec 7, 2023
1 parent 7346c9b commit 2d3fc72
Show file tree
Hide file tree
Showing 3 changed files with 163 additions and 4 deletions.
13 changes: 13 additions & 0 deletions legend-engine-ide-lsp-default-extensions/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -156,11 +156,24 @@
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>

<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
</dependency>
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
</dependency>

<dependency>
<groupId>org.eclipse.collections</groupId>
<artifactId>eclipse-collections-api</artifactId>
Expand Down
69 changes: 65 additions & 4 deletions .../src/main/java/org/finos/legend/engine/ide/lsp/extension/AbstractLSPGrammarExtension.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,17 @@

package org.finos.legend.engine.ide.lsp.extension;

import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.core.StreamReadFeature;
import com.fasterxml.jackson.core.StreamWriteFeature;
import com.fasterxml.jackson.databind.json.JsonMapper;
import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClientBuilder;
import org.apache.http.util.EntityUtils;
import org.eclipse.collections.api.factory.Lists;
import org.eclipse.collections.api.factory.Maps;
import org.eclipse.collections.api.list.ImmutableList;
Expand Down Expand Up @@ -66,6 +76,7 @@
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.io.InputStream;
import java.io.PrintWriter;
import java.io.StringWriter;
import java.io.UncheckedIOException;
Expand All @@ -78,6 +89,8 @@ abstract class AbstractLSPGrammarExtension implements LegendLSPGrammarExtension
{
private static final Logger LOGGER = LoggerFactory.getLogger(AbstractLSPGrammarExtension.class);

private static final String ENGINE_SERVER_URL = System.getProperty("legend.engine.server.url");

private static final String RUN_TESTS_COMMAND_ID = "legend.testable.runTests";
private static final String RUN_TESTS_COMMAND_TITLE = "Run tests";

Expand All @@ -95,7 +108,7 @@ abstract class AbstractLSPGrammarExtension implements LegendLSPGrammarExtension
private final Map<String, ? extends TestableRunnerExtension> testableRunners = TestableRunnerExtensionLoader.getClassifierPathToTestableRunnerMap();
private final Map<Class<? extends PackageableElement>, String> classToClassifier = ProtocolToClassifierPathLoader.getProtocolClassToClassifierMap();

private ObjectMapper protocolMapper;
private JsonMapper protocolMapper;
private PureGrammarComposer composer;

@Override
Expand Down Expand Up @@ -504,18 +517,60 @@ protected PureModelContextData deserializePMCD(String json)
}
}

private ObjectMapper getProtocolMapper()
private JsonMapper getProtocolMapper()
{
synchronized (this)
{
if (this.protocolMapper == null)
{
this.protocolMapper = PureProtocolObjectMapperFactory.getNewObjectMapper();
this.protocolMapper = PureProtocolObjectMapperFactory.withPureProtocolExtensions(JsonMapper.builder()
.disable(StreamWriteFeature.AUTO_CLOSE_TARGET)
.disable(StreamReadFeature.AUTO_CLOSE_SOURCE)
.build());
}
return this.protocolMapper;
}
}

protected boolean isEngineServerConfigured()
{
return ENGINE_SERVER_URL != null;
}

protected <T> T postEngineServer(String path, Object payload, Class<T> responseType)
{
if (!isEngineServerConfigured())
{
throw new IllegalStateException("Engine server is not configured");
}
String url = ENGINE_SERVER_URL + path;
try
{
JsonMapper json = getProtocolMapper();
HttpPost httpPost = new HttpPost(url);
httpPost.setEntity(new StringEntity(json.writeValueAsString(payload), ContentType.APPLICATION_JSON));

try (CloseableHttpClient client = newHttpClient();
CloseableHttpResponse response = client.execute(httpPost))
{
HttpEntity entity = response.getEntity();
if (response.getStatusLine().getStatusCode() != 200)
{
String responseString = EntityUtils.toString(entity);
throw new RuntimeException("Engine server responded to " + url + " with status: " + response.getStatusLine().getStatusCode() + "\nresponse: " + responseString);
}
try (InputStream stream = entity.getContent())
{
return json.readValue(stream, responseType);
}
}
}
catch (IOException e)
{
throw new UncheckedIOException(e);
}
}

protected String toGrammar(PureModelContextData pmcd)
{
return getComposer().renderPureModelContextData(pmcd);
Expand All @@ -533,6 +588,12 @@ private PureGrammarComposer getComposer()
}
}

private CloseableHttpClient newHttpClient()
{
// TODO configure the client properly
return HttpClientBuilder.create().build();
}

protected LegendDeclaration getDeclaration(PackageableElement element)
{
String path = element.getPath();
Expand Down
85 changes: 85 additions & 0 deletions ...rc/main/java/org/finos/legend/engine/ide/lsp/extension/ConnectionLSPGrammarExtension.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,18 +15,33 @@
package org.finos.legend.engine.ide.lsp.extension;

import org.eclipse.collections.api.factory.Lists;
import org.eclipse.collections.api.factory.Maps;
import org.eclipse.collections.api.factory.Sets;
import org.eclipse.collections.api.list.ListIterable;
import org.eclipse.collections.api.map.MutableMap;
import org.eclipse.collections.api.set.MutableSet;
import org.finos.legend.engine.ide.lsp.extension.execution.LegendExecutionResult;
import org.finos.legend.engine.ide.lsp.extension.execution.LegendExecutionResult.Type;
import org.finos.legend.engine.ide.lsp.extension.state.SectionState;
import org.finos.legend.engine.language.pure.grammar.from.connection.ConnectionParser;
import org.finos.legend.engine.language.pure.grammar.from.extension.PureGrammarParserExtensionLoader;
import org.finos.legend.engine.language.pure.grammar.from.extension.PureGrammarParserExtensions;
import org.finos.legend.engine.protocol.pure.v1.model.context.PureModelContextData;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.PackageableElement;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.connection.PackageableConnection;
import org.finos.legend.engine.protocol.pure.v1.model.packageableElement.store.relational.connection.RelationalDatabaseConnection;

import java.util.Collections;
import java.util.Map;

/**
* Extension for the Connection grammar.
*/
public class ConnectionLSPGrammarExtension extends AbstractLegacyParserLSPGrammarExtension
{
private static final String GENERATE_DB_COMMAND_ID = "legend.service.generateDatabase";
private static final String GENERATE_DB_COMMAND_TITLE = "Generate database";

private final ListIterable<String> keywords;

public ConnectionLSPGrammarExtension()
Expand All @@ -41,6 +56,76 @@ public Iterable<? extends String> getKeywords()
return this.keywords;
}

@Override
protected void collectCommands(SectionState sectionState, PackageableElement element, CommandConsumer consumer)
{
super.collectCommands(sectionState, element, consumer);
if (isEngineServerConfigured() && (element instanceof PackageableConnection))
{
PackageableConnection packageableConn = (PackageableConnection) element;
if (packageableConn.connectionValue instanceof RelationalDatabaseConnection)
{
consumer.accept(GENERATE_DB_COMMAND_ID, GENERATE_DB_COMMAND_TITLE, packageableConn.sourceInformation);
}
}
}

@Override
public Iterable<? extends LegendExecutionResult> execute(SectionState section, String entityPath, String commandId, Map<String, String> executableArgs)
{
return GENERATE_DB_COMMAND_ID.equals(commandId) ?
generateDBFromConnection(section, entityPath) :
super.execute(section, entityPath, commandId, executableArgs);
}

private Iterable<? extends LegendExecutionResult> generateDBFromConnection(SectionState section, String entityPath)
{
if (!isEngineServerConfigured())
{
return Collections.singletonList(LegendExecutionResult.newResult(entityPath, Type.ERROR, "Engine server is not configured"));
}

PackageableElement element = getParseResult(section).getElement(entityPath);
if (!(element instanceof PackageableConnection))
{
return Collections.singletonList(LegendExecutionResult.newResult(entityPath, Type.ERROR, "Unable to find connection " + entityPath));
}

PackageableConnection packageableConn = (PackageableConnection) element;
if (!(packageableConn.connectionValue instanceof RelationalDatabaseConnection))
{
return Collections.singletonList(LegendExecutionResult.newResult(entityPath, Type.ERROR, "Not a " + RelationalDatabaseConnection.class.getSimpleName() + ": " + entityPath));
}

MutableMap<String, Object> input = buildInput(packageableConn);
PureModelContextData result = postEngineServer("/pure/v1/utilities/database/schemaExploration", input, PureModelContextData.class);
String code = toGrammar(result);
return Collections.singletonList(LegendExecutionResult.newResult(entityPath, Type.SUCCESS, code));
}

private MutableMap<String, Object> buildInput(PackageableConnection packageableConn)
{
MutableMap<String, String> pattern = Maps.mutable.with(
"catalog", "%",
"schemaPattern", "%",
"tablePattern", "%"
);
MutableMap<String, Object> config = Maps.mutable.with(
"enrichTables", true,
"enrichPrimaryKeys", true,
"enrichColumns", true,
"patterns", Lists.fixedSize.with(pattern)
);
MutableMap<String, String> targetDB = Maps.mutable.with(
"name", packageableConn.name + "Database",
"package", packageableConn._package
);
return Maps.mutable.with(
"config", config,
"connection", packageableConn.connectionValue,
"targetDatabase", targetDB);
}

private static ListIterable<String> findKeywords()
{
MutableSet<String> keywords = Sets.mutable.empty();
Expand Down

0 comments on commit 2d3fc72

Please sign in to comment.