diff --git a/legend-engine-ide-lsp-default-extensions/src/main/java/org/finos/legend/engine/ide/lsp/extension/AbstractLSPGrammarExtension.java b/legend-engine-ide-lsp-default-extensions/src/main/java/org/finos/legend/engine/ide/lsp/extension/AbstractLSPGrammarExtension.java index 3c9fb327..fb422804 100644 --- a/legend-engine-ide-lsp-default-extensions/src/main/java/org/finos/legend/engine/ide/lsp/extension/AbstractLSPGrammarExtension.java +++ b/legend-engine-ide-lsp-default-extensions/src/main/java/org/finos/legend/engine/ide/lsp/extension/AbstractLSPGrammarExtension.java @@ -30,6 +30,7 @@ import java.util.Optional; import java.util.ServiceLoader; import java.util.Set; +import java.util.concurrent.CompletableFuture; import java.util.function.Consumer; import java.util.stream.Collectors; import org.eclipse.collections.api.factory.Lists; @@ -326,18 +327,30 @@ public CompileResult getCompileResult(SectionState sectionState) { DocumentState documentState = sectionState.getDocumentState(); GlobalState globalState = documentState.getGlobalState(); - return globalState.getProperty(COMPILE_RESULT, () -> tryCompile(globalState, documentState, sectionState)); + // when looking for compile results, there might be another thread working on it already + // if current thread is the one that sets the completable future, then do the actual compilation and set it on the completable future + // if current thread does not set the completable future, then just join and wait for result from another thread + CompletableFuture maybeCompileResultFuture = new CompletableFuture<>(); + CompletableFuture compileResultFuture = globalState.getProperty(COMPILE_RESULT, () -> maybeCompileResultFuture); + + if (compileResultFuture == maybeCompileResultFuture) + { + compileResultFuture.complete(this.tryCompile(globalState, documentState, sectionState)); + } + + return compileResultFuture.join(); } protected CompileResult tryCompile(GlobalState globalState, DocumentState documentState, SectionState sectionState) { + long started = System.currentTimeMillis(); globalState.logInfo("Starting compilation"); PureModelContextData pureModelContextData = null; try { pureModelContextData = buildPureModelContextData(globalState); PureModel pureModel = Compiler.compile(pureModelContextData, DeploymentMode.PROD, ""); - globalState.logInfo("Compilation completed successfully"); + globalState.logInfo("Compilation completed successfully in " + (System.currentTimeMillis() - started) + "ms"); return new CompileResult(pureModel, pureModelContextData); } catch (EngineException e) @@ -345,11 +358,11 @@ protected CompileResult tryCompile(GlobalState globalState, DocumentState docume SourceInformation sourceInfo = e.getSourceInformation(); if (isValidSourceInfo(sourceInfo)) { - globalState.logInfo("Compilation completed with error " + "(" + sourceInfo.sourceId + " " + SourceInformationUtil.toLocation(sourceInfo) + "): " + e.getMessage()); + globalState.logInfo("Compilation completed in " + (System.currentTimeMillis() - started) + "ms with error " + "(" + sourceInfo.sourceId + " " + SourceInformationUtil.toLocation(sourceInfo) + "): " + e.getMessage()); } else { - globalState.logInfo("Compilation completed with error: " + e.getMessage()); + globalState.logInfo("Compilation completed in " + (System.currentTimeMillis() - started) + "ms with error: " + e.getMessage()); globalState.logWarning("Invalid source information for compilation error"); LOGGER.warn("Invalid source information in exception during compilation requested for section {} of {}: {}", sectionState.getSectionNumber(), documentState.getDocumentId(), (sourceInfo == null) ? null : sourceInfo.getMessage(), e); } diff --git a/legend-engine-ide-lsp-default-extensions/src/test/java/org/finos/legend/engine/ide/lsp/extension/AbstractLSPGrammarExtensionTest.java b/legend-engine-ide-lsp-default-extensions/src/test/java/org/finos/legend/engine/ide/lsp/extension/AbstractLSPGrammarExtensionTest.java index c64b8a78..2d5e3d94 100644 --- a/legend-engine-ide-lsp-default-extensions/src/test/java/org/finos/legend/engine/ide/lsp/extension/AbstractLSPGrammarExtensionTest.java +++ b/legend-engine-ide-lsp-default-extensions/src/test/java/org/finos/legend/engine/ide/lsp/extension/AbstractLSPGrammarExtensionTest.java @@ -21,7 +21,10 @@ import java.util.List; import java.util.Optional; import java.util.ServiceLoader; +import java.util.concurrent.CompletableFuture; +import java.util.function.BiFunction; import java.util.function.Consumer; +import java.util.function.Function; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -300,6 +303,28 @@ public void forEachDocumentState(Consumer consumer) this.docStates.forEachValue(consumer::accept); } + @Override + public CompletableFuture forEachDocumentStateParallel(Consumer consumer) + { + this.docStates.forEachValue(consumer::accept); + return CompletableFuture.completedFuture(null); + } + + @Override + public CompletableFuture> collectFromEachDocumentState(Function> func) + { + List results = this.docStates.stream().map(func).flatMap(List::stream).collect(Collectors.toList()); + return CompletableFuture.completedFuture(results); + } + + @Override + public CompletableFuture> collectFromEachDocumentSectionState(BiFunction> func) + { + List results = new ArrayList<>(); + this.docStates.stream().forEach(x -> x.forEachSectionState(s -> results.addAll(func.apply(x, s)))); + return CompletableFuture.completedFuture(results); + } + @Override public Collection getAvailableGrammarExtensions() { diff --git a/legend-engine-ide-lsp-extension-api/src/main/java/org/finos/legend/engine/ide/lsp/extension/state/GlobalState.java b/legend-engine-ide-lsp-extension-api/src/main/java/org/finos/legend/engine/ide/lsp/extension/state/GlobalState.java index d5dbac63..32bb094f 100644 --- a/legend-engine-ide-lsp-extension-api/src/main/java/org/finos/legend/engine/ide/lsp/extension/state/GlobalState.java +++ b/legend-engine-ide-lsp-extension-api/src/main/java/org/finos/legend/engine/ide/lsp/extension/state/GlobalState.java @@ -15,7 +15,11 @@ package org.finos.legend.engine.ide.lsp.extension.state; import java.util.Collection; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.function.BiFunction; import java.util.function.Consumer; +import java.util.function.Function; import java.util.stream.Stream; import org.finos.legend.engine.ide.lsp.extension.LegendLSPFeature; import org.finos.legend.engine.ide.lsp.extension.LegendLSPGrammarExtension; @@ -41,6 +45,26 @@ public interface GlobalState extends State */ void forEachDocumentState(Consumer consumer); + /** + * Apply the given consumer to each document state. No particular order is guaranteed. + * Implementation could do this in parallel + * + * @param consumer document state consumer, needs to be threadsafe + * @return future to track when this completes + */ + CompletableFuture forEachDocumentStateParallel(Consumer consumer); + + /** + * Apply the given function to each document state, collecting its result in a list. No particular order is guaranteed. + * Implementation could do this in parallel. + * + * @param func function to apply to each document state, needs to be threadsafe + * @return future to with the collected results + */ + CompletableFuture> collectFromEachDocumentState(Function> func); + + CompletableFuture> collectFromEachDocumentSectionState(BiFunction> func); + /** * List of available grammar extensions. This is useful for extensions that need to dispatch to other extensions * for further processing diff --git a/legend-engine-ide-lsp-server/src/main/java/org/finos/legend/engine/ide/lsp/server/LegendLanguageServer.java b/legend-engine-ide-lsp-server/src/main/java/org/finos/legend/engine/ide/lsp/server/LegendLanguageServer.java index de7113ed..b1818bfb 100644 --- a/legend-engine-ide-lsp-server/src/main/java/org/finos/legend/engine/ide/lsp/server/LegendLanguageServer.java +++ b/legend-engine-ide-lsp-server/src/main/java/org/finos/legend/engine/ide/lsp/server/LegendLanguageServer.java @@ -248,13 +248,14 @@ public String getProjectVersion() @Override public void initialized(InitializedParams params) { + long start = System.currentTimeMillis(); checkReady(); this.classpathFactory.initialize(this); CompletableFuture initializeExtensions = this.initializeExtensions(); CompletableFuture engineServerUrl = this.initializeEngineServerUrl(); CompletableFuture.allOf(initializeExtensions, engineServerUrl) - .thenRun(() -> this.logInfoToClient("Extension finished post-initialization")); + .thenRun(() -> this.logInfoToClient("Extension finished post-initialization in " + (System.currentTimeMillis() - start) + "ms")); } @@ -292,10 +293,16 @@ private CompletableFuture initializeExtensions() return this.classpathFactory.create(Collections.unmodifiableSet(this.rootFolders)) .thenAccept(this.extensionGuard::initialize) - .thenRun(this.extensionGuard.wrapOnClasspath(this::reprocessDocuments)) + .thenCompose(_x -> this.reprocessDocuments()) .thenRun(this.legendLanguageService::loadVirtualFileSystemContent) // trigger compilation - .thenRun(this.extensionGuard.wrapOnClasspath(() -> this.globalState.forEachDocumentState(this.textDocumentService::getLegendDiagnostics))) + .thenCompose(_x -> this.globalState.forEachDocumentStateParallel(x -> + { + long diagnosticStarted = System.currentTimeMillis(); + this.textDocumentService.getLegendDiagnostics(x); + LOGGER.info("Diagnostics computed for {} took {}ms", x.getDocumentId(), System.currentTimeMillis() - diagnosticStarted); + + })) .thenRun(() -> { LanguageClient languageClient = this.getLanguageClient(); @@ -314,10 +321,14 @@ private CompletableFuture initializeExtensions() }); } - private void reprocessDocuments() + private CompletableFuture reprocessDocuments() { - this.globalState.forEachDocumentState(x -> ((LegendServerGlobalState.LegendServerDocumentState) x).recreateSectionStates()); - this.globalState.clearProperties(); + return this.globalState.forEachDocumentStateParallel(x -> + { + long startTime = System.currentTimeMillis(); + ((LegendServerGlobalState.LegendServerDocumentState) x).recreateSectionStates(); + LOGGER.info("Reprocessing {} took {}ms", x.getDocumentId(), System.currentTimeMillis() - startTime); + }).thenRun(this.globalState::clearProperties); } @Override @@ -484,10 +495,10 @@ public CompletableFuture supplyPossiblyAsync(Supplier supplier) return this.supplyPossiblyAsync_internal(this.extensionGuard.wrapOnClasspath(supplier)); } - void runPossiblyAsync(Runnable runnable) + CompletableFuture runPossiblyAsync(Runnable runnable) { checkReady(); - this.runPossiblyAsync_internal(this.extensionGuard.wrapOnClasspath(runnable)); + return this.runPossiblyAsync_internal(this.extensionGuard.wrapOnClasspath(runnable)); } private CompletableFuture runPossiblyAsync_internal(Runnable work) diff --git a/legend-engine-ide-lsp-server/src/main/java/org/finos/legend/engine/ide/lsp/server/LegendLanguageService.java b/legend-engine-ide-lsp-server/src/main/java/org/finos/legend/engine/ide/lsp/server/LegendLanguageService.java index f45637ab..8e4476a1 100644 --- a/legend-engine-ide-lsp-server/src/main/java/org/finos/legend/engine/ide/lsp/server/LegendLanguageService.java +++ b/legend-engine-ide-lsp-server/src/main/java/org/finos/legend/engine/ide/lsp/server/LegendLanguageService.java @@ -120,24 +120,17 @@ public CompletableFuture replClasspath() @Override public CompletableFuture> testCases() { - return this.server.supplyPossiblyAsync(() -> - { - List commands = new ArrayList<>(); - - this.server.getGlobalState().forEachDocumentState(docState -> - { - docState.forEachSectionState(sectionState -> + return this.server.getGlobalState().collectFromEachDocumentSectionState((docState, sectionState) -> { + List commands = new ArrayList<>(); LegendLSPGrammarExtension extension = sectionState.getExtension(); if (extension != null) { commands.addAll(extension.testCases(sectionState)); } - }); - }); - - return commands; - }); + return commands; + } + ); } @Override diff --git a/legend-engine-ide-lsp-server/src/main/java/org/finos/legend/engine/ide/lsp/server/LegendServerGlobalState.java b/legend-engine-ide-lsp-server/src/main/java/org/finos/legend/engine/ide/lsp/server/LegendServerGlobalState.java index da0185cf..c6153783 100644 --- a/legend-engine-ide-lsp-server/src/main/java/org/finos/legend/engine/ide/lsp/server/LegendServerGlobalState.java +++ b/legend-engine-ide-lsp-server/src/main/java/org/finos/legend/engine/ide/lsp/server/LegendServerGlobalState.java @@ -26,8 +26,12 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; +import java.util.function.BiFunction; import java.util.function.Consumer; +import java.util.function.Function; +import java.util.stream.Collectors; import java.util.stream.Stream; import org.finos.legend.engine.ide.lsp.extension.LegendLSPFeature; import org.finos.legend.engine.ide.lsp.extension.LegendLSPGrammarExtension; @@ -64,6 +68,37 @@ public void forEachDocumentState(Consumer consumer) this.docs.values().forEach(consumer); } + @Override + public CompletableFuture forEachDocumentStateParallel(Consumer consumer) + { + return CompletableFuture.allOf( + this.docs.values() + .stream() + .map(x -> this.server.runPossiblyAsync(() -> consumer.accept(x))) + .toArray(CompletableFuture[]::new) + ); + } + + @Override + public CompletableFuture> collectFromEachDocumentState(Function> func) + { + return this.docs.values() + .stream() + .map(x -> this.server.supplyPossiblyAsync(() -> func.apply(x))) + .reduce((x, y) -> x.thenCombine(y, (r, l) -> Stream.concat(r.stream(), l.stream()).collect(Collectors.toList()))) + .orElseGet(() -> CompletableFuture.completedFuture(List.of())); + } + + @Override + public CompletableFuture> collectFromEachDocumentSectionState(BiFunction> func) + { + return this.docs.values() + .stream() + .flatMap(x -> x.collectFromEachSectionState(func)) + .reduce((x, y) -> x.thenCombine(y, (r, l) -> Stream.concat(r.stream(), l.stream()).collect(Collectors.toList()))) + .orElseGet(() -> CompletableFuture.completedFuture(List.of())); + } + @Override public void logInfo(String message) { @@ -247,6 +282,16 @@ public void forEachSectionState(Consumer consumer) } } + private Stream>> collectFromEachSectionState(BiFunction> func) + { + List currentSectionsStates = this.sectionStates; + if (currentSectionsStates != null) + { + return currentSectionsStates.stream().map(x -> this.globalState.server.supplyPossiblyAsync(() -> func.apply(this, x))); + } + return Stream.empty(); + } + Integer getVersion() { return version; diff --git a/legend-engine-ide-lsp-server/src/main/java/org/finos/legend/engine/ide/lsp/server/LegendWorkspaceService.java b/legend-engine-ide-lsp-server/src/main/java/org/finos/legend/engine/ide/lsp/server/LegendWorkspaceService.java index becb6c36..5ef0ca20 100644 --- a/legend-engine-ide-lsp-server/src/main/java/org/finos/legend/engine/ide/lsp/server/LegendWorkspaceService.java +++ b/legend-engine-ide-lsp-server/src/main/java/org/finos/legend/engine/ide/lsp/server/LegendWorkspaceService.java @@ -21,6 +21,7 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; import org.eclipse.lsp4j.CreateFilesParams; @@ -310,27 +311,17 @@ private void fileRenamed(LegendServerGlobalState globalState, String oldUri, Str @Override public CompletableFuture, List>> symbol(WorkspaceSymbolParams params) { - return this.server.supplyPossiblyAsync(() -> getWorkspaceSymbols(params)); - } - - private Either, List> getWorkspaceSymbols(WorkspaceSymbolParams params) - { - List symbols = new ArrayList<>(); - - this.server.getGlobalState().forEachDocumentState(doc -> + return this.server.getGlobalState().collectFromEachDocumentSectionState((doc, sec) -> { - doc.forEachSectionState(sec -> + List symbols = new ArrayList<>(); + if (sec.getExtension() != null) { - if (sec.getExtension() != null) - { - sec.getExtension() - .getDeclarations(sec) - .forEach(declaration -> toWorkspaceSymbol(params, symbols, doc, declaration, null)); - } - }); - }); - - return Either.forRight(symbols); + sec.getExtension() + .getDeclarations(sec) + .forEach(declaration -> toWorkspaceSymbol(params, symbols, doc, declaration, null)); + } + return symbols; + }).thenApply(Either::forRight); } private static void toWorkspaceSymbol(WorkspaceSymbolParams params, List symbols, DocumentState doc, LegendDeclaration declaration, WorkspaceSymbol parent) @@ -368,65 +359,63 @@ private static void toWorkspaceSymbol(WorkspaceSymbolParams params, List diagnostic(WorkspaceDiagnosticParams params) { - return this.server.supplyPossiblyAsync(() -> - { - Map previousResultIds = params.getPreviousResultIds() - .stream() - .collect(Collectors.toMap(PreviousResultId::getUri, PreviousResultId::getValue)); + Map previousResultIds = params.getPreviousResultIds() + .stream() + .collect(Collectors.toMap(PreviousResultId::getUri, PreviousResultId::getValue)); - Map> previousResultIdToDiagnostic = this.previousResultIdToDiagnosticReference.get(); - Map> resultIdToDiagnostic = new HashMap<>(); + Map> previousResultIdToDiagnostic = this.previousResultIdToDiagnosticReference.get(); + Map> resultIdToDiagnostic = new ConcurrentHashMap<>(); - List items = new ArrayList<>(); - - this.server.getGlobalState().forEachDocumentState(d -> - { - String previousResultId = previousResultIds.getOrDefault(d.getDocumentId(), ""); - Set prevDiagnostic = previousResultIdToDiagnostic.get(previousResultId); + return this.server.getGlobalState().collectFromEachDocumentState(d -> + { + String previousResultId = previousResultIds.getOrDefault(d.getDocumentId(), ""); + Set prevDiagnostic = previousResultIdToDiagnostic.get(previousResultId); - LegendServerGlobalState.LegendServerDocumentState doc = (LegendServerGlobalState.LegendServerDocumentState) d; - Set diagnostics = this.server.getTextDocumentService().getLegendDiagnostics(doc); + LegendServerGlobalState.LegendServerDocumentState doc = (LegendServerGlobalState.LegendServerDocumentState) d; + Set diagnostics = this.server.getTextDocumentService().getLegendDiagnostics(doc); - String publishResultId = null; + String publishResultId = null; - // no previous results - if (prevDiagnostic == null) - { - // there are new diagnostics - if (!diagnostics.isEmpty()) + // no previous results + if (prevDiagnostic == null) { - publishResultId = UUID.randomUUID().toString(); - resultIdToDiagnostic.put(publishResultId, diagnostics); + // there are new diagnostics + if (!diagnostics.isEmpty()) + { + publishResultId = UUID.randomUUID().toString(); + resultIdToDiagnostic.put(publishResultId, diagnostics); + } } - } - // there are previous results - else - { - // diagnostics are different between previous and now - if (!diagnostics.equals(prevDiagnostic)) + // there are previous results + else { - publishResultId = UUID.randomUUID().toString(); - resultIdToDiagnostic.put(publishResultId, diagnostics); + // diagnostics are different between previous and now + if (!diagnostics.equals(prevDiagnostic)) + { + publishResultId = UUID.randomUUID().toString(); + resultIdToDiagnostic.put(publishResultId, diagnostics); + } + // only track old diagnostics if same as new and non-empty (ie don't track empty ones) + // otherwise, next time prevDiagnostic will be null, and only we start tracking again if there are new diagnostics + else if (!diagnostics.isEmpty()) + { + resultIdToDiagnostic.put(previousResultId, diagnostics); + } } - // only track old diagnostics if same as new and non-empty (ie don't track empty ones) - // otherwise, next time prevDiagnostic will be null, and only we start tracking again if there are new diagnostics - else if (!diagnostics.isEmpty()) + + if (publishResultId != null) { - resultIdToDiagnostic.put(previousResultId, diagnostics); + WorkspaceFullDocumentDiagnosticReport fullReport = new WorkspaceFullDocumentDiagnosticReport(diagnostics.stream().map(LegendToLSPUtilities::toDiagnostic).collect(Collectors.toList()), doc.getDocumentId(), doc.getVersion()); + fullReport.setResultId(publishResultId); + return List.of(new WorkspaceDocumentDiagnosticReport(fullReport)); } - } - if (publishResultId != null) - { - WorkspaceFullDocumentDiagnosticReport fullReport = new WorkspaceFullDocumentDiagnosticReport(diagnostics.stream().map(LegendToLSPUtilities::toDiagnostic).collect(Collectors.toList()), doc.getDocumentId(), doc.getVersion()); - fullReport.setResultId(publishResultId); - items.add(new WorkspaceDocumentDiagnosticReport(fullReport)); + return List.of(); } - }); - + ).thenApply(x -> + { this.previousResultIdToDiagnosticReference.compareAndSet(previousResultIdToDiagnostic, resultIdToDiagnostic); - - return new WorkspaceDiagnosticReport(items); + return new WorkspaceDiagnosticReport(x); }); } }