Semantic tokenizer improvements: tests, simplification, syntax-in-syntax bugfix

rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/parametric/ILanguageContributions.java

@@ -73,6 +73,8 @@ public interface ILanguageContributions {
     public CompletableFuture<Boolean> hasImplementation();
     public CompletableFuture<Boolean> hasCodeAction();
 
+    public CompletableFuture<Boolean> specialCaseHighlighting();
+
     public CompletableFuture<SummaryConfig> getAnalyzerSummaryConfig();
     public CompletableFuture<SummaryConfig> getBuilderSummaryConfig();
     public CompletableFuture<SummaryConfig> getOndemandSummaryConfig();

rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/parametric/InterpretedLanguageContributions.java

@@ -100,6 +100,8 @@ public class InterpretedLanguageContributions implements ILanguageContributions
     private final CompletableFuture<Boolean> hasImplementation;
     private final CompletableFuture<Boolean> hasCodeAction;
 
+    private final CompletableFuture<Boolean> specialCaseHighlighting;
+
     private final CompletableFuture<SummaryConfig> analyzerSummaryConfig;
     private final CompletableFuture<SummaryConfig> builderSummaryConfig;
     private final CompletableFuture<SummaryConfig> ondemandSummaryConfig;
@@ -151,6 +153,10 @@ public InterpretedLanguageContributions(LanguageParameter lang, IBaseTextDocumen
             this.hasImplementation = nonNull(this.implementation);
             this.hasCodeAction = nonNull(this.codeAction);
 
+            this.specialCaseHighlighting = getContributionParameter(contributions,
+                LanguageContributions.PARSING,
+                LanguageContributions.Parameters.USES_SPECIAL_CASE_HIGHLIGHTING);
+
             this.analyzerSummaryConfig = scheduledSummaryConfig(contributions, LanguageContributions.ANALYSIS);
             this.builderSummaryConfig = scheduledSummaryConfig(contributions, LanguageContributions.BUILD);
             this.ondemandSummaryConfig = ondemandSummaryConfig(contributions);
@@ -170,10 +176,10 @@ private static CompletableFuture<SummaryConfig> scheduledSummaryConfig(Completab
             var constructor = getContribution(c, summarizer);
             if (constructor != null) {
                 return new SummaryConfig(
-                    isTrue(constructor, LanguageContributions.Summarizers.PROVIDES_HOVERS),
-                    isTrue(constructor, LanguageContributions.Summarizers.PROVIDES_DEFINITIONS),
-                    isTrue(constructor, LanguageContributions.Summarizers.PROVIDES_REFERENCES),
-                    isTrue(constructor, LanguageContributions.Summarizers.PROVIDES_IMPLEMENTATIONS));
+                    isTrue(constructor, LanguageContributions.Parameters.PROVIDES_HOVERS),
+                    isTrue(constructor, LanguageContributions.Parameters.PROVIDES_DEFINITIONS),
+                    isTrue(constructor, LanguageContributions.Parameters.PROVIDES_REFERENCES),
+                    isTrue(constructor, LanguageContributions.Parameters.PROVIDES_IMPLEMENTATIONS));
             } else {
                 return SummaryConfig.FALSY;
             }
@@ -202,6 +208,12 @@ private static boolean hasContribution(ISet contributions, String name) {
         return getContribution(contributions, name) != null;
     }
 
+    private static CompletableFuture<Boolean> getContributionParameter(
+            CompletableFuture<ISet> contributions, String name, String parameter) {
+
+        return contributions.thenApply(c -> isTrue(getContribution(c, name), parameter));
+    }
+
     private static boolean isTrue(@Nullable IConstructor constructor, String parameter) {
         if (constructor == null) {
             return false;
@@ -388,6 +400,11 @@ public CompletableFuture<Boolean> hasBuild() {
         return hasBuild;
     }
 
+    @Override
+    public CompletableFuture<Boolean> specialCaseHighlighting() {
+        return specialCaseHighlighting;
+    }
+
     @Override
     public CompletableFuture<SummaryConfig> getAnalyzerSummaryConfig() {
         return analyzerSummaryConfig;

rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/parametric/LanguageContributionsMultiplexer.java

@@ -77,6 +77,8 @@ private static final <T> CompletableFuture<T> failedInitialization() {
     private volatile CompletableFuture<Boolean> hasImplementation = failedInitialization();
     private volatile CompletableFuture<Boolean> hasCodeAction = failedInitialization();
 
+    private volatile CompletableFuture<Boolean> specialCaseHighlighting = failedInitialization();
+
     private volatile CompletableFuture<SummaryConfig> analyzerSummaryConfig;
     private volatile CompletableFuture<SummaryConfig> builderSummaryConfig;
     private volatile CompletableFuture<SummaryConfig> ondemandSummaryConfig;
@@ -159,6 +161,11 @@ private synchronized void calculateRouting() {
         hasReferences = anyTrue(ILanguageContributions::hasReferences);
         hasImplementation = anyTrue(ILanguageContributions::hasImplementation);
 
+        // Always use the special-case highlighting status of *the first*
+        // contribution (possibly using the default value in the Rascal ADT if
+        // it's not explicitly set), just as for `parsing` itself
+        specialCaseHighlighting = firstOrFail().specialCaseHighlighting();
+
         analyzerSummaryConfig = anyTrue(ILanguageContributions::getAnalyzerSummaryConfig, SummaryConfig.FALSY, SummaryConfig::or);
         builderSummaryConfig = anyTrue(ILanguageContributions::getBuilderSummaryConfig, SummaryConfig.FALSY, SummaryConfig::or);
         ondemandSummaryConfig = anyTrue(ILanguageContributions::getOndemandSummaryConfig, SummaryConfig.FALSY, SummaryConfig::or);
@@ -343,6 +350,11 @@ public CompletableFuture<Boolean> hasInlayHint() {
         return hasInlayHint;
     }
 
+    @Override
+    public CompletableFuture<Boolean> specialCaseHighlighting() {
+        return specialCaseHighlighting;
+    }
+
     @Override
     public CompletableFuture<SummaryConfig> getAnalyzerSummaryConfig() {
         return analyzerSummaryConfig;

rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/parametric/ParametricTextDocumentService.java

@@ -462,9 +462,10 @@ public void shutdown() {
     }
 
     private CompletableFuture<SemanticTokens> getSemanticTokens(TextDocumentIdentifier doc) {
+        var specialCaseHighlighting = contributions(doc).specialCaseHighlighting();
         return recoverExceptions(getFile(doc).getCurrentTreeAsync()
                 .thenApply(Versioned::get)
-                .thenApplyAsync(tokenizer::semanticTokensFull, ownExecuter)
+                .thenCombineAsync(specialCaseHighlighting, tokenizer::semanticTokensFull, ownExecuter)
                 .whenComplete((r, e) ->
                     logger.trace("Semantic tokens success, reporting {} tokens back", r == null ? 0 : r.getData().size() / 5)
                 )

rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/parametric/ParserOnlyContribution.java

@@ -59,6 +59,7 @@ public class ParserOnlyContribution implements ILanguageContributions {
     private final String name;
     private final @Nullable Exception loadingParserError;
     private final @Nullable IFunction parser;
+    private final CompletableFuture<Boolean> specialCaseHighlighting;
 
     public ParserOnlyContribution(String name, ParserSpecification spec) {
         this.name = name;
@@ -67,6 +68,7 @@ public ParserOnlyContribution(String name, ParserSpecification spec) {
         Either<IFunction,Exception> result = loadParser(spec);
         this.parser = result.getLeft();
         this.loadingParserError = result.getRight();
+        this.specialCaseHighlighting = CompletableFuture.completedFuture(spec.getSpecialCaseHighlighting());
     }
 
     @Override
@@ -228,6 +230,11 @@ public CompletableFuture<Boolean> hasInlayHint() {
         return CompletableFuture.completedFuture(false);
     }
 
+    @Override
+    public CompletableFuture<Boolean> specialCaseHighlighting() {
+        return specialCaseHighlighting;
+    }
+
     @Override
     public CompletableFuture<SummaryConfig> getAnalyzerSummaryConfig() {
         return CompletableFuture.completedFuture(SummaryConfig.FALSY);

rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/parametric/model/RascalADTs.java

@@ -47,8 +47,10 @@ private LanguageContributions () {}
         public static final String IMPLEMENTATION = "implementation";
         public static final String CODE_ACTION = "codeAction";
 
-        public static class Summarizers {
-            private Summarizers() {}
+        public static class Parameters {
+            private Parameters() {}
+
+            public static final String USES_SPECIAL_CASE_HIGHLIGHTING = "usesSpecialCaseHighlighting";
 
             public static final String PROVIDES_HOVERS = "providesHovers";
             public static final String PROVIDES_DEFINITIONS = "providesDefinitions";

rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/rascal/RascalTextDocumentService.java

@@ -361,9 +361,10 @@ public void shutdown() {
     }
 
     private CompletableFuture<SemanticTokens> getSemanticTokens(TextDocumentIdentifier doc) {
+        var specialCaseHighlighting = CompletableFuture.completedFuture(false);
         return getFile(doc).getCurrentTreeAsync()
                 .thenApply(Versioned::get)
-                .thenApplyAsync(tokenizer::semanticTokensFull, ownExecuter)
+                .thenCombineAsync(specialCaseHighlighting, tokenizer::semanticTokensFull, ownExecuter)
                 .exceptionally(e -> {
                     logger.error("Tokenization failed", e);
                     return new SemanticTokens(Collections.emptyList());

rascal-lsp/src/main/java/org/rascalmpl/vscode/lsp/terminal/ITerminalIDEServer.java

@@ -528,13 +528,17 @@ public static class ParserSpecification {
         private final @Nullable Boolean nonTerminalIsStart;
         /** allowAmbiguity (default is false) */
         private final @Nullable Boolean allowAmbiguity;
+        /** apply the special case for highlighting syntax-in-syntax, default: true */
+        private final @Nullable Boolean specialCaseHighlighting;
 
 
-        public ParserSpecification(String parserLocation, String nonTerminalName, @Nullable Boolean nonTerminalIsStart, @Nullable Boolean allowAmbiguity) {
+        public ParserSpecification(String parserLocation, String nonTerminalName,
+                @Nullable Boolean nonTerminalIsStart, @Nullable Boolean allowAmbiguity, @Nullable Boolean specialCaseHighlighting) {
             this.parserLocation = parserLocation;
             this.nonTerminalName = nonTerminalName;
             this.nonTerminalIsStart = nonTerminalIsStart;
             this.allowAmbiguity = allowAmbiguity;
+            this.specialCaseHighlighting = specialCaseHighlighting;
         }
 
         public ISourceLocation getParserLocation() throws FactTypeUseException {
@@ -553,6 +557,10 @@ public boolean getAllowAmbiguity() {
             return allowAmbiguity != null && allowAmbiguity;
         }
 
+        public boolean getSpecialCaseHighlighting() {
+            return specialCaseHighlighting == null || specialCaseHighlighting;
+        }
+
         @Override
         public String toString() {
             return "ParserSpecification [parserLocation=" + parserLocation + ", nonTerminalName=" + nonTerminalName