From 93faa5b5d9e3766fd5c67b3dd47b29c389e26dcf Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Mon, 19 Jun 2023 13:31:37 +0200 Subject: [PATCH 01/16] removed public from all functions in ParseTree --- src/org/rascalmpl/library/ParseTree.rsc | 63 +++++++++---------- .../test/infrastructure/QuickCheck.java | 2 - 2 files changed, 31 insertions(+), 34 deletions(-) diff --git a/src/org/rascalmpl/library/ParseTree.rsc b/src/org/rascalmpl/library/ParseTree.rsc index 482dc4a3825..2e0ba1f7362 100644 --- a/src/org/rascalmpl/library/ParseTree.rsc +++ b/src/org/rascalmpl/library/ParseTree.rsc @@ -310,7 +310,7 @@ data Symbol data Symbol // <19> = \conditional(Symbol symbol, set[Condition] conditions); -public bool subtype(Symbol::\sort(_), Symbol::\adt("Tree", _)) = true; +bool subtype(Symbol::\sort(_), Symbol::\adt("Tree", _)) = true; @doc{ #### Synopsis @@ -341,7 +341,7 @@ data Condition Nested priority is flattened. } -public Production priority(Symbol s, [*Production a, priority(Symbol _, list[Production] b), *Production c]) +Production priority(Symbol s, [*Production a, priority(Symbol _, list[Production] b), *Production c]) = priority(s,a+b+c); @doc{ @@ -446,13 +446,13 @@ catch ParseError(loc l): { } ``` } -public &T<:Tree parse(type[&T<:Tree] begin, str input, bool allowAmbiguity=false, bool hasSideEffects=false, set[Tree(Tree)] filters={}) +&T<:Tree parse(type[&T<:Tree] begin, str input, bool allowAmbiguity=false, bool hasSideEffects=false, set[Tree(Tree)] filters={}) = parser(begin, allowAmbiguity=allowAmbiguity, hasSideEffects=hasSideEffects, filters=filters)(input, |unknown:///|); -public &T<:Tree parse(type[&T<:Tree] begin, str input, loc origin, bool allowAmbiguity=false, bool hasSideEffects=false, set[Tree(Tree)] filters={}) +&T<:Tree parse(type[&T<:Tree] begin, str input, loc origin, bool allowAmbiguity=false, bool hasSideEffects=false, set[Tree(Tree)] filters={}) = parser(begin, allowAmbiguity=allowAmbiguity, hasSideEffects=hasSideEffects, filters=filters)(input, origin); -public &T<:Tree parse(type[&T<:Tree] begin, loc input, bool allowAmbiguity=false, bool hasSideEffects=false, set[Tree(Tree)] filters={}) +&T<:Tree parse(type[&T<:Tree] begin, loc input, bool allowAmbiguity=false, bool hasSideEffects=false, set[Tree(Tree)] filters={}) = parser(begin, allowAmbiguity=allowAmbiguity, hasSideEffects=hasSideEffects, filters=filters)(input, input); @doc{ @@ -485,7 +485,7 @@ The parse function behaves differently depending of the given keyword parameters * parse forest to be constructed in polynomial time. } @javaClass{org.rascalmpl.library.Prelude} -public java &T (value input, loc origin) parser(type[&T] grammar, bool allowAmbiguity=false, bool hasSideEffects=false, bool firstAmbiguity=false, set[Tree(Tree)] filters={}); +java &T (value input, loc origin) parser(type[&T] grammar, bool allowAmbiguity=false, bool hasSideEffects=false, bool firstAmbiguity=false, set[Tree(Tree)] filters={}); @doc{ #### Synopsis @@ -498,7 +498,7 @@ This parser generator behaves the same as the `parser` function, but it produces nonterminal parameter. This can be used to select a specific non-terminal from the grammar to use as start-symbol for parsing. } @javaClass{org.rascalmpl.library.Prelude} -public java &U (type[&U] nonterminal, value input, loc origin) parsers(type[&T] grammar, bool allowAmbiguity=false, bool hasSideEffects=false, bool firstAmbiguity=false, set[Tree(Tree)] filters={}); +java &U (type[&U] nonterminal, value input, loc origin) parsers(type[&T] grammar, bool allowAmbiguity=false, bool hasSideEffects=false, bool firstAmbiguity=false, set[Tree(Tree)] filters={}); @synopsis{Parse the input but instead of returning the entire tree, return the trees for the first ambiguous substring.} @description{ @@ -508,13 +508,12 @@ the cost of constructing nested ambiguity clusters. If the input sentence is not ambiguous after all, simply the entire tree is returned. } -public Tree firstAmbiguity(type[Tree] begin, str input) +Tree firstAmbiguity(type[Tree] begin, str input) = parser(begin, firstAmbiguity=true)(input, |unknown:///|); -public Tree firstAmbiguity(type[Tree] begin, loc input) +Tree firstAmbiguity(type[Tree] begin, loc input) = parser(begin, firstAmbiguity=true)(input, input); - @doc{ #### Synopsis @@ -540,10 +539,10 @@ First parse an expression, this results in a parse tree. Then unparse this parse unparse(parse(#Exp, "2+3")); ``` } -public str unparse(Tree tree) = ""; +str unparse(Tree tree) = ""; @javaClass{org.rascalmpl.library.Prelude} -public java str printSymbol(Symbol sym, bool withLayout); +java str printSymbol(Symbol sym, bool withLayout); @javaClass{org.rascalmpl.library.Prelude} @doc{ @@ -682,21 +681,21 @@ Can be imploded into: data Exp = add(Exp, Exp); ``` } -public java &T<:value implode(type[&T<:value] t, Tree tree); +java &T<:value implode(type[&T<:value] t, Tree tree); @doc{ #### Synopsis Annotate a parse tree node with an (error) message. } -public anno Message Tree@message; +anno Message Tree@message; @doc{ #### Synopsis Annotate a parse tree node with a list of (error) messages. } -public anno set[Message] Tree@messages; +anno set[Message] Tree@messages; @doc{ #### Synopsis @@ -742,7 +741,7 @@ anno rel[loc,loc] Tree@hyperlinks; Tree search result type for ((treeAt)). } -public data TreeSearchResult[&T<:Tree] = treeFound(&T tree) | treeNotFound(); +data TreeSearchResult[&T<:Tree] = treeFound(&T tree) | treeNotFound(); @doc{ @@ -753,7 +752,7 @@ Select the innermost Tree of a given type which is enclosed by a given location. #### Description } -public TreeSearchResult[&T<:Tree] treeAt(type[&T<:Tree] t, loc l, Tree a:appl(_, _)) { +TreeSearchResult[&T<:Tree] treeAt(type[&T<:Tree] t, loc l, Tree a:appl(_, _)) { if ((a@\loc)?, al := a@\loc, al.offset <= l.offset, al.offset + al.length >= l.offset + l.length) { for (arg <- a.args, TreeSearchResult[&T<:Tree] r:treeFound(&T<:Tree _) := treeAt(t, l, arg)) { return r; @@ -766,25 +765,25 @@ public TreeSearchResult[&T<:Tree] treeAt(type[&T<:Tree] t, loc l, Tree a:appl(_, return treeNotFound(); } -public default TreeSearchResult[&T<:Tree] treeAt(type[&T<:Tree] t, loc l, Tree root) = treeNotFound(); +default TreeSearchResult[&T<:Tree] treeAt(type[&T<:Tree] t, loc l, Tree root) = treeNotFound(); -public bool sameType(label(_,Symbol s),Symbol t) = sameType(s,t); -public bool sameType(Symbol s,label(_,Symbol t)) = sameType(s,t); -public bool sameType(Symbol s,conditional(Symbol t,_)) = sameType(s,t); -public bool sameType(conditional(Symbol s,_), Symbol t) = sameType(s,t); -public bool sameType(Symbol s, s) = true; -public default bool sameType(Symbol s, Symbol t) = false; +bool sameType(label(_,Symbol s),Symbol t) = sameType(s,t); +bool sameType(Symbol s,label(_,Symbol t)) = sameType(s,t); +bool sameType(Symbol s,conditional(Symbol t,_)) = sameType(s,t); +bool sameType(conditional(Symbol s,_), Symbol t) = sameType(s,t); +bool sameType(Symbol s, s) = true; +default bool sameType(Symbol s, Symbol t) = false; @doc{ #### Synopsis Determine if the given type is a non-terminal type. } -public bool isNonTerminalType(Symbol::\sort(str _)) = true; -public bool isNonTerminalType(Symbol::\lex(str _)) = true; -public bool isNonTerminalType(Symbol::\layouts(str _)) = true; -public bool isNonTerminalType(Symbol::\keywords(str _)) = true; -public bool isNonTerminalType(Symbol::\parameterized-sort(str _, list[Symbol] _)) = true; -public bool isNonTerminalType(Symbol::\parameterized-lex(str _, list[Symbol] _)) = true; -public bool isNonTerminalType(Symbol::\start(Symbol s)) = isNonTerminalType(s); -public default bool isNonTerminalType(Symbol s) = false; +bool isNonTerminalType(Symbol::\sort(str _)) = true; +bool isNonTerminalType(Symbol::\lex(str _)) = true; +bool isNonTerminalType(Symbol::\layouts(str _)) = true; +bool isNonTerminalType(Symbol::\keywords(str _)) = true; +bool isNonTerminalType(Symbol::\parameterized-sort(str _, list[Symbol] _)) = true; +bool isNonTerminalType(Symbol::\parameterized-lex(str _, list[Symbol] _)) = true; +bool isNonTerminalType(Symbol::\start(Symbol s)) = isNonTerminalType(s); +default bool isNonTerminalType(Symbol s) = false; diff --git a/src/org/rascalmpl/test/infrastructure/QuickCheck.java b/src/org/rascalmpl/test/infrastructure/QuickCheck.java index 48bc5ba23c1..25ba7105bd1 100644 --- a/src/org/rascalmpl/test/infrastructure/QuickCheck.java +++ b/src/org/rascalmpl/test/infrastructure/QuickCheck.java @@ -225,6 +225,4 @@ public ExceptionNotThrownResult(String functionName, Type[] actualTypes, super(functionName, "test did not throw '" + expected + "' exception", actualTypes, tpbindings, values); } } - - } From fb319d7b8012e6c3d8d8dffbb05877a64b881caf Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Tue, 20 Jun 2023 11:18:22 +0200 Subject: [PATCH 02/16] added interface for saving and loading parsers from disk without exposing the file internals (could be class files or grammar values, etc.) --- .../interpreter/utils/JavaBridge.java | 26 +++++++++++++ .../interpreter/utils/JavaCompiler.java | 38 ++++++++++++++++++- src/org/rascalmpl/library/ParseTree.rsc | 15 ++++++++ src/org/rascalmpl/library/Prelude.java | 13 +++++++ src/org/rascalmpl/library/Relation.rsc | 4 +- src/org/rascalmpl/parser/ParserGenerator.java | 37 ++++++++++++++++++ .../rascalmpl/values/IRascalValueFactory.java | 20 +++++++++- .../values/RascalFunctionValueFactory.java | 25 ++++++++++++ 8 files changed, 172 insertions(+), 6 deletions(-) diff --git a/src/org/rascalmpl/interpreter/utils/JavaBridge.java b/src/org/rascalmpl/interpreter/utils/JavaBridge.java index 0569e92c43b..1dbf6bc8d91 100644 --- a/src/org/rascalmpl/interpreter/utils/JavaBridge.java +++ b/src/org/rascalmpl/interpreter/utils/JavaBridge.java @@ -107,6 +107,11 @@ public JavaBridge(List classLoaders, IValueFactory valueFactory, Co public Class compileJava(ISourceLocation loc, String className, String source) { return compileJava(loc, className, getClass(), source); } + + public void compileJava(ISourceLocation loc, String className, String source, OutputStream classBytes) { + compileJava(loc, className, getClass(), source); + } + public Class compileJava(ISourceLocation loc, String className, Class parent, String source) { try { @@ -130,6 +135,27 @@ public Class compileJava(ISourceLocation loc, String className, Class } } + public void compileJava(ISourceLocation loc, String className, Class parent, String source, OutputStream classBytes) { + try { + // watch out, if you start sharing this compiler, classes will not be able to reload + List commandline = Arrays.asList(new String[] {"-proc:none", "-cp", config.getRascalJavaClassPathProperty()}); + JavaCompiler javaCompiler = new JavaCompiler(parent.getClassLoader(), null, commandline); + javaCompiler.compile(classBytes, className, source, null); + } + catch (ClassCastException e) { + throw new JavaCompilation(e.getMessage(), e); + } + catch (JavaCompilerException e) { + if (!e.getDiagnostics().getDiagnostics().isEmpty()) { + Diagnostic msg = e.getDiagnostics().getDiagnostics().iterator().next(); + throw new JavaCompilation(msg.getMessage(null) + " at " + msg.getLineNumber() + ", " + msg.getColumnNumber() + " with classpath [" + config.getRascalJavaClassPathProperty() + "]", e); + } + else { + throw new JavaCompilation(e.getMessage(), e); + } + } + } + private String getClassName(FunctionDeclaration declaration) { Tags tags = declaration.getTags(); diff --git a/src/org/rascalmpl/interpreter/utils/JavaCompiler.java b/src/org/rascalmpl/interpreter/utils/JavaCompiler.java index 56dea130839..9b0090e83e0 100644 --- a/src/org/rascalmpl/interpreter/utils/JavaCompiler.java +++ b/src/org/rascalmpl/interpreter/utils/JavaCompiler.java @@ -16,6 +16,7 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -162,10 +163,13 @@ public synchronized Class compile(final String qualifiedClassName, final DiagnosticCollector diagnosticsList, final Class... types) throws JavaCompilerException, ClassCastException { - if (diagnosticsList != null) + if (diagnosticsList != null) { diagnostics = diagnosticsList; - else + } + else { diagnostics = new DiagnosticCollector(); + } + Map classes = new HashMap(1); classes.put(qualifiedClassName, javaSource); Map> compiled = compile(classes, diagnostics); @@ -245,6 +249,24 @@ public synchronized Map> compile( } } + public void compile(OutputStream classBytes, String qualifiedClassName, CharSequence classSource, final DiagnosticCollector diagnostics) throws JavaCompilerException { + Map fileMap = new HashMap<>(); + + try { + fileMap.put(qualifiedClassName, classSource); + + // ignoring return class here + compile(fileMap, diagnostics); + + // side-effect alert: + // now the local classloader contains the .class file + classLoader.outputClassBytes(qualifiedClassName, classBytes); + } + catch (IOException e) { + throw new JavaCompilerException(fileMap.keySet(), e, diagnostics); + } + } + /** * Load a class that was generated by this instance or accessible from its * parent class loader. Use this method if you need access to additional @@ -310,6 +332,8 @@ public ClassLoader getClassLoader() { public JavaFileManager getFileManager() { return javaFileManager; } + + } /** @@ -561,6 +585,16 @@ final class ClassLoaderImpl extends ClassLoader { super(parentClassLoader); } + public void outputClassBytes(String qualifiedClassName, OutputStream output) throws IOException { + JavaFileObject file = classes.get(qualifiedClassName); + if (file != null) { + output.write(((JavaFileObjectImpl) file).getByteCode()); + } + else { + throw new FileNotFoundException(qualifiedClassName); + } + } + /** * @return An collection of JavaFileObject instances for the classes in the * class loader. diff --git a/src/org/rascalmpl/library/ParseTree.rsc b/src/org/rascalmpl/library/ParseTree.rsc index 2e0ba1f7362..2a4497c778f 100644 --- a/src/org/rascalmpl/library/ParseTree.rsc +++ b/src/org/rascalmpl/library/ParseTree.rsc @@ -514,6 +514,21 @@ Tree firstAmbiguity(type[Tree] begin, str input) Tree firstAmbiguity(type[Tree] begin, loc input) = parser(begin, firstAmbiguity=true)(input, input); +@javaClass{org.rascalmpl.library.Prelude} +@synopsis{Generate a parser and store it in serialized form for later reuse.} +@description{ +The stored parsers would be able to be recovered later using ((loadParsers)). +} +java void storeParsers(type[Tree] grammar, loc saveLocation); + +@javaClass{org.rascalmpl.library.Prelude} +@synopsis{Load a previously serialized parser from disk for usage} +@description{ +The semantics of loadParsers is described by the following equation: + loadParsers o storeParsers (#Type, inputLoc, inputLoc) = +} +java &U (type[&U] nonterminal, value input, loc origin) loadParsers(loc savedParsers, bool allowAmbiguity=false, bool hasSideEffects=false, bool firstAmbiguity=false, set[Tree(Tree)] filters={}); + @doc{ #### Synopsis diff --git a/src/org/rascalmpl/library/Prelude.java b/src/org/rascalmpl/library/Prelude.java index 177ffedb1ac..91f36b9c0a7 100644 --- a/src/org/rascalmpl/library/Prelude.java +++ b/src/org/rascalmpl/library/Prelude.java @@ -2326,6 +2326,19 @@ public IFunction parser(IValue start, IBool allowAmbiguity, IBool hasSideEffect public IFunction parsers(IValue start, IBool allowAmbiguity, IBool hasSideEffects, IBool firstAmbiguity, ISet filters) { return rascalValues.parsers(start, allowAmbiguity, hasSideEffects, firstAmbiguity, filters); } + + public void storeParsers(IValue start, ISourceLocation saveLocation) { + try { + rascalValues.storeParsers(start, saveLocation); + } + catch (IOException e) { + throw RuntimeExceptionFactory.io(e.getMessage()); + } + } + + public IFunction loadParsers(ISourceLocation savedLocation, IBool allowAmbiguity, IBool hasSideEffects, IBool firstAmbiguity, ISet filters) { + return rascalValues.loadParsers(savedLocation, allowAmbiguity, hasSideEffects, firstAmbiguity, filters); + } // REFLECT -- copy in {@link PreludeCompiled} protected IConstructor makeConstructor(TypeStore store, Type returnType, String name, IValue ...args) { diff --git a/src/org/rascalmpl/library/Relation.rsc b/src/org/rascalmpl/library/Relation.rsc index d971b5f0008..12c33276286 100644 --- a/src/org/rascalmpl/library/Relation.rsc +++ b/src/org/rascalmpl/library/Relation.rsc @@ -31,9 +31,9 @@ import Relation; carrier({<1,10>, <2,20>}); carrier({<1,10,100,1000>, <2,20,200,2000>}); ```} -public set[&T] carrier (rel[&T,&T] R) +public set[&T] carrier (rel[&T from, &T to] R) { - return R<0> + R<1>; + return R.from + R.to; } public set[&T] carrier (rel[&T,&T,&T] R) diff --git a/src/org/rascalmpl/parser/ParserGenerator.java b/src/org/rascalmpl/parser/ParserGenerator.java index 237404ffd56..ff678b7f5e2 100644 --- a/src/org/rascalmpl/parser/ParserGenerator.java +++ b/src/org/rascalmpl/parser/ParserGenerator.java @@ -30,6 +30,7 @@ import org.rascalmpl.interpreter.utils.JavaBridge; import org.rascalmpl.interpreter.utils.Profiler; import org.rascalmpl.parser.gtd.IGTD; +import org.rascalmpl.uri.URIResolverRegistry; import org.rascalmpl.values.IRascalValueFactory; import org.rascalmpl.values.ValueFactoryFactory; import org.rascalmpl.values.parsetrees.ITree; @@ -251,6 +252,42 @@ public Class> getNewParser(IRascalMon } } + /** + * Generate a parser from a Rascal grammar and write it to disk + * + * @param monitor a progress monitor; this method will contribute 100 work units + * @param loc a location for error reporting + * @param name the name of the parser for use in code generation and for later reference + * @param grammar a grammar + * @return A parser class, ready for instantiation + * @throws IOException + */ + public void writeNewParser(IRascalMonitor monitor, ISourceLocation loc, String name, IMap definition, ISourceLocation target) throws IOException { + String JOB = "Generating parser:" + name; + monitor.jobStart(JOB, 100, 60); + + try { + String normName = name.replaceAll("::", "_").replaceAll("\\\\", "_"); + monitor.jobStep(JOB, "Generating java source code for parser: " + name,30); + IString classString; + IConstructor grammar = IRascalValueFactory.getInstance().grammar(definition); + + synchronized (evaluator) { + classString = (IString) evaluator.call(monitor, "newGenerate", vf.string(packageName), vf.string(normName), grammar); + } + debugOutput(classString, System.getProperty("java.io.tmpdir") + "/parser.java"); + monitor.jobStep(JOB,"Compiling generated java code: " + name, 30); + + bridge.compileJava(loc, packageName + "." + normName, classString.getValue(), URIResolverRegistry.getInstance().getOutputStream(target, false)); + } catch (ClassCastException e) { + throw new ImplementationError("parser generator:" + e.getMessage(), e); + } catch (Throw e) { + throw new ImplementationError("parser generator: " + e.getMessage() + e.getTrace()); + } finally { + monitor.jobEnd(JOB, true); + } +} + public IString createHole(IConstructor part, IInteger size) { synchronized (evaluator) { return (IString) evaluator.call("createHole", part, size); diff --git a/src/org/rascalmpl/values/IRascalValueFactory.java b/src/org/rascalmpl/values/IRascalValueFactory.java index 405924792bf..518532c16b4 100644 --- a/src/org/rascalmpl/values/IRascalValueFactory.java +++ b/src/org/rascalmpl/values/IRascalValueFactory.java @@ -22,6 +22,7 @@ import io.usethesource.vallang.IList; import io.usethesource.vallang.IMap; import io.usethesource.vallang.ISet; +import io.usethesource.vallang.ISourceLocation; import io.usethesource.vallang.IValue; import io.usethesource.vallang.IValueFactory; import io.usethesource.vallang.type.Type; @@ -105,6 +106,21 @@ default IFunction parser(IValue reifiedGrammar, IBool allowAmbiguity, IBool hasS default IFunction parsers(IValue reifiedGrammar, IBool allowAmbiguity, IBool hasSideEffects, IBool firstAmbiguity, ISet filters) { throw new UnsupportedOperationException("This Rascal value factory does not support a parser generator:" + getClass()); } - - + + /** + * Same as `parsers` but saves the result to disk rather than wrapping it as an IFunction. + * + * @param start + * @param saveLocation + */ + default void storeParsers(IValue start, ISourceLocation saveLocation) { + throw new UnsupportedOperationException("This Rascal value factory does not support a parser generator that can store parsers on disk." + getClass()); + } + + /** + * Reverse of storeParsers and with the same effect as the {@see parsers} method. + */ + default IFunction loadParsers(ISourceLocation saveLocation, IBool allowAmbiguity, IBool hasSideEffects, IBool firstAmbiguity, ISet filters) { + throw new UnsupportedOperationException("This Rascal value factory does not support a parser generator that can restore parsers from disk." + getClass()); + } } diff --git a/src/org/rascalmpl/values/RascalFunctionValueFactory.java b/src/org/rascalmpl/values/RascalFunctionValueFactory.java index de27887d9f6..1c8d1ec9db4 100644 --- a/src/org/rascalmpl/values/RascalFunctionValueFactory.java +++ b/src/org/rascalmpl/values/RascalFunctionValueFactory.java @@ -111,9 +111,28 @@ private Class> generateParser(IMap gr } } + private void writeParsers(IMap grammar, ISourceLocation target) throws IOException { + try { + getParserGenerator().writeNewParser(new NullRascalMonitor(), URIUtil.rootLocation("parser-generator"), "$GENERATED_PARSER$" + Math.abs(grammar.hashCode()), grammar, target); + } + catch (ExceptionInInitializerError e) { + throw new ImplementationError(e.getMessage(), e); + } + } + protected Class> getParserClass(IMap grammar) { return parserCache.get(grammar); } + + protected void writeParserClass(IMap grammar, ISourceLocation target) throws IOException { + getParserGenerator().writeNewParser( + new NullRascalMonitor(), + URIUtil.rootLocation("parser-generator"), + "$GENERATED_PARSER$" + Math.abs(grammar.hashCode()), + grammar, + target + ); + } @Override public IFunction function(io.usethesource.vallang.type.Type functionType, BiFunction, IValue> func) { @@ -246,6 +265,12 @@ public IFunction parsers(IValue reifiedGrammar, IBool allowAmbiguity, IBool hasS return function(functionType, new ParametrizedParseFunction(() -> getParserGenerator(), this, caller, parser, allowAmbiguity, hasSideEffects, firstAmbiguity, filters)); } + @Override + public void storeParsers(IValue reifiedGrammar, ISourceLocation saveLocation) throws IOException { + IMap grammar = (IMap) ((IConstructor) reifiedGrammar).get("definitions"); + getParserGenerator().writeNewParser(new NullRascalMonitor(), URIUtil.rootLocation("parser-generator"), "$GENERATED_PARSER$" + Math.abs(grammar.hashCode()), grammar, saveLocation); + } + /** * This function mimicks `parsers(#start[Module]) inside lang::rascal::\syntax::Rascal. * so it produces a parse function for the Rascal language, where non-terminal is the From 5759ad8b7f8ae623a2f9799f59cd7ca2b25e8c69 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Tue, 20 Jun 2023 11:34:10 +0200 Subject: [PATCH 03/16] saving now works --- src/org/rascalmpl/interpreter/utils/JavaBridge.java | 3 +-- src/org/rascalmpl/parser/ParserGenerator.java | 4 ++-- src/org/rascalmpl/values/IRascalValueFactory.java | 4 +++- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/org/rascalmpl/interpreter/utils/JavaBridge.java b/src/org/rascalmpl/interpreter/utils/JavaBridge.java index 1dbf6bc8d91..3a7b2c93e68 100644 --- a/src/org/rascalmpl/interpreter/utils/JavaBridge.java +++ b/src/org/rascalmpl/interpreter/utils/JavaBridge.java @@ -109,9 +109,8 @@ public Class compileJava(ISourceLocation loc, String className, String so } public void compileJava(ISourceLocation loc, String className, String source, OutputStream classBytes) { - compileJava(loc, className, getClass(), source); + compileJava(loc, className, getClass(), source, classBytes); } - public Class compileJava(ISourceLocation loc, String className, Class parent, String source) { try { diff --git a/src/org/rascalmpl/parser/ParserGenerator.java b/src/org/rascalmpl/parser/ParserGenerator.java index ff678b7f5e2..d174bca78f5 100644 --- a/src/org/rascalmpl/parser/ParserGenerator.java +++ b/src/org/rascalmpl/parser/ParserGenerator.java @@ -266,7 +266,7 @@ public void writeNewParser(IRascalMonitor monitor, ISourceLocation loc, String n String JOB = "Generating parser:" + name; monitor.jobStart(JOB, 100, 60); - try { + try (OutputStream out = URIResolverRegistry.getInstance().getOutputStream(target, false)) { String normName = name.replaceAll("::", "_").replaceAll("\\\\", "_"); monitor.jobStep(JOB, "Generating java source code for parser: " + name,30); IString classString; @@ -278,7 +278,7 @@ public void writeNewParser(IRascalMonitor monitor, ISourceLocation loc, String n debugOutput(classString, System.getProperty("java.io.tmpdir") + "/parser.java"); monitor.jobStep(JOB,"Compiling generated java code: " + name, 30); - bridge.compileJava(loc, packageName + "." + normName, classString.getValue(), URIResolverRegistry.getInstance().getOutputStream(target, false)); + bridge.compileJava(loc, packageName + "." + normName, classString.getValue(), out); } catch (ClassCastException e) { throw new ImplementationError("parser generator:" + e.getMessage(), e); } catch (Throw e) { diff --git a/src/org/rascalmpl/values/IRascalValueFactory.java b/src/org/rascalmpl/values/IRascalValueFactory.java index 518532c16b4..c0d4f0ea8a5 100644 --- a/src/org/rascalmpl/values/IRascalValueFactory.java +++ b/src/org/rascalmpl/values/IRascalValueFactory.java @@ -10,6 +10,7 @@ *******************************************************************************/ package org.rascalmpl.values; +import java.io.IOException; import java.util.Map; import java.util.function.BiFunction; @@ -112,8 +113,9 @@ default IFunction parsers(IValue reifiedGrammar, IBool allowAmbiguity, IBool has * * @param start * @param saveLocation + * @throws IOException */ - default void storeParsers(IValue start, ISourceLocation saveLocation) { + default void storeParsers(IValue start, ISourceLocation saveLocation) throws IOException { throw new UnsupportedOperationException("This Rascal value factory does not support a parser generator that can store parsers on disk." + getClass()); } From 7ab7be7aa892a0fee226a49f71e6721365870b1c Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Tue, 20 Jun 2023 19:46:19 +0200 Subject: [PATCH 04/16] added loading of the parser class, but it's not enough because I forgot about the nested classes --- .../interpreter/utils/JavaBridge.java | 39 ++++++++++++++++++ src/org/rascalmpl/library/Prelude.java | 17 +++++++- .../rascalmpl/values/IRascalValueFactory.java | 3 +- .../values/RascalFunctionValueFactory.java | 40 ++++++++++++++----- 4 files changed, 88 insertions(+), 11 deletions(-) diff --git a/src/org/rascalmpl/interpreter/utils/JavaBridge.java b/src/org/rascalmpl/interpreter/utils/JavaBridge.java index 3a7b2c93e68..de9242c5de5 100644 --- a/src/org/rascalmpl/interpreter/utils/JavaBridge.java +++ b/src/org/rascalmpl/interpreter/utils/JavaBridge.java @@ -15,6 +15,7 @@ *******************************************************************************/ package org.rascalmpl.interpreter.utils; +import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.PrintWriter; @@ -30,6 +31,7 @@ import javax.tools.JavaFileObject; import javax.tools.ToolProvider; +import org.objectweb.asm.ClassReader; import org.rascalmpl.ast.Expression; import org.rascalmpl.ast.FunctionDeclaration; import org.rascalmpl.ast.KeywordFormal; @@ -51,8 +53,10 @@ import org.rascalmpl.interpreter.staticErrors.MissingTag; import org.rascalmpl.interpreter.staticErrors.NonAbstractJavaFunction; import org.rascalmpl.interpreter.staticErrors.UndeclaredJavaMethod; +import org.rascalmpl.library.Prelude; import org.rascalmpl.types.DefaultRascalTypeVisitor; import org.rascalmpl.types.RascalType; +import org.rascalmpl.uri.URIResolverRegistry; import org.rascalmpl.util.ListClassLoader; import org.rascalmpl.values.IRascalValueFactory; import org.rascalmpl.values.functions.IFunction; @@ -108,6 +112,41 @@ public Class compileJava(ISourceLocation loc, String className, String so return compileJava(loc, className, getClass(), source); } + public Class loadClass(ISourceLocation loc) throws IOException { + try (InputStream forClassName = URIResolverRegistry.getInstance().getInputStream(loc)) { + ClassReader cr = new ClassReader(forClassName); + String className = cr.getClassName(); + + // ad-hoc loader to just get the bytes of _this_ specific file without + // searching in a classpath or anything. It needs access to the parser classes + // as parent. + ClassLoader loader = new ClassLoader(getClass().getClassLoader()) { + @Override + protected Class findClass(final String name) throws ClassNotFoundException { + if (!name.equals(className)) { + return super.findClass(name); + } + else { + try (InputStream forClass = URIResolverRegistry.getInstance().getInputStream(loc)) { + byte[] bytes = Prelude.consumeInputStream(URIResolverRegistry.getInstance().getInputStream(loc)); + return this.defineClass(null, bytes, 0, bytes.length); + } + catch (IOException e) { + throw new ClassNotFoundException(className); + } + } + } + }; + + try { + return (Class) loader.loadClass(className); + } + catch (ClassNotFoundException e) { + throw new IOException(e.getMessage(), e); + } + } + } + public void compileJava(ISourceLocation loc, String className, String source, OutputStream classBytes) { compileJava(loc, className, getClass(), source, classBytes); } diff --git a/src/org/rascalmpl/library/Prelude.java b/src/org/rascalmpl/library/Prelude.java index 91f36b9c0a7..5513818750b 100644 --- a/src/org/rascalmpl/library/Prelude.java +++ b/src/org/rascalmpl/library/Prelude.java @@ -1149,6 +1149,16 @@ public static String consumeInputStream(Reader in) throws IOException { } return res.toString(); } + + public static byte[] consumeInputStream(InputStream in) throws IOException { + ByteArrayOutputStream res = new ByteArrayOutputStream(); + byte[] chunk = new byte[FILE_BUFFER_SIZE]; + int read; + while ((read = in.read(chunk, 0, chunk.length)) != -1) { + res.write(chunk, 0, read); + } + return res.toByteArray(); + } public IValue md5HashFile(ISourceLocation sloc){ try { @@ -2337,7 +2347,12 @@ public void storeParsers(IValue start, ISourceLocation saveLocation) { } public IFunction loadParsers(ISourceLocation savedLocation, IBool allowAmbiguity, IBool hasSideEffects, IBool firstAmbiguity, ISet filters) { - return rascalValues.loadParsers(savedLocation, allowAmbiguity, hasSideEffects, firstAmbiguity, filters); + try { + return rascalValues.loadParsers(savedLocation, allowAmbiguity, hasSideEffects, firstAmbiguity, filters); + } + catch (IOException e) { + throw RuntimeExceptionFactory.io(e.getMessage()); + } } // REFLECT -- copy in {@link PreludeCompiled} diff --git a/src/org/rascalmpl/values/IRascalValueFactory.java b/src/org/rascalmpl/values/IRascalValueFactory.java index c0d4f0ea8a5..5c22b05eeed 100644 --- a/src/org/rascalmpl/values/IRascalValueFactory.java +++ b/src/org/rascalmpl/values/IRascalValueFactory.java @@ -121,8 +121,9 @@ default void storeParsers(IValue start, ISourceLocation saveLocation) throws IOE /** * Reverse of storeParsers and with the same effect as the {@see parsers} method. + * @throws IOException */ - default IFunction loadParsers(ISourceLocation saveLocation, IBool allowAmbiguity, IBool hasSideEffects, IBool firstAmbiguity, ISet filters) { + default IFunction loadParsers(ISourceLocation saveLocation, IBool allowAmbiguity, IBool hasSideEffects, IBool firstAmbiguity, ISet filters) throws IOException { throw new UnsupportedOperationException("This Rascal value factory does not support a parser generator that can restore parsers from disk." + getClass()); } } diff --git a/src/org/rascalmpl/values/RascalFunctionValueFactory.java b/src/org/rascalmpl/values/RascalFunctionValueFactory.java index 1c8d1ec9db4..1d726e2544b 100644 --- a/src/org/rascalmpl/values/RascalFunctionValueFactory.java +++ b/src/org/rascalmpl/values/RascalFunctionValueFactory.java @@ -111,15 +111,6 @@ private Class> generateParser(IMap gr } } - private void writeParsers(IMap grammar, ISourceLocation target) throws IOException { - try { - getParserGenerator().writeNewParser(new NullRascalMonitor(), URIUtil.rootLocation("parser-generator"), "$GENERATED_PARSER$" + Math.abs(grammar.hashCode()), grammar, target); - } - catch (ExceptionInInitializerError e) { - throw new ImplementationError(e.getMessage(), e); - } - } - protected Class> getParserClass(IMap grammar) { return parserCache.get(grammar); } @@ -271,6 +262,37 @@ public void storeParsers(IValue reifiedGrammar, ISourceLocation saveLocation) th getParserGenerator().writeNewParser(new NullRascalMonitor(), URIUtil.rootLocation("parser-generator"), "$GENERATED_PARSER$" + Math.abs(grammar.hashCode()), grammar, saveLocation); } + @Override + public IFunction loadParsers(ISourceLocation saveLocation, IBool allowAmbiguity, IBool hasSideEffects,IBool firstAmbiguity, ISet filters) throws IOException { + RascalTypeFactory rtf = RascalTypeFactory.getInstance(); + TypeFactory tf = TypeFactory.getInstance(); + + // here the return type is parametrized and instantiated when the parser function is called with the + // given start non-terminal: + Type parameterType = tf.parameterType("U", RascalValueFactory.Tree); + + Type functionType = tf.functionType(parameterType, + tf.tupleType(rtf.reifiedType(parameterType), tf.valueType(), tf.sourceLocationType()), + tf.tupleEmpty()); + + + + final Class> parser + = (Class>) ctx.getEvaluator() + .__getJavaBridge().loadClass(saveLocation); + + AbstractAST current = ctx.getCurrentAST(); + ISourceLocation caller = current != null ? current.getLocation() : URIUtil.rootLocation("unknown"); + + return function( + functionType, + new ParametrizedParseFunction(() -> getParserGenerator(), + this, + caller, + parser, + allowAmbiguity, hasSideEffects, firstAmbiguity, filters)); + } + /** * This function mimicks `parsers(#start[Module]) inside lang::rascal::\syntax::Rascal. * so it produces a parse function for the Rascal language, where non-terminal is the From 6227783e2f8e6e978f08f12de14ee7d20d62d02c Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Wed, 21 Jun 2023 08:06:39 +0200 Subject: [PATCH 05/16] now collecting all necessary sub-classes in a jar as well and storing the main class in the manifest --- .../interpreter/utils/JavaCompiler.java | 30 +++++++++++++------ 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/src/org/rascalmpl/interpreter/utils/JavaCompiler.java b/src/org/rascalmpl/interpreter/utils/JavaCompiler.java index 9b0090e83e0..bbd5e2586a8 100644 --- a/src/org/rascalmpl/interpreter/utils/JavaCompiler.java +++ b/src/org/rascalmpl/interpreter/utils/JavaCompiler.java @@ -29,6 +29,10 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.jar.Attributes; +import java.util.jar.JarEntry; +import java.util.jar.JarOutputStream; +import java.util.jar.Manifest; import java.util.Set; import javax.tools.DiagnosticCollector; @@ -260,7 +264,7 @@ public void compile(OutputStream classBytes, String qualifiedClassName, CharSequ // side-effect alert: // now the local classloader contains the .class file - classLoader.outputClassBytes(qualifiedClassName, classBytes); + classLoader.outputClassesToJar(qualifiedClassName, classBytes); } catch (IOException e) { throw new JavaCompilerException(fileMap.keySet(), e, diagnostics); @@ -585,14 +589,22 @@ final class ClassLoaderImpl extends ClassLoader { super(parentClassLoader); } - public void outputClassBytes(String qualifiedClassName, OutputStream output) throws IOException { - JavaFileObject file = classes.get(qualifiedClassName); - if (file != null) { - output.write(((JavaFileObjectImpl) file).getByteCode()); - } - else { - throw new FileNotFoundException(qualifiedClassName); - } + public void outputClassesToJar(String qualifiedClassName, OutputStream output) throws IOException { + Manifest manifest = new Manifest(); + manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0"); + manifest.getMainAttributes().put(Attributes.Name.MAIN_CLASS, qualifiedClassName); + + try (JarOutputStream target = new JarOutputStream(output, manifest)) { + for (Entry entry : classes.entrySet()) { + String className = entry.getKey(); + JavaFileObjectImpl file = (JavaFileObjectImpl) entry.getValue(); + JarEntry jarEntry = new JarEntry(className); + jarEntry.setTime(file.getLastModified()); + target.putNextEntry(jarEntry); + target.write(file.getByteCode()); + target.closeEntry(); + } + } } /** From 1011cde002ad79aebc36da75a3ac0bfc1fbc4b06 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Wed, 21 Jun 2023 19:14:41 +0200 Subject: [PATCH 06/16] now we store and retrieve a parser from a jar file because the generated parser consists of many subclasses each stored in a separate .class file --- .../interpreter/utils/JavaBridge.java | 45 +++---------------- .../interpreter/utils/JavaCompiler.java | 36 +++++++++++++++ src/org/rascalmpl/library/Prelude.java | 2 +- .../rascalmpl/values/IRascalValueFactory.java | 3 +- .../values/RascalFunctionValueFactory.java | 9 ++-- 5 files changed, 50 insertions(+), 45 deletions(-) diff --git a/src/org/rascalmpl/interpreter/utils/JavaBridge.java b/src/org/rascalmpl/interpreter/utils/JavaBridge.java index de9242c5de5..fc0c079a853 100644 --- a/src/org/rascalmpl/interpreter/utils/JavaBridge.java +++ b/src/org/rascalmpl/interpreter/utils/JavaBridge.java @@ -31,7 +31,7 @@ import javax.tools.JavaFileObject; import javax.tools.ToolProvider; -import org.objectweb.asm.ClassReader; +import org.checkerframework.checker.units.qual.t; import org.rascalmpl.ast.Expression; import org.rascalmpl.ast.FunctionDeclaration; import org.rascalmpl.ast.KeywordFormal; @@ -53,10 +53,8 @@ import org.rascalmpl.interpreter.staticErrors.MissingTag; import org.rascalmpl.interpreter.staticErrors.NonAbstractJavaFunction; import org.rascalmpl.interpreter.staticErrors.UndeclaredJavaMethod; -import org.rascalmpl.library.Prelude; import org.rascalmpl.types.DefaultRascalTypeVisitor; import org.rascalmpl.types.RascalType; -import org.rascalmpl.uri.URIResolverRegistry; import org.rascalmpl.util.ListClassLoader; import org.rascalmpl.values.IRascalValueFactory; import org.rascalmpl.values.functions.IFunction; @@ -112,41 +110,6 @@ public Class compileJava(ISourceLocation loc, String className, String so return compileJava(loc, className, getClass(), source); } - public Class loadClass(ISourceLocation loc) throws IOException { - try (InputStream forClassName = URIResolverRegistry.getInstance().getInputStream(loc)) { - ClassReader cr = new ClassReader(forClassName); - String className = cr.getClassName(); - - // ad-hoc loader to just get the bytes of _this_ specific file without - // searching in a classpath or anything. It needs access to the parser classes - // as parent. - ClassLoader loader = new ClassLoader(getClass().getClassLoader()) { - @Override - protected Class findClass(final String name) throws ClassNotFoundException { - if (!name.equals(className)) { - return super.findClass(name); - } - else { - try (InputStream forClass = URIResolverRegistry.getInstance().getInputStream(loc)) { - byte[] bytes = Prelude.consumeInputStream(URIResolverRegistry.getInstance().getInputStream(loc)); - return this.defineClass(null, bytes, 0, bytes.length); - } - catch (IOException e) { - throw new ClassNotFoundException(className); - } - } - } - }; - - try { - return (Class) loader.loadClass(className); - } - catch (ClassNotFoundException e) { - throw new IOException(e.getMessage(), e); - } - } - } - public void compileJava(ISourceLocation loc, String className, String source, OutputStream classBytes) { compileJava(loc, className, getClass(), source, classBytes); } @@ -173,6 +136,12 @@ public Class compileJava(ISourceLocation loc, String className, Class } } + public Class loadClass(InputStream in) throws IOException, ClassNotFoundException { + List commandline = Arrays.asList(new String[] {"-proc:none", "-cp", config.getRascalJavaClassPathProperty()}); + JavaCompiler javaCompiler = new JavaCompiler(getClass().getClassLoader(), null, commandline); + return javaCompiler.load(in); + } + public void compileJava(ISourceLocation loc, String className, Class parent, String source, OutputStream classBytes) { try { // watch out, if you start sharing this compiler, classes will not be able to reload diff --git a/src/org/rascalmpl/interpreter/utils/JavaCompiler.java b/src/org/rascalmpl/interpreter/utils/JavaCompiler.java index bbd5e2586a8..74123d6cf78 100644 --- a/src/org/rascalmpl/interpreter/utils/JavaCompiler.java +++ b/src/org/rascalmpl/interpreter/utils/JavaCompiler.java @@ -31,6 +31,7 @@ import java.util.Map.Entry; import java.util.jar.Attributes; import java.util.jar.JarEntry; +import java.util.jar.JarInputStream; import java.util.jar.JarOutputStream; import java.util.jar.Manifest; import java.util.Set; @@ -46,7 +47,9 @@ import javax.tools.StandardLocation; import javax.tools.ToolProvider; +import org.rascalmpl.library.Prelude; import org.rascalmpl.uri.URIUtil; +import org.rascalmpl.uri.jar.JarInputStreamFileTree; /** * Compile a String or other {@link CharSequence}, returning a Java @@ -271,6 +274,10 @@ public void compile(OutputStream classBytes, String qualifiedClassName, CharSequ } } + public Class load(InputStream file) throws IOException, ClassNotFoundException { + return classLoader.inputClassesFromJar(file); + } + /** * Load a class that was generated by this instance or accessible from its * parent class loader. Use this method if you need access to additional @@ -607,6 +614,35 @@ public void outputClassesToJar(String qualifiedClassName, OutputStream output) t } } + public Class inputClassesFromJar(InputStream in) throws IOException, ClassNotFoundException { + try (JarInputStream jarIn = new JarInputStream(in)) { + Manifest mf = jarIn.getManifest(); + String mainClass = (String) mf.getMainAttributes().get(Attributes.Name.MAIN_CLASS); + JarEntry jarEntry; + + if (mainClass == null) { + throw new IOException("missing Main-Class in jar manifest"); + } + + while ((jarEntry = jarIn.getNextJarEntry()) != null) { + String className = jarEntry.getName(); + System.err.println("loading className: " + className); + long size = (int) jarEntry.getCompressedSize(); + System.err.println("size: " + size); + if (!jarEntry.isDirectory()) { + var file = new JavaFileObjectImpl(className, JavaFileObject.Kind.CLASS); + try (var fo = file.openOutputStream()) { + fo.write(Prelude.consumeInputStream(jarIn)); + } + + add(className, file); + } + } + + return loadClass(mainClass); + } + } + /** * @return An collection of JavaFileObject instances for the classes in the * class loader. diff --git a/src/org/rascalmpl/library/Prelude.java b/src/org/rascalmpl/library/Prelude.java index 5513818750b..d9c3a787353 100644 --- a/src/org/rascalmpl/library/Prelude.java +++ b/src/org/rascalmpl/library/Prelude.java @@ -2350,7 +2350,7 @@ public IFunction loadParsers(ISourceLocation savedLocation, IBool allowAmbiguity try { return rascalValues.loadParsers(savedLocation, allowAmbiguity, hasSideEffects, firstAmbiguity, filters); } - catch (IOException e) { + catch (IOException | ClassNotFoundException e) { throw RuntimeExceptionFactory.io(e.getMessage()); } } diff --git a/src/org/rascalmpl/values/IRascalValueFactory.java b/src/org/rascalmpl/values/IRascalValueFactory.java index 5c22b05eeed..8ebd79ffc77 100644 --- a/src/org/rascalmpl/values/IRascalValueFactory.java +++ b/src/org/rascalmpl/values/IRascalValueFactory.java @@ -122,8 +122,9 @@ default void storeParsers(IValue start, ISourceLocation saveLocation) throws IOE /** * Reverse of storeParsers and with the same effect as the {@see parsers} method. * @throws IOException + * @throws ClassNotFoundException */ - default IFunction loadParsers(ISourceLocation saveLocation, IBool allowAmbiguity, IBool hasSideEffects, IBool firstAmbiguity, ISet filters) throws IOException { + default IFunction loadParsers(ISourceLocation saveLocation, IBool allowAmbiguity, IBool hasSideEffects, IBool firstAmbiguity, ISet filters) throws IOException, ClassNotFoundException { throw new UnsupportedOperationException("This Rascal value factory does not support a parser generator that can restore parsers from disk." + getClass()); } } diff --git a/src/org/rascalmpl/values/RascalFunctionValueFactory.java b/src/org/rascalmpl/values/RascalFunctionValueFactory.java index 1d726e2544b..2c3a6080998 100644 --- a/src/org/rascalmpl/values/RascalFunctionValueFactory.java +++ b/src/org/rascalmpl/values/RascalFunctionValueFactory.java @@ -263,7 +263,7 @@ public void storeParsers(IValue reifiedGrammar, ISourceLocation saveLocation) th } @Override - public IFunction loadParsers(ISourceLocation saveLocation, IBool allowAmbiguity, IBool hasSideEffects,IBool firstAmbiguity, ISet filters) throws IOException { + public IFunction loadParsers(ISourceLocation saveLocation, IBool allowAmbiguity, IBool hasSideEffects,IBool firstAmbiguity, ISet filters) throws IOException, ClassNotFoundException { RascalTypeFactory rtf = RascalTypeFactory.getInstance(); TypeFactory tf = TypeFactory.getInstance(); @@ -275,11 +275,9 @@ public IFunction loadParsers(ISourceLocation saveLocation, IBool allowAmbiguity, tf.tupleType(rtf.reifiedType(parameterType), tf.valueType(), tf.sourceLocationType()), tf.tupleEmpty()); - - final Class> parser = (Class>) ctx.getEvaluator() - .__getJavaBridge().loadClass(saveLocation); + .__getJavaBridge().loadClass(URIResolverRegistry.getInstance().getInputStream(saveLocation)); AbstractAST current = ctx.getCurrentAST(); ISourceLocation caller = current != null ? current.getLocation() : URIUtil.rootLocation("unknown"); @@ -290,7 +288,8 @@ public IFunction loadParsers(ISourceLocation saveLocation, IBool allowAmbiguity, this, caller, parser, - allowAmbiguity, hasSideEffects, firstAmbiguity, filters)); + allowAmbiguity, hasSideEffects, firstAmbiguity, filters) + ); } /** From d7abd510b9c5d596f23a5314d10b5f6799f038a9 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Wed, 21 Jun 2023 19:21:39 +0200 Subject: [PATCH 07/16] avoid to load a parser generator only for getting the parser method name --- .../values/RascalFunctionValueFactory.java | 30 +++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/src/org/rascalmpl/values/RascalFunctionValueFactory.java b/src/org/rascalmpl/values/RascalFunctionValueFactory.java index 2c3a6080998..958cd4fa918 100644 --- a/src/org/rascalmpl/values/RascalFunctionValueFactory.java +++ b/src/org/rascalmpl/values/RascalFunctionValueFactory.java @@ -227,11 +227,34 @@ public IFunction parser(IValue reifiedGrammar, IBool allowAmbiguity, IBool hasSi AbstractAST current = ctx.getCurrentAST(); ISourceLocation caller = current != null ? current.getLocation() : URIUtil.rootLocation("unknown"); - String name = getParserGenerator().getParserMethodName(startSort); + String name = getParserMethodName(startSort); + if (name == null) { + name = generator.getParserMethodName(startSort); + } return function(functionType, new ParseFunction(ctx.getValueFactory(), caller, parser, name, allowAmbiguity, hasSideEffects, firstAmbiguity, filters)); } + protected static String getParserMethodName(IConstructor symbol) { + // we use a fast non-synchronized path for simple cases; + // this is to prevent locking the evaluator in IDE contexts + // where many calls into the evaluator/parser are fired in rapid + // succession. + + switch (symbol.getName()) { + case "start": + return "start__" + getParserMethodName(SymbolAdapter.getStart(symbol)); + case "layouts": + return "layouts_" + SymbolAdapter.getName(symbol); + case "sort": + case "lex": + case "keywords": + return SymbolAdapter.getName(symbol); + } + + return null; + } + @Override public IFunction parsers(IValue reifiedGrammar, IBool allowAmbiguity, IBool hasSideEffects, IBool firstAmbiguity, ISet filters) { RascalTypeFactory rtf = RascalTypeFactory.getInstance(); @@ -533,7 +556,10 @@ public IValue apply(IValue[] parameters, Map keywordParameters) throw fail(parameters); } - String name = generator.get().getParserMethodName(startSort); + String name = getParserMethodName(startSort); + if (name == null) { + name = generator.get().getParserMethodName(startSort); + } if (firstAmbiguity) { if (parameters[1].getType().isString()) { From addd58adbb3041c7b559ab5d85fc289ddbbe39e8 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Wed, 21 Jun 2023 19:54:15 +0200 Subject: [PATCH 08/16] removed debug prints --- src/org/rascalmpl/interpreter/utils/JavaCompiler.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/org/rascalmpl/interpreter/utils/JavaCompiler.java b/src/org/rascalmpl/interpreter/utils/JavaCompiler.java index 74123d6cf78..d7dbfc1d0cf 100644 --- a/src/org/rascalmpl/interpreter/utils/JavaCompiler.java +++ b/src/org/rascalmpl/interpreter/utils/JavaCompiler.java @@ -625,12 +625,10 @@ public Class inputClassesFromJar(InputStream in) throws IOException, ClassNot } while ((jarEntry = jarIn.getNextJarEntry()) != null) { - String className = jarEntry.getName(); - System.err.println("loading className: " + className); - long size = (int) jarEntry.getCompressedSize(); - System.err.println("size: " + size); if (!jarEntry.isDirectory()) { + var className = jarEntry.getName(); var file = new JavaFileObjectImpl(className, JavaFileObject.Kind.CLASS); + try (var fo = file.openOutputStream()) { fo.write(Prelude.consumeInputStream(jarIn)); } From f64a18ed752630b8e3aee4a2ef59e81d263af854 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Wed, 21 Jun 2023 19:54:32 +0200 Subject: [PATCH 09/16] added tests and documentation for the store/load parsing features --- src/org/rascalmpl/library/ParseTree.rsc | 53 ++++++++++++++++++- .../lang/rascal/tests/concrete/Parsing.rsc | 6 +++ 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/src/org/rascalmpl/library/ParseTree.rsc b/src/org/rascalmpl/library/ParseTree.rsc index 2a4497c778f..35fd08603e2 100644 --- a/src/org/rascalmpl/library/ParseTree.rsc +++ b/src/org/rascalmpl/library/ParseTree.rsc @@ -518,14 +518,63 @@ Tree firstAmbiguity(type[Tree] begin, loc input) @synopsis{Generate a parser and store it in serialized form for later reuse.} @description{ The stored parsers would be able to be recovered later using ((loadParsers)). + +For any concrete grammar, a.k.a. reified syntax type, `g` it holds that: +* after `storeParsers(g, file);` +* then `g = loadParsers(file);` +* and given `h = parsers(g);` +* then for all valid `nt`, `input` and `origin`: `g(nt, input, origin) == g(nt, input, origin)` + +In other words, a loaded parser function behaves exactly as a freshly generated parser +for the same grammar, if (and only if) it was stored for the same grammar value. +} +@benefits{ +* storing parsers is to cache the work of reifing a grammar, and generating a parser from that grammar. +* stored parsers are nice for deployment scenerios where the language is fixed and efficiency is appreciated. +} +@pitfalls{ +* caching parsers with `storeParsers` is your responsibility; the cache is not cleaned automatically when the grammar changes. } java void storeParsers(type[Tree] grammar, loc saveLocation); @javaClass{org.rascalmpl.library.Prelude} @synopsis{Load a previously serialized parser from disk for usage} @description{ -The semantics of loadParsers is described by the following equation: - loadParsers o storeParsers (#Type, inputLoc, inputLoc) = +For any concrete grammar, a.k.a. reified syntax type, `g` it holds that: +* after `storeParsers(g, file);` +* then `g = loadParsers(file);` +* and given `h = parsers(g);` +* then for all valid `nt`, `input` and `origin`: `g(nt, input, origin) == g(nt, input, origin)` + +In other words, a loaded parser function behaves exactly as a freshly generated parser +for the same grammar, if (and only if) it was stored for the same grammar value. +} +@examples{ + +First we store a parser: +```rascal-shell +import ParseTree; +syntax E = E "+" E | "e"; +storeParsers(#E, |test-temp:///E.parsers|) +``` + +Here we show a new shell does not even know about the grammar: +```rascal-shell,errors +#E +``` + +Then in a next run, we load the parser and use it: +``` +import ParseTree; +p = loadParsers(|test-temp:///E.parsers|); +p(type(sort("E"), ()), "e+e", |src:///|); +} +@benefits{ +* loaded parsers can be used immediately without the need of loadig and executing a parser generator. +} +@pitfalls{ +* reifiying types (use of `#`) will trigger the loading of a parser generator anyway. You have to use +this notation for types to avoid that: `type(\start(sort("MySort")), ())` to avoid the computation for `#start[A]` } java &U (type[&U] nonterminal, value input, loc origin) loadParsers(loc savedParsers, bool allowAmbiguity=false, bool hasSideEffects=false, bool firstAmbiguity=false, set[Tree(Tree)] filters={}); diff --git a/src/org/rascalmpl/library/lang/rascal/tests/concrete/Parsing.rsc b/src/org/rascalmpl/library/lang/rascal/tests/concrete/Parsing.rsc index 26b8db77af1..24fec10936d 100644 --- a/src/org/rascalmpl/library/lang/rascal/tests/concrete/Parsing.rsc +++ b/src/org/rascalmpl/library/lang/rascal/tests/concrete/Parsing.rsc @@ -62,3 +62,9 @@ test bool parsingWithAManualGrammar() && Tree t := parse(gr, "hello") && "" == "hello"; +test bool saveAndRestoreParser() { + storeParsers(#start[A], |test-temp:///parsers.jar|); + p = loadParsers(|test-temp:///parsers.jar|); + + return p(type(\start(sort("A")), ()), "a", |origin:///|) == parse(#start[A], "a", |origin:///|); +} \ No newline at end of file From 6dadd63bc41e991d41b44f82f26f8a23f194e9bb Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Wed, 21 Jun 2023 19:58:30 +0200 Subject: [PATCH 10/16] removed examples because they crash the tutor due to bootstrapping reasons --- src/org/rascalmpl/library/ParseTree.rsc | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/src/org/rascalmpl/library/ParseTree.rsc b/src/org/rascalmpl/library/ParseTree.rsc index 35fd08603e2..2ee397fdb7b 100644 --- a/src/org/rascalmpl/library/ParseTree.rsc +++ b/src/org/rascalmpl/library/ParseTree.rsc @@ -549,26 +549,6 @@ For any concrete grammar, a.k.a. reified syntax type, `g` it holds that: In other words, a loaded parser function behaves exactly as a freshly generated parser for the same grammar, if (and only if) it was stored for the same grammar value. } -@examples{ - -First we store a parser: -```rascal-shell -import ParseTree; -syntax E = E "+" E | "e"; -storeParsers(#E, |test-temp:///E.parsers|) -``` - -Here we show a new shell does not even know about the grammar: -```rascal-shell,errors -#E -``` - -Then in a next run, we load the parser and use it: -``` -import ParseTree; -p = loadParsers(|test-temp:///E.parsers|); -p(type(sort("E"), ()), "e+e", |src:///|); -} @benefits{ * loaded parsers can be used immediately without the need of loadig and executing a parser generator. } From 6024915827cdf7a3f92d0eeaf53c055dea8ebf22 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Wed, 21 Jun 2023 19:59:55 +0200 Subject: [PATCH 11/16] set errorsAsWarnings to true otherwise the tutor chokes on uses of new java methods --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index f3b9ce930b5..eab0ab9e17f 100644 --- a/pom.xml +++ b/pom.xml @@ -125,7 +125,7 @@ rascal-maven-plugin ${rascal-maven.version} - false + true ${project.build.outputDirectory} ${project.basedir}/src/org/rascalmpl/library From 800c6c340d159f4f73dc8bf5d1a487fd0c91c4d5 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Wed, 21 Jun 2023 20:00:09 +0200 Subject: [PATCH 12/16] Revert "removed examples because they crash the tutor due to bootstrapping reasons" This reverts commit 6dadd63bc41e991d41b44f82f26f8a23f194e9bb. --- src/org/rascalmpl/library/ParseTree.rsc | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/org/rascalmpl/library/ParseTree.rsc b/src/org/rascalmpl/library/ParseTree.rsc index 2ee397fdb7b..35fd08603e2 100644 --- a/src/org/rascalmpl/library/ParseTree.rsc +++ b/src/org/rascalmpl/library/ParseTree.rsc @@ -549,6 +549,26 @@ For any concrete grammar, a.k.a. reified syntax type, `g` it holds that: In other words, a loaded parser function behaves exactly as a freshly generated parser for the same grammar, if (and only if) it was stored for the same grammar value. } +@examples{ + +First we store a parser: +```rascal-shell +import ParseTree; +syntax E = E "+" E | "e"; +storeParsers(#E, |test-temp:///E.parsers|) +``` + +Here we show a new shell does not even know about the grammar: +```rascal-shell,errors +#E +``` + +Then in a next run, we load the parser and use it: +``` +import ParseTree; +p = loadParsers(|test-temp:///E.parsers|); +p(type(sort("E"), ()), "e+e", |src:///|); +} @benefits{ * loaded parsers can be used immediately without the need of loadig and executing a parser generator. } From b643d7a02f5c91e30a94957b552a86e06160581d Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 22 Jun 2023 09:52:06 +0200 Subject: [PATCH 13/16] Update Relation.rsc undo debug fix --- src/org/rascalmpl/library/Relation.rsc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/org/rascalmpl/library/Relation.rsc b/src/org/rascalmpl/library/Relation.rsc index 12c33276286..6a2ab14917a 100644 --- a/src/org/rascalmpl/library/Relation.rsc +++ b/src/org/rascalmpl/library/Relation.rsc @@ -33,7 +33,7 @@ carrier({<1,10,100,1000>, <2,20,200,2000>}); ```} public set[&T] carrier (rel[&T from, &T to] R) { - return R.from + R.to; + return R<0> + R<1>; } public set[&T] carrier (rel[&T,&T,&T] R) From 6ba6a97c42fa799b13dc493cd87e1cd1cc3375d2 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 22 Jun 2023 09:52:49 +0200 Subject: [PATCH 14/16] Update Relation.rsc more undo --- src/org/rascalmpl/library/Relation.rsc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/org/rascalmpl/library/Relation.rsc b/src/org/rascalmpl/library/Relation.rsc index 6a2ab14917a..4b513f10650 100644 --- a/src/org/rascalmpl/library/Relation.rsc +++ b/src/org/rascalmpl/library/Relation.rsc @@ -31,12 +31,12 @@ import Relation; carrier({<1,10>, <2,20>}); carrier({<1,10,100,1000>, <2,20,200,2000>}); ```} -public set[&T] carrier (rel[&T from, &T to] R) +public set[&T] carrier (rel[&T,&T] R) { return R<0> + R<1>; } -public set[&T] carrier (rel[&T,&T,&T] R) +public set[&T] carrier (rel[&T, &T, &T] R) { return (R<0> + R<1>) + R<2>; } From db7f82f4341bda5a566c23ec04899264fdc1586b Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 22 Jun 2023 11:34:31 +0200 Subject: [PATCH 15/16] Update ParseTree.rsc fixed typo thanks to @DavyLandman --- src/org/rascalmpl/library/ParseTree.rsc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/org/rascalmpl/library/ParseTree.rsc b/src/org/rascalmpl/library/ParseTree.rsc index 35fd08603e2..8485820437b 100644 --- a/src/org/rascalmpl/library/ParseTree.rsc +++ b/src/org/rascalmpl/library/ParseTree.rsc @@ -523,7 +523,7 @@ For any concrete grammar, a.k.a. reified syntax type, `g` it holds that: * after `storeParsers(g, file);` * then `g = loadParsers(file);` * and given `h = parsers(g);` -* then for all valid `nt`, `input` and `origin`: `g(nt, input, origin) == g(nt, input, origin)` +* then for all valid `nt`, `input` and `origin`: `g(nt, input, origin) == h(nt, input, origin)` In other words, a loaded parser function behaves exactly as a freshly generated parser for the same grammar, if (and only if) it was stored for the same grammar value. From 01cbacee9fffe2407fd5d8a06572bc59e37e5227 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Thu, 22 Jun 2023 11:36:40 +0200 Subject: [PATCH 16/16] Update ParseTree.rsc same typo in documentation clone. --- src/org/rascalmpl/library/ParseTree.rsc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/org/rascalmpl/library/ParseTree.rsc b/src/org/rascalmpl/library/ParseTree.rsc index 8485820437b..a517e1210a2 100644 --- a/src/org/rascalmpl/library/ParseTree.rsc +++ b/src/org/rascalmpl/library/ParseTree.rsc @@ -544,7 +544,7 @@ For any concrete grammar, a.k.a. reified syntax type, `g` it holds that: * after `storeParsers(g, file);` * then `g = loadParsers(file);` * and given `h = parsers(g);` -* then for all valid `nt`, `input` and `origin`: `g(nt, input, origin) == g(nt, input, origin)` +* then for all valid `nt`, `input` and `origin`: `g(nt, input, origin) == h(nt, input, origin)` In other words, a loaded parser function behaves exactly as a freshly generated parser for the same grammar, if (and only if) it was stored for the same grammar value.