This is an experiment, not for merging. Using generated Java code instead of old interpreter code to run the parser generator"

.settings/org.eclipse.core.resources.prefs

@@ -4,7 +4,7 @@ encoding//src/org/rascalmpl/library/experiments/vis2/lib/MarkdownConverter.js=UT
 encoding//src/org/rascalmpl/library/experiments/vis2/lib/nv.d3.css=UTF-8
 encoding//src/org/rascalmpl/library/experiments/vis2/lib/nv.d3.js=UTF-8
 encoding//src/org/rascalmpl/library/experiments/vis2/lib/reset.css=UTF-8
-encoding//src/org/rascalmpl/library/lang/rascal/tests/functionality/DataType.rsc=UTF-8
+encoding//src/rascal=UTF-8
 encoding//test/org/rascalmpl/test/data=UTF-8
 encoding/<project>=UTF-8
 encoding/src=UTF-8

pom.xml

@@ -66,7 +66,7 @@
                 </includes>
             </resource>
             <resource>
-                <directory>${project.build.directory}/generated-resources</directory>
+                <directory>${project.build.directory}/src/rascal</directory>
             </resource>
         </resources>
         <testSourceDirectory>test</testSourceDirectory>
@@ -178,7 +178,7 @@
                             <funding>${project.basedir}/FUNDING</funding>
                             <citation>${project.basedir}/CITATION.md</citation>
                             <license>${project.basedir}/LICENSE</license>
-                            <authors>${project.basedir}/AUTHORS.md</authors>
+                            <!--<authors>${project.basedir}/AUTHORS.md</authors> -->
                             <sources>|http://github.com/usethesource/rascal/blob/main|</sources>
                             <issues>|http://github.com/usethesource/rascal/issues|</issues>
                             <version>${project.version}</version>

src/org/rascalmpl/interpreter/Evaluator.java

@@ -719,17 +719,6 @@ public IConstructor getGrammar(Environment env) {
         return getParserGenerator().getGrammarFromModules(monitor, root.getName(), root.getSyntaxDefinition());
     }
 
-    public IValue diagnoseAmbiguity(IRascalMonitor monitor, IConstructor parseTree) {
-        IRascalMonitor old = setMonitor(monitor);
-        try {
-            ParserGenerator pgen = getParserGenerator();
-            return pgen.diagnoseAmbiguity(parseTree);
-        }
-        finally {
-            setMonitor(old);
-        }
-    }
-
     public IConstructor getExpandedGrammar(IRascalMonitor monitor, ISourceLocation uri) {
         IRascalMonitor old = setMonitor(monitor);
         try {

src/org/rascalmpl/parser/ParserGenerator.java

@@ -18,7 +18,6 @@
 import java.io.IOException;
 import java.io.OutputStream;
 import java.io.PrintWriter;
-import java.io.Reader;
 import java.util.Collections;
 
 import org.rascalmpl.debug.IRascalMonitor;
@@ -27,59 +26,60 @@
 import org.rascalmpl.interpreter.Configuration;
 import org.rascalmpl.interpreter.Evaluator;
 import org.rascalmpl.interpreter.utils.JavaBridge;
-import org.rascalmpl.interpreter.utils.Profiler;
 import org.rascalmpl.parser.gtd.IGTD;
-import org.rascalmpl.shell.ShellEvaluatorFactory;
+import org.rascalmpl.runtime.ModuleStore;
+import org.rascalmpl.runtime.RascalExecutionContext;
 import org.rascalmpl.uri.URIResolverRegistry;
 import org.rascalmpl.values.IRascalValueFactory;
 import org.rascalmpl.values.parsetrees.ITree;
-import org.rascalmpl.values.parsetrees.SymbolAdapter;
+import rascal.lang.rascal.grammar.$ParserGenerator;
+import rascal.lang.rascal.grammar.$ConcreteSyntax;
+import rascal.lang.rascal.grammar.definition.$Modules;
+import rascal.lang.rascal.grammar.definition.$Priorities;
+import rascal.lang.rascal.grammar.definition.$Regular;
+import rascal.lang.rascal.grammar.definition.$Keywords;
+import rascal.lang.rascal.grammar.definition.$Literals;
+import rascal.lang.rascal.grammar.definition.$Symbols;
+import rascal.lang.rascal.grammar.definition.$Parameters;
 
 import io.usethesource.vallang.IConstructor;
 import io.usethesource.vallang.IInteger;
 import io.usethesource.vallang.IMap;
 import io.usethesource.vallang.ISet;
 import io.usethesource.vallang.ISourceLocation;
 import io.usethesource.vallang.IString;
-import io.usethesource.vallang.IValue;
 import io.usethesource.vallang.IValueFactory;
 
 public class ParserGenerator {
-	private final Evaluator evaluator;
+	private final $ParserGenerator pgen;
+	private final $ConcreteSyntax concreteSyntax;
+	private final $Modules modules;
+	private final $Priorities priorities;
+	private final $Regular regular;
+	private final $Keywords keywords;
+	private final $Literals literals;
+	private final $Parameters parameters;
+	private final $Symbols symbols;
+
 	private final JavaBridge bridge;
 	private final IValueFactory vf;
 	private static final String packageName = "org.rascalmpl.java.parser.object";
 	private static final boolean debug = false;
 
 	public ParserGenerator(IRascalMonitor monitor, PrintWriter out, IValueFactory factory, Configuration config) {
-		this.evaluator = ShellEvaluatorFactory.getBasicEvaluator(Reader.nullReader(), out, out, monitor, "$parsergenerator$");
-		this.evaluator.getConfiguration().setGeneratorProfiling(config.getGeneratorProfilingProperty());
-		this.evaluator.setBootstrapperProperty(true);
+		var rex = new RascalExecutionContext(null, out, out, null, null, $ParserGenerator.class);
+		ModuleStore ms = rex.getModuleStore();
+		pgen = new $ParserGenerator(rex);
+		concreteSyntax = ms.getModule($ConcreteSyntax.class);
+		modules =  ms.getModule($Modules.class);
+		priorities = ms.getModule($Priorities.class);
+		symbols =  ms.getModule($Symbols.class);
+		regular =  ms.getModule($Regular.class);
+		literals =  ms.getModule($Literals.class);
+		parameters = ms.getModule($Parameters.class);
+		keywords =  ms.getModule($Keywords.class);
 		this.bridge = new JavaBridge(Collections.singletonList(Evaluator.class.getClassLoader()), factory, config);
 		this.vf = factory;
-
-		evaluator.doImport(monitor, 
-	"lang::rascal::grammar::ParserGenerator",
-			"lang::rascal::grammar::ConcreteSyntax",
-			"lang::rascal::grammar::definition::Modules",
-			"lang::rascal::grammar::definition::Priorities", 
-			"lang::rascal::grammar::definition::Regular", 
-			"lang::rascal::grammar::definition::Keywords",
-			"lang::rascal::grammar::definition::Literals",
-			"lang::rascal::grammar::definition::Parameters",
-			"lang::rascal::grammar::definition::Symbols",
-			"analysis::grammars::Ambiguity"
-		);
-	}
-
-	public void setGeneratorProfiling(boolean f) {
-		evaluator.getConfiguration().setGeneratorProfiling(f);
-	}
-
-	public IValue diagnoseAmbiguity(IConstructor parseForest) {
-		synchronized(evaluator) {
-			return evaluator.call("diagnose", parseForest);
-		}
 	}
 
 	private void debugOutput(Object thing, String file) {
@@ -107,63 +107,37 @@ private void debugOutput(Object thing, String file) {
 		}
 	}
 
-	public IConstructor getGrammarFromModules(IRascalMonitor monitor, String main, IMap modules) {
-		synchronized(evaluator) {
-			return (IConstructor) evaluator.call(monitor, "modules2grammar", vf.string(main), modules);
-		}
+	public IConstructor getGrammarFromModules(IRascalMonitor monitor, String main, IMap mods) {
+		return modules.modules2grammar(vf.string(main), mods);
 	}
 
 	public IConstructor getExpandedGrammar(IRascalMonitor monitor, String main, IMap definition) {
-		synchronized(evaluator) {
-			IConstructor g = getGrammarFromModules(monitor, main, definition);
-			g = (IConstructor) evaluator.call(monitor, "expandKeywords", g);
-			g = (IConstructor) evaluator.call(monitor, "makeRegularStubs", g);
-			g = (IConstructor) evaluator.call(monitor, "expandRegularSymbols", g);
-			g = (IConstructor) evaluator.call(monitor, "expandParameterizedSymbols", g);
-			g = (IConstructor) evaluator.call(monitor, "literals", g);
-			return g;
-		}
+		IConstructor g = getGrammarFromModules(monitor, main, definition);
+		g = keywords.expandKeywords(g);
+		g = (IConstructor) regular.makeRegularStubs(g); // why is the return type IValue here?
+		g = regular.expandRegularSymbols(g);
+		g = parameters.expandParameterizedSymbols(g);
+		g = literals.literals(g);
+
+		return g;
 	}
 
-	public ISet getNestingRestrictions(IRascalMonitor monitor,
-			IConstructor g) {
-		synchronized (evaluator) {
-			return (ISet) evaluator.call(monitor, "doNotNest", g);
-		}
+	public ISet getNestingRestrictions(IRascalMonitor monitor, IConstructor g) {
+		return priorities.doNotNest(g);
 	}
 
 	/** 
 	 * Produces the name generated by the parser generator for a parse method for the given symbol
 	 */
 	public String getParserMethodName(IConstructor symbol) {
-		// we use a fast non-synchronized path for simple cases; 
-		// this is to prevent locking the evaluator in IDE contexts
-		// where many calls into the evaluator/parser are fired in rapid
-		// succession.
-
-		switch (symbol.getName()) {
-			case "start":
-				return "start__" + getParserMethodName(SymbolAdapter.getStart(symbol));
-			case "layouts":
-				return "layouts_" + SymbolAdapter.getName(symbol);
-			case "sort":
-			case "lex":
-			case "keywords":
-				return SymbolAdapter.getName(symbol);
-		}
-
-		synchronized (evaluator) {
-			return ((IString) evaluator.call((IRascalMonitor) null, "getParserMethodName", symbol)).getValue();
-		}
+		return pgen.getParserMethodName(symbol).getValue();
 	}
 
 	/**
 	 * Converts the parse tree of a symbol to a UPTR symbol
 	 */
 	public IConstructor symbolTreeToSymbol(IConstructor symbol) {
-		synchronized (evaluator) {
-	  		return (IConstructor) evaluator.call((IRascalMonitor) null,"sym2symbol", symbol);
-		}
+		return symbols.sym2symbol(symbol);
 	}
 
   /**
@@ -176,12 +150,7 @@ public IConstructor symbolTreeToSymbol(IConstructor symbol) {
    * @return A parser class, ready for instantiation
    */
 	public Class<IGTD<IConstructor, ITree, ISourceLocation>> getNewParser(IRascalMonitor monitor, ISourceLocation loc, String name, IMap definition) {
-		Profiler profiler = evaluator.getConfiguration().getGeneratorProfilingProperty() ? new Profiler(evaluator) : null;
-
 		try {
-			if (profiler != null) {
-				profiler.start();
-			}
 			IConstructor grammar = IRascalValueFactory.getInstance().grammar(definition);
 			debugOutput(grammar, System.getProperty("java.io.tmpdir") + "/grammar.trm");
 			return getNewParser(monitor, loc, name, grammar);
@@ -192,14 +161,6 @@ public Class<IGTD<IConstructor, ITree, ISourceLocation>> getNewParser(IRascalMon
 		catch (Throw e) {
 			throw new ImplementationError("parser generator: " + e.getMessage() + e.getTrace());
 		} 
-		finally {
-			if (profiler != null) {
-				profiler.pleaseStop();
-				evaluator.getOutPrinter().println("PROFILE:");
-				profiler.report();
-				profiler = null;
-			}
-		}
 	}
 
   /**
@@ -215,11 +176,8 @@ public Class<IGTD<IConstructor, ITree, ISourceLocation>> getNewParser(IRascalMon
 		try {
 			String normName = name.replaceAll("::", "_").replaceAll("\\\\", "_");
 
-			IString classString;
-			synchronized (evaluator) {
-				classString = (IString) evaluator.call(monitor, "newGenerate", vf.string(packageName), vf.string(normName), grammar);
-			}
-			debugOutput(classString.getValue(), System.getProperty("java.io.tmpdir") + "/parser.java");
+			IString classString = pgen.newGenerate(vf.string(packageName), vf.string(normName), grammar);
+			debugOutput(classString, System.getProperty("java.io.tmpdir") + "/parser.java");
 
 			return bridge.compileJava(loc, packageName + "." + normName, classString.getValue());
 		} catch (ClassCastException e) {
@@ -242,13 +200,11 @@ public Class<IGTD<IConstructor, ITree, ISourceLocation>> getNewParser(IRascalMon
   public void writeNewParser(IRascalMonitor monitor, ISourceLocation loc, String name, IMap definition, ISourceLocation target) throws IOException {
 	try (OutputStream out = URIResolverRegistry.getInstance().getOutputStream(target, false)) {
 		String normName = name.replaceAll("::", "_").replaceAll("\\\\", "_");
-		IString classString;
 		IConstructor grammar = IRascalValueFactory.getInstance().grammar(definition);
 
-		synchronized (evaluator) {
-			classString = (IString) evaluator.call(monitor, "newGenerate", vf.string(packageName), vf.string(normName), grammar);
-		}
-		debugOutput(classString.getValue(), System.getProperty("java.io.tmpdir") + "/parser.java");
+		IString classString = pgen.newGenerate(vf.string(packageName), vf.string(normName), grammar);
+
+		debugOutput(classString, System.getProperty("java.io.tmpdir") + "/parser.java");
 
 		bridge.compileJava(loc, packageName + "." + normName, classString.getValue(), out);
 	} catch (ClassCastException e) {
@@ -259,8 +215,6 @@ public void writeNewParser(IRascalMonitor monitor, ISourceLocation loc, String n
 }
 
 	public IString createHole(IConstructor part, IInteger size) {
-		synchronized (evaluator) {
-			return (IString) evaluator.call("createHole", part, size);
-		}
+		return concreteSyntax.createHole(part, size);
 	}
 }

src/org/rascalmpl/runtime/$RascalModule.java

@@ -37,6 +37,8 @@
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.nio.channels.FileChannel;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -429,10 +431,23 @@ public Set<ISourceLocation> findResources(String fileName) {
 		Type adtType = $TF.abstractDataType($TS, adtName);
 		return adtType;
 	}
+
+	public io.usethesource.vallang.type.Type $parameterizedAdt(String adtName, Type[] params){
+		return $TF.abstractDataType($TS, adtName, params);
+	}
 
 	public io.usethesource.vallang.type.Type $sort(String adtName){
 		return $RTF.nonTerminalType($RVF.constructor(RascalValueFactory.Symbol_Sort, $RVF.string(adtName)));
 	}
+
+	public io.usethesource.vallang.type.Type $parameterizedSort(String adtName, Type[] parameters, IList bindings) {
+		return $RTF.nonTerminalType($RVF.constructor(RascalValueFactory.Symbol_ParameterizedSort, $VF.string(adtName), bindings));
+	}
+
+	public io.usethesource.vallang.type.Type $parameterizedLex(String adtName, Type[] parameters, IList bindings) {
+		return $RTF.nonTerminalType($RVF.constructor(RascalValueFactory.Symbol_ParameterizedLex, $VF.string(adtName), bindings));
+	}
+
 
 	public io.usethesource.vallang.type.Type $lex(String adtName){
 		return $RTF.nonTerminalType($RVF.constructor(RascalValueFactory.Symbol_Lex, $RVF.string(adtName)));

src/org/rascalmpl/runtime/RascalExecutionContext.java

@@ -90,7 +90,7 @@ public RascalExecutionContext(
 		$TRAVERSE = new Traverse($RVF);
 		mstore = new ModuleStore();
 		$TS = new TypeStore();
-		rascalSearchPath = new RascalSearchPath();
+		rascalSearchPath = null ; // JV: unused new RascalSearchPath();
 
 	    URIResolverRegistry reg = URIResolverRegistry.getInstance();
 	    String projectName = new RascalManifest().getProjectName(projectRoot);
@@ -120,10 +120,6 @@ public RascalExecutionContext(
 				}
 				//System.err.println("");
 			}
-		} catch (IOException e) {
-			return;
-		} catch (URISyntaxException e) {
-			return;
 		}
 	}
 
@@ -204,9 +200,11 @@ public void warning(String message, ISourceLocation src) {
 	 public static ISourceLocation inferProjectRoot(Class<?> clazz) {
 	        try {
 	            String file = clazz.getProtectionDomain().getCodeSource().getLocation().getPath();
-	            if (file.endsWith(".jar")) {
-	                throw new IllegalArgumentException("can not run Rascal JUnit tests from within a jar file");
-	            }
+
+				// This has to run from a jar file too!
+	            // if (file.endsWith(".jar")) {
+	            //     throw new IllegalArgumentException("can not run Rascal JUnit tests from within a jar file");
+	            // }
 
 	            File current = new File(file);
 

src/org/rascalmpl/semantics/dynamic/ShellCommand.java

@@ -227,7 +227,6 @@ private static void setOption(IEvaluator<Result<IValue>> __eval, String name, St
 		switch (name) {
 			case Configuration.GENERATOR_PROFILING_PROPERTY:
 				  __eval.getConfiguration().setGeneratorProfiling(Boolean.parseBoolean(value));
-				  __eval.getParserGenerator().setGeneratorProfiling(Boolean.parseBoolean(value));
 				  break;
 			case Configuration.PROFILING_PROPERTY: 
 			  __eval.getConfiguration().setProfiling(Boolean.parseBoolean(value));