Represent scanner tokens as enums internally (eclipse-jdt#3303) (eclipse-jdt#3772)

* Represent scanner tokens as enums internally (eclipse-jdt#3303)

 - Token production and processing is not type checkable from tokenization to parser tables
 - Renames `TerminalTokens` to `TerminalToken`
 - LexStream uses enums, but `DiagnoseParser`'s mix of terminals and non-terminal int id is retained.

Author: jespersm

org.eclipse.jdt.core.compiler.batch/scripts/GenerateParserScript.java

@@ -59,17 +59,21 @@ public static void main(String[] args) throws IOException, InterruptedException
 			Files.move(new File(filename).toPath(), new File(parserDir, filename).toPath(), StandardCopyOption.REPLACE_EXISTING);
 		}
 
-		// Update TerminalTokens.java
+		// Update terminalToken.java
 		File javasymFile = new File(grammarDir, "javasym.java");
-		File terminalTokensFile = new File(parserDir, "TerminalTokens.java");
+		File terminalTokenFile = new File(parserDir, "TerminalToken.java");
 		String javasymText = new String(Files.readAllBytes(javasymFile.toPath())).replace("\r\n",  "\n");
-		String terminalTokensText = new String(Files.readAllBytes(terminalTokensFile.toPath())).replace("\r\n",  "\n");
+		String terminalTokenText = new String(Files.readAllBytes(terminalTokenFile.toPath())).replace("\r\n",  "\n");
 		{
 			String startTag = "// BEGIN_AUTOGENERATED_REGION\n";
-			int start = terminalTokensText.indexOf(startTag);
+			String endTag = "// END_AUTOGENERATED_REGION\n";
+			int start = terminalTokenText.indexOf(startTag);
 			assertTrue(start >= 0);
 			start += startTag.length();
-			String terminalTokensProlog = terminalTokensText.substring(0, start);
+			int end = terminalTokenText.indexOf(endTag);
+			assertTrue(end >= 0);
+			String terminalTokenProlog = terminalTokenText.substring(0, start);
+			String terminalTokenEpilog = terminalTokenText.substring(end);
 
 			String javasymProlog =
 					"interface javasym\n" + 
@@ -82,7 +86,10 @@ public static void main(String[] args) throws IOException, InterruptedException
 			javasymText = javasymText.replace("TokenName$eof", "TokenNameEOF");
 			javasymText = javasymText.replace("TokenName$error", "TokenNameERROR");
 			javasymText = javasymText.replace("TokenNamenon-sealed", "TokenNamenon_sealed");
-			Files.write(terminalTokensFile.toPath(), (terminalTokensProlog + "\tint " + javasymText).getBytes());
+			javasymText = javasymText.replace("}", "");
+			javasymText = javasymText.replaceAll("\\s*=\\s*(\\d+)", "($1)");
+			
+			Files.write(terminalTokenFile.toPath(), (terminalTokenProlog + "\t\t\t\t\t\t\t" + javasymText + terminalTokenEpilog).getBytes());
 		}
 
 		// Update ParserBasicInformation.java

org.eclipse.jdt.core.compiler.batch/src/org/eclipse/jdt/internal/compiler/parser/AbstractCommentParser.java

@@ -20,13 +20,13 @@ public interface ConflictedParser {
 	   whether an @ begins a SE8 style type annotation or a SE5 declaration annotation. Where they can co-exist,
 	   we treat the type annotation as a declarative annotation.
 	*/
-	boolean atConflictScenario(int token);
+	boolean atConflictScenario(TerminalToken token);
 
 	/* Return true if at the configuration the parser finds itself in, it would shift the token.
 	   It is axiomatic of the push down automaton that corresponds to the LALR grammar that it
 	   will never shift on invalid input.
 	*/
-	boolean automatonWillShift(int token);
+	boolean automatonWillShift(TerminalToken token);
 
 	/*
 	 * Return true if the parser is parsing a module declaration. In Java 9, module, requires, exports,

org.eclipse.jdt.core.compiler.batch/src/org/eclipse/jdt/internal/compiler/parser/ConflictedParser.java

@@ -404,12 +404,12 @@ protected void addSnippetInnerTag(Object tag, Object snippetTag) {
 	}
 
 	@Override
-	protected Object createTypeReference(int primitiveToken) {
+	protected Object createTypeReference(TerminalToken primitiveToken) {
 		return createTypeReference(primitiveToken, false);
 	}
 
 	@Override
-	protected Object createTypeReference(int primitiveToken, boolean canBeModule) {
+	protected Object createTypeReference(TerminalToken primitiveToken, boolean canBeModule) {
 		TypeReference typeRef = null;
 		int size = this.identifierLengthStack[this.identifierLengthPtr];
 		if (size == 1) { // Single Type ref
@@ -440,7 +440,7 @@ protected JavadocModuleReference createModuleReference(int moduleRefTokenCount)
 	}
 
 	@Override
-	protected Object createModuleTypeReference(int primitiveToken, int moduleRefTokenCount) {
+	protected Object createModuleTypeReference(TerminalToken primitiveToken, int moduleRefTokenCount) {
 		JavadocModuleReference moduleRef= createModuleReference(moduleRefTokenCount);
 
 		TypeReference typeRef = null;

org.eclipse.jdt.core.compiler.batch/src/org/eclipse/jdt/internal/compiler/parser/JavadocParser.java

@@ -13,6 +13,9 @@
  *******************************************************************************/
 package org.eclipse.jdt.internal.compiler.parser;
 
+import static org.eclipse.jdt.internal.compiler.parser.TerminalToken.TokenNameSingleQuoteStringLiteral;
+import static org.eclipse.jdt.internal.compiler.parser.TerminalToken.TokenNameStringLiteral;
+
 import org.eclipse.jdt.core.compiler.InvalidInputException;
 import org.eclipse.jdt.internal.compiler.classfmt.ClassFileConstants;
 
@@ -123,7 +126,7 @@ public JavadocScanner(
 	}
 
 	@Override
-	protected int scanForStringLiteral() throws InvalidInputException {
+	protected TerminalToken scanForStringLiteral() throws InvalidInputException {
 		if (this.considerRegexInStringLiteral) {
 			this.unicodeAsBackSlash = false;
 			boolean isUnicode = false;
@@ -250,15 +253,15 @@ protected int scanForStringLiteral() throws InvalidInputException {
 	}
 
 	@Override
-	protected int processSingleQuotes(boolean checkIfUnicode) throws InvalidInputException{
+	protected TerminalToken processSingleQuotes(boolean checkIfUnicode) throws InvalidInputException{
 		if (this.tokenizeSingleQuotes) {
 			return scanForSingleQuoteStringLiteral();
 		} else {
 			return super.processSingleQuotes(checkIfUnicode);
 		}
 	}
 
-	protected int scanForSingleQuoteStringLiteral() throws InvalidInputException {
+	protected TerminalToken scanForSingleQuoteStringLiteral() throws InvalidInputException {
 		this.unicodeAsBackSlash = false;
 		boolean isUnicode = false;
 		try {