java-parser: New entrypoint lexAndParse to return both tokens and CST.

Author: Max Schaefer

packages/java-parser/api.d.ts

@@ -30,6 +30,14 @@ export declare type CstChildrenDictionary = {
   [identifier: string]: CstElement[];
 };
 
+export function lexAndParse(
+  text: string,
+  startProduction?: string
+): {
+  tokens: IToken[];
+  cst: CstNode;
+};
+
 export function parse(text: string, startProduction?: string): CstNode;
 
 export const BaseJavaCstVisitor: JavaCstVisitorConstructor<any, any>;

packages/java-parser/src/index.js

@@ -9,7 +9,7 @@ const BaseJavaCstVisitor = parser.getBaseCstVisitorConstructor();
 const BaseJavaCstVisitorWithDefaults =
   parser.getBaseCstVisitorConstructorWithDefaults();
 
-function parse(inputText, entryPoint = "compilationUnit") {
+function lexAndParse(inputText, entryPoint = "compilationUnit") {
   // Lex
   const lexResult = JavaLexer.tokenize(inputText);
 
@@ -25,7 +25,8 @@ function parse(inputText, entryPoint = "compilationUnit") {
     );
   }
 
-  parser.input = lexResult.tokens;
+  const tokens = lexResult.tokens;
+  parser.input = tokens;
   parser.mostEnclosiveCstNodeByStartOffset = {};
   parser.mostEnclosiveCstNodeByEndOffset = {};
 
@@ -51,16 +52,21 @@ function parse(inputText, entryPoint = "compilationUnit") {
   }
 
   attachComments(
-    lexResult.tokens,
+    tokens,
     lexResult.groups.comments,
     parser.mostEnclosiveCstNodeByStartOffset,
     parser.mostEnclosiveCstNodeByEndOffset
   );
 
-  return cst;
+  return { cst, tokens };
+}
+
+function parse(inputText, entryPoint = "compilationUnit") {
+  return lexAndParse(inputText, entryPoint).cst;
 }
 
 module.exports = {
+  lexAndParse,
   parse,
   BaseJavaCstVisitor,
   BaseJavaCstVisitorWithDefaults