Polish grammar bundle builder and stabilize grammar endpoint doctest

Signed-off-by: Eric Wei <mengwei.eric@gmail.com>

Author: mengweieric

docs/user/ppl/interfaces/endpoint.md

@@ -211,25 +211,15 @@ You can send an HTTP GET request to endpoint **/_plugins/_ppl/_grammar** to fetc
 ### Example
 
 ```bash
-curl -sS -X GET localhost:9200/_plugins/_ppl/_grammar
+curl -sS -X GET localhost:9200/_plugins/_ppl/_grammar | python3 -m json.tool | grep -E '"bundleVersion"|"antlrVersion"|"startRuleIndex"|"ignoredTokens"|"rulesToVisit"'
 ```
 
 Expected output (trimmed):
 
-```json
-{
-  "bundleVersion": "1.0",
-  "antlrVersion": "4.13.2",
-  "grammarHash": "sha256:...",
-  "startRuleIndex": 0,
-  "lexerSerializedATN": [4, ...],
-  "parserSerializedATN": [4, ...],
-  "lexerRuleNames": ["SEARCH", "..."],
-  "parserRuleNames": ["root", "..."],
-  "literalNames": [null, "'SEARCH'", "..."],
-  "symbolicNames": [null, "SEARCH", "..."],
-  "tokenDictionary": {"PIPE": 196, "...": 0},
-  "ignoredTokens": [472, 473, "..."],
-  "rulesToVisit": [200, 201, "..."]
-}
+```text
+"bundleVersion": "1.0",
+"antlrVersion": "4.13.2",
+"startRuleIndex": 0,
+"ignoredTokens": [
+"rulesToVisit": [
 ```

ppl/src/main/java/org/opensearch/sql/ppl/autocomplete/PPLGrammarBundleBuilder.java

@@ -81,7 +81,7 @@ public GrammarBundle build() {
         .startRuleIndex(resolveStartRuleIndex(parser.getRuleNames()))
         .literalNames(literalNames)
         .symbolicNames(symbolicNames)
-        .tokenDictionary(buildTokenDictionary(vocabulary))
+        .tokenDictionary(buildTokenDictionary())
         .ignoredTokens(buildIgnoredTokens(vocabulary))
         .rulesToVisit(buildRulesToVisit(parser.getRuleNames()))
         .build();
@@ -90,12 +90,10 @@ public GrammarBundle build() {
   /**
    * Build the token dictionary — semantic name → token type ID mapping. Uses lexer constants since
    * token type IDs are defined by the lexer. The frontend autocomplete enrichment uses these to
-   * identify tokens like SPACE, PIPE, SOURCE by name.
+   * identify tokens like PIPE and SOURCE by name.
    */
-  private static Map<String, Integer> buildTokenDictionary(Vocabulary vocabulary) {
+  private static Map<String, Integer> buildTokenDictionary() {
     Map<String, Integer> dict = new LinkedHashMap<>();
-    // SPACE token may not exist in this grammar (whitespace may be implicitly skipped).
-    // Resolve by searching symbolic names; use -1 if not found.
     dict.put("WHITESPACE", OpenSearchPPLLexer.WHITESPACE);
     dict.put("FROM", OpenSearchPPLLexer.FROM);
     dict.put("OPENING_BRACKET", OpenSearchPPLLexer.LT_PRTHS);
@@ -182,16 +180,6 @@ private static int[] buildRulesToVisit(String[] ruleNames) {
     return indices;
   }
 
-  /** Resolve a token type ID from the vocabulary by symbolic name. Returns -1 if not found. */
-  private static int resolveTokenType(Vocabulary vocabulary, String name) {
-    for (int i = 0; i <= vocabulary.getMaxTokenType(); i++) {
-      if (name.equals(vocabulary.getSymbolicName(i))) {
-        return i;
-      }
-    }
-    return -1;
-  }
-
   private static int resolveStartRuleIndex(String[] ruleNames) {
     int idx = Arrays.asList(ruleNames).indexOf("root");
     return Math.max(idx, 0);