Add PSLR runtime fallback and LAC support

Author: ydah

lib/lrama/output.rb

@@ -424,7 +424,19 @@ def pslr_function_declarations
 
       if pslr_scanner_enabled?
         declarations << <<~C_CODE
+          #ifndef YYPSLR_SCAN_RESULT_DEFINED
+          # define YYPSLR_SCAN_RESULT_DEFINED
+          typedef struct yypslr_scan_result {
+            int token;
+            int length;
+            int is_layout;
+            int is_character_token;
+          } yypslr_scan_result;
+          #endif
+
+          int yy_pseudo_scan_result (int parser_state, const char *input, yypslr_scan_result *result);
           int yy_pseudo_scan (int parser_state, const char *input, int *match_length);
+          int yy_pslr_token_is_layout (int token);
         C_CODE
 
         declarations << <<~C_CODE
@@ -435,6 +447,15 @@ def pslr_function_declarations
           # define YYPSLR_PSEUDO_SCAN_STATE(ParserState, Input, MatchLength) \\
             yy_pseudo_scan ((ParserState), (Input), (MatchLength))
           #endif
+
+          #ifndef YYPSLR_PSEUDO_SCAN_RESULT_STATE
+          # define YYPSLR_PSEUDO_SCAN_RESULT_STATE(ParserState, Input, Result) \\
+            yy_pseudo_scan_result ((ParserState), (Input), (Result))
+          #endif
+
+          #ifndef YYPSLR_TOKEN_IS_LAYOUT
+          # define YYPSLR_TOKEN_IS_LAYOUT(Token) yy_pslr_token_is_layout ((Token))
+          #endif
         C_CODE
       end
 
@@ -453,6 +474,13 @@ def pslr_function_declarations
                ? YYPSLR_PSEUDO_SCAN_STATE (YYGETSTATE_CONTEXT (Context), (Input), (MatchLength)) \\
                : YYEMPTY)
             #endif
+
+            #ifndef YYPSLR_PSEUDO_SCAN_RESULT
+            # define YYPSLR_PSEUDO_SCAN_RESULT(Context, Input, Result) \\
+              ((Context) != 0 \\
+               ? YYPSLR_PSEUDO_SCAN_RESULT_STATE (YYGETSTATE_CONTEXT (Context), (Input), (Result)) \\
+               : YYEMPTY)
+            #endif
           C_CODE
         end
 
@@ -549,6 +577,20 @@ def token_pattern_token_ids_table
       lines << "static const int yy_token_pattern_to_token_id[YY_NUM_TOKEN_PATTERNS] = {"
       lines << "  #{@context.states.token_patterns.map {|token_pattern| pslr_token_id(token_pattern) }.join(', ')}"
       lines << "};"
+      lines << ""
+      lines << "static const int yy_token_pattern_is_layout[YY_NUM_TOKEN_PATTERNS] = {"
+      lines << "  #{@context.states.token_patterns.map {|token_pattern| token_pattern.layout? ? 1 : 0 }.join(', ')}"
+      lines << "};"
+      lines << ""
+      lines << "int"
+      lines << "yy_pslr_token_is_layout (int token)"
+      lines << "{"
+      lines << "  int i;"
+      lines << "  for (i = 0; i < YY_NUM_TOKEN_PATTERNS; i++)"
+      lines << "    if (yy_token_pattern_to_token_id[i] == token)"
+      lines << "      return yy_token_pattern_is_layout[i];"
+      lines << "  return 0;"
+      lines << "}"
       lines.join("\n")
     end
 
@@ -617,6 +659,14 @@ def scanner_accepts_table_code
         end
 
         lines << "};"
+        lines << ""
+        lines << "static const int yy_scanner_fallback_accepts[YY_NUM_ACCEPTING_STATES] = {"
+        fallback_row = pslr_accepting_states.map do |fsa_state|
+          token = scanner_accepts.fallback_table[fsa_state.id]
+          token ? token_pattern_indexes.fetch(token) : -1
+        end
+        lines << "  #{fallback_row.join(', ')}"
+        lines << "};"
       end
 
       lines.join("\n")
@@ -651,6 +701,95 @@ def length_precedences_table_code
       lines.join("\n")
     end
 
+    def pslr_lac_function
+      return "" unless pslr_enabled?
+
+      <<~C_CODE
+
+        static int
+        yy_lac_check_ (yy_state_t *yyss, yy_state_t *yyssp, yysymbol_kind_t yytoken)
+        {
+          YYPTRDIFF_T yylac_len = yyssp - yyss + 1;
+          yy_state_t *yylac_base = YY_CAST (yy_state_t *,
+            YYMALLOC (YY_CAST (YYSIZE_T, YYMAXDEPTH * YYSIZEOF (yy_state_t))));
+          yy_state_t *yylac_top;
+
+          if (!yylac_base)
+            return 1;
+
+          if (YYMAXDEPTH < yylac_len)
+            {
+              YYFREE (yylac_base);
+              return 1;
+            }
+
+          YYCOPY (yylac_base, yyss, yylac_len);
+          yylac_top = yylac_base + yylac_len - 1;
+
+          for (;;)
+            {
+              int yystate = *yylac_top;
+              int yyn = yypact[yystate];
+
+              if (!yypact_value_is_default (yyn))
+                {
+                  yyn += yytoken;
+                  if (0 <= yyn && yyn <= YYLAST && yycheck[yyn] == yytoken)
+                    {
+                      yyn = yytable[yyn];
+                      if (0 < yyn)
+                        {
+                          YYFREE (yylac_base);
+                          return 1;
+                        }
+                      if (yytable_value_is_error (yyn))
+                        {
+                          YYFREE (yylac_base);
+                          return 0;
+                        }
+                      yyn = -yyn;
+                    }
+                  else
+                    {
+                      yyn = yydefact[yystate];
+                      if (yyn == 0)
+                        {
+                          YYFREE (yylac_base);
+                          return 0;
+                        }
+                    }
+                }
+              else
+                {
+                  yyn = yydefact[yystate];
+                  if (yyn == 0)
+                    {
+                      YYFREE (yylac_base);
+                      return 0;
+                    }
+                }
+
+              yylac_top -= yyr2[yyn];
+              {
+                const int yylhs = yyr1[yyn] - YYNTOKENS;
+                const int yyi = yypgoto[yylhs] + *yylac_top;
+                yystate = (0 <= yyi && yyi <= YYLAST && yycheck[yyi] == *yylac_top
+                           ? yytable[yyi]
+                           : yydefgoto[yylhs]);
+              }
+
+              if (yylac_base + YYMAXDEPTH - 1 <= yylac_top)
+                {
+                  YYFREE (yylac_base);
+                  return 1;
+                }
+
+              *++yylac_top = YY_CAST (yy_state_t, yystate);
+            }
+        }
+      C_CODE
+    end
+
     # Generate pseudo_scan function as C code
     def pseudo_scan_function
       return "" unless pslr_scanner_enabled?
@@ -666,27 +805,36 @@ def pseudo_scan_function
          *   input: Input buffer pointer
          *   match_length: Output parameter for matched length
          *
-         * Returns: Selected parser token ID, or YYEMPTY if no match
+         * Returns: Selected parser token ID. If neither the parser-state row
+         * nor fallback row matches, returns YYUNDEF and consumes one byte.
          */
         int
-        yy_pseudo_scan(int parser_state, const char *input, int *match_length)
+        yy_pseudo_scan_result (int parser_state, const char *input, yypslr_scan_result *result)
         {
-          int local_match_length = 0;
           int ss = 0;  /* FSA initial state */
           int ibest = 0;
           int pbest = YY_PSLR_EMPTY_PATTERN;
+          int fallback_ibest = 0;
+          int fallback_pbest = YY_PSLR_EMPTY_PATTERN;
           int i = 0;
 
-          if (match_length == NULL) {
-            match_length = &local_match_length;
+          if (result == NULL) {
+            return YYEMPTY;
           }
 
-          *match_length = 0;
+          result->token = YYEMPTY;
+          result->length = 0;
+          result->is_layout = 0;
+          result->is_character_token = 0;
 
           if (parser_state < 0 || parser_state >= YY_NUM_PARSER_STATES || input == NULL) {
             return YYEMPTY;
           }
 
+          if (input[0] == '\\0') {
+            return YYEMPTY;
+          }
+
           while (input[i] != '\\0') {
             int c = (unsigned char)input[i];
             int next_ss = yy_scanner_transition[ss][c];
@@ -709,15 +857,46 @@ def pseudo_scan_function
                   ibest = i;
                 }
               }
+
+              pattern_index = yy_scanner_fallback_accepts[sa];
+              if (pattern_index != YY_PSLR_EMPTY_PATTERN) {
+                fallback_pbest = pattern_index;
+                fallback_ibest = i;
+              }
             }
           }
 
-          *match_length = ibest;
-          if (pbest == YY_PSLR_EMPTY_PATTERN) {
-            return YYEMPTY;
+          if (pbest != YY_PSLR_EMPTY_PATTERN) {
+            result->token = yy_token_pattern_to_token_id[pbest];
+            result->length = ibest;
+            result->is_layout = yy_token_pattern_is_layout[pbest];
+            return result->token;
+          }
+
+          if (fallback_pbest != YY_PSLR_EMPTY_PATTERN) {
+            result->token = yy_token_pattern_to_token_id[fallback_pbest];
+            result->length = fallback_ibest;
+            result->is_layout = yy_token_pattern_is_layout[fallback_pbest];
+            return result->token;
+          }
+
+          result->token = YYUNDEF;
+          result->length = 1;
+          result->is_character_token = 1;
+          return result->token;
+        }
+
+        int
+        yy_pseudo_scan(int parser_state, const char *input, int *match_length)
+        {
+          yypslr_scan_result result;
+          int token = yy_pseudo_scan_result (parser_state, input, &result);
+
+          if (match_length != NULL) {
+            *match_length = result.length;
           }
 
-          return yy_token_pattern_to_token_id[pbest];
+          return token;
         }
       C_CODE
     end

spec/fixtures/integration/pslr_template_argument_lists.l

@@ -0,0 +1,34 @@
+%option noinput nounput noyywrap never-interactive
+
+%{
+#include <stdio.h>
+#include "pslr_template_argument_lists.h"
+
+#define YY_DECL int yylex(YYSTYPE *yylval, struct parse_params *p)
+%}
+
+%%
+
+(.|\n)+ {
+    yypslr_scan_result result;
+    int token;
+
+    (void)yylval;
+    token = YYPSLR_PSEUDO_SCAN_RESULT(p, yytext, &result);
+    if (result.length <= 0) {
+        result.length = 1;
+    }
+    if (result.length < yyleng) {
+        yyless(result.length);
+    }
+    if (result.is_layout) {
+        return yylex(yylval, p);
+    }
+    return token;
+}
+
+<<EOF>> {
+    return YYEOF;
+}
+
+%%