Skip to content

Commit 665a565

Browse files
kddnewtonkddnewton
committed
Refactor do keyword parsing
Mirror much more closely the parse.y COND and CMDARG stacks. This fixes a whole host of really really weird edge cases.
1 parent 880833d commit 665a565

2 files changed

Lines changed: 92 additions & 49 deletions

File tree

‎src/prism.c‎

Lines changed: 78 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -7405,6 +7405,42 @@ pm_do_loop_stack_p(pm_parser_t *parser) {
74057405
return pm_state_stack_p(&parser->do_loop_stack);
74067406
}
74077407

7408+
/**
7409+
* When the lexer finds an opening delimiter (`(`, `[`, `{`, or `#{`) it pushes
7410+
* an enclosure frame onto both bit-stacks that gate how a subsequent `do` is
7411+
* lexed: the do-loop stack (so a `do` inside is not read as a `while`/`until`
7412+
* loop body) and the accepts-block stack (so a `do` inside is accepted as a
7413+
* block rather than binding to an enclosing command). The matching close pops
7414+
* both. This mirrors parse.y's paired `COND_PUSH(0); CMDARG_PUSH(0)`. Keep the
7415+
* following in mind before touching either stack:
7416+
*
7417+
* - The LEXER owns the frame for delimiter-bound stuff. Every token that opens
7418+
* a matched pair calls this on the open and `pm_enclosure_frame_pop` on the
7419+
* close. The exhaustive push sites are the `(`, `[`, `{` cases in
7420+
* `parser_lex` and the `#{` case in `lex_interpolation`; the pop sites are
7421+
* the matching `)`, `]`, `}` (including `}` as `EMBEXPR_END`).
7422+
* - The PARSER owns the frame for keyword-bounded blocks, where there is no
7423+
* delimiter token to hang it on: the `do`/`end` forms in `parse_block` and
7424+
* the lambda push `accepts_block` directly (and pop before consuming `end`).
7425+
* - The `parse_arguments_list` command-args branch juggles ONLY `accepts_block`
7426+
* (never through this helper), because the lexer has pushed a delimiter frame
7427+
* one token early and the command-args frame must be threaded beneath it. Its
7428+
* match lookahead sets must stay in sync with the lexer push sites above.
7429+
* This matches parse.y, whose `command_args` rule juggles CMDARG but not
7430+
* COND.
7431+
*/
7432+
static PRISM_INLINE void
7433+
pm_enclosure_frame_push(pm_parser_t *parser) {
7434+
pm_do_loop_stack_push(parser, false);
7435+
pm_accepts_block_stack_push(parser, true);
7436+
}
7437+
7438+
static PRISM_INLINE void
7439+
pm_enclosure_frame_pop(pm_parser_t *parser) {
7440+
pm_do_loop_stack_pop(parser);
7441+
pm_accepts_block_stack_pop(parser);
7442+
}
7443+
74087444
/******************************************************************************/
74097445
/* Lexer check helpers */
74107446
/******************************************************************************/
@@ -8789,7 +8825,7 @@ lex_interpolation(pm_parser_t *parser, const uint8_t *pound) {
87898825
lex_mode_push(parser, (pm_lex_mode_t) { .mode = PM_LEX_EMBEXPR });
87908826
parser->current.end = pound + 2;
87918827
parser->command_start = true;
8792-
pm_do_loop_stack_push(parser, false);
8828+
pm_enclosure_frame_push(parser);
87938829
return PM_TOKEN_EMBEXPR_BEGIN;
87948830
default:
87958831
// In this case we've hit a # that doesn't constitute interpolation. We'll
@@ -10394,15 +10430,15 @@ parser_lex(pm_parser_t *parser) {
1039410430

1039510431
parser->enclosure_nesting++;
1039610432
lex_state_set(parser, PM_LEX_STATE_BEG | PM_LEX_STATE_LABEL);
10397-
pm_do_loop_stack_push(parser, false);
10433+
pm_enclosure_frame_push(parser);
1039810434
LEX(type);
1039910435
}
1040010436

1040110437
// )
1040210438
case ')':
1040310439
parser->enclosure_nesting--;
1040410440
lex_state_set(parser, PM_LEX_STATE_ENDFN);
10405-
pm_do_loop_stack_pop(parser);
10441+
pm_enclosure_frame_pop(parser);
1040610442
LEX(PM_TOKEN_PARENTHESIS_RIGHT);
1040710443

1040810444
// ;
@@ -10432,14 +10468,14 @@ parser_lex(pm_parser_t *parser) {
1043210468
}
1043310469

1043410470
lex_state_set(parser, PM_LEX_STATE_BEG | PM_LEX_STATE_LABEL);
10435-
pm_do_loop_stack_push(parser, false);
10471+
pm_enclosure_frame_push(parser);
1043610472
LEX(type);
1043710473

1043810474
// ]
1043910475
case ']':
1044010476
parser->enclosure_nesting--;
1044110477
lex_state_set(parser, PM_LEX_STATE_END);
10442-
pm_do_loop_stack_pop(parser);
10478+
pm_enclosure_frame_pop(parser);
1044310479
LEX(PM_TOKEN_BRACKET_RIGHT);
1044410480

1044510481
// {
@@ -10469,15 +10505,15 @@ parser_lex(pm_parser_t *parser) {
1046910505

1047010506
parser->enclosure_nesting++;
1047110507
parser->brace_nesting++;
10472-
pm_do_loop_stack_push(parser, false);
10508+
pm_enclosure_frame_push(parser);
1047310509

1047410510
LEX(type);
1047510511
}
1047610512

1047710513
// }
1047810514
case '}':
1047910515
parser->enclosure_nesting--;
10480-
pm_do_loop_stack_pop(parser);
10516+
pm_enclosure_frame_pop(parser);
1048110517

1048210518
if ((parser->lex_modes.current->mode == PM_LEX_EMBEXPR) && (parser->brace_nesting == 0)) {
1048310519
lex_mode_pop(parser);
@@ -15261,7 +15297,12 @@ parse_block(pm_parser_t *parser, uint16_t depth) {
1526115297
pm_token_t opening = parser->previous;
1526215298
accept1(parser, PM_TOKEN_NEWLINE);
1526315299

15264-
pm_accepts_block_stack_push(parser, true);
15300+
/* A brace block is delimited by `{`/`}`, whose block-accepting frame is
15301+
* managed by the lexer. A `do`/`end` block is delimited by keywords, so we
15302+
* push the frame here (covering the block parameters and body) and pop it
15303+
* before consuming `end`, mirroring parse.y's `do_body` rule. */
15304+
bool do_block = opening.type != PM_TOKEN_BRACE_LEFT;
15305+
if (do_block) pm_accepts_block_stack_push(parser, true);
1526515306
pm_parser_scope_push(parser, false);
1526615307

1526715308
pm_block_parameters_node_t *block_parameters = NULL;
@@ -15290,19 +15331,11 @@ parse_block(pm_parser_t *parser, uint16_t depth) {
1529015331
statements = UP(parse_statements(parser, PM_CONTEXT_BLOCK_BRACES, (uint16_t) (depth + 1)));
1529115332
}
1529215333

15293-
/* Pop before consuming the closing `}` so the following token (e.g. a
15294-
* `do`) is lexed in the enclosing context rather than as a block
15295-
* belonging to this block's interior. Otherwise a `do` block would
15296-
* wrongly bind to a command whose argument ends in a brace block, as in
15297-
* `foo(m a { } do end)`. */
15298-
pm_accepts_block_stack_pop(parser);
1529915334
expect1_opening(parser, PM_TOKEN_BRACE_RIGHT, PM_ERR_BLOCK_TERM_BRACE, &opening);
1530015335
} else {
1530115336
if (!match1(parser, PM_TOKEN_KEYWORD_END)) {
1530215337
if (!match3(parser, PM_TOKEN_KEYWORD_RESCUE, PM_TOKEN_KEYWORD_ELSE, PM_TOKEN_KEYWORD_ENSURE)) {
15303-
pm_accepts_block_stack_push(parser, true);
1530415338
statements = UP(parse_statements(parser, PM_CONTEXT_BLOCK_KEYWORDS, (uint16_t) (depth + 1)));
15305-
pm_accepts_block_stack_pop(parser);
1530615339
}
1530715340

1530815341
if (match2(parser, PM_TOKEN_KEYWORD_RESCUE, PM_TOKEN_KEYWORD_ENSURE)) {
@@ -15311,7 +15344,8 @@ parse_block(pm_parser_t *parser, uint16_t depth) {
1531115344
}
1531215345
}
1531315346

15314-
/* As with the brace case above, pop before consuming `end`. */
15347+
/* Pop the `do`/`end` frame before consuming `end` so the token
15348+
* following the block is lexed in the enclosing context. */
1531515349
pm_accepts_block_stack_pop(parser);
1531615350
expect1_opening(parser, PM_TOKEN_KEYWORD_END, PM_ERR_BLOCK_TERM_END, &opening);
1531715351
}
@@ -15321,7 +15355,6 @@ parse_block(pm_parser_t *parser, uint16_t depth) {
1532115355
pm_node_t *parameters = parse_blocklike_parameters(parser, UP(block_parameters), &opening, &parser->previous);
1532215356

1532315357
pm_parser_scope_pop(parser);
15324-
1532515358
return pm_block_node_create(parser, &locals, &opening, parameters, statements, &parser->previous);
1532615359
}
1532715360

@@ -15358,7 +15391,6 @@ parse_arguments_list(pm_parser_t *parser, pm_arguments_t *arguments, bool full_a
1535815391
if (accept1(parser, PM_TOKEN_PARENTHESIS_RIGHT)) {
1535915392
arguments->closing_loc = TOK2LOC(parser, &parser->previous);
1536015393
} else {
15361-
pm_accepts_block_stack_push(parser, true);
1536215394
parse_arguments(parser, arguments, full_arguments, PM_TOKEN_PARENTHESIS_RIGHT, (uint8_t) (flags & ~PM_PARSE_ACCEPTS_DO_BLOCK), (uint16_t) (depth + 1));
1536315395

1536415396
// `yield` parses its arguments through the restricted `call_args`
@@ -15377,13 +15409,24 @@ parse_arguments_list(pm_parser_t *parser, pm_arguments_t *arguments, bool full_a
1537715409
parser->previous.type = 0;
1537815410
}
1537915411

15380-
pm_accepts_block_stack_pop(parser);
1538115412
arguments->closing_loc = TOK2LOC(parser, &parser->previous);
1538215413
}
1538315414
} else if ((flags & PM_PARSE_ACCEPTS_COMMAND_CALL) && (token_begins_expression_p(parser->current.type) || match3(parser, PM_TOKEN_USTAR, PM_TOKEN_USTAR_STAR, PM_TOKEN_UAMPERSAND)) && !match1(parser, PM_TOKEN_BRACE_LEFT)) {
1538415415
found |= true;
1538515416
parsed_command_args = true;
15417+
15418+
/* The command-args frame does not accept blocks, so that a trailing
15419+
* `do` binds to this command rather than to an argument. Mirroring
15420+
* parse.y's `command_args` rule: when the first argument begins with an
15421+
* opening delimiter, the lexer has already pushed that delimiter's
15422+
* (block-accepting) frame. We must push the command-args frame beneath
15423+
* it, so pop the delimiter frame, push the command-args frame, and then
15424+
* restore the delimiter frame on top (the delimiter's closing token
15425+
* will pop it back off during argument parsing). */
15426+
bool lookahead_delimiter = match4(parser, PM_TOKEN_PARENTHESIS_LEFT, PM_TOKEN_PARENTHESIS_LEFT_PARENTHESES, PM_TOKEN_BRACKET_LEFT, PM_TOKEN_BRACKET_LEFT_ARRAY);
15427+
if (lookahead_delimiter) pm_accepts_block_stack_pop(parser);
1538615428
pm_accepts_block_stack_push(parser, false);
15429+
if (lookahead_delimiter) pm_accepts_block_stack_push(parser, true);
1538715430

1538815431
// If we get here, then the subsequent token cannot be used as an infix
1538915432
// operator. In this case we assume the subsequent token is part of an
@@ -15397,7 +15440,15 @@ parse_arguments_list(pm_parser_t *parser, pm_arguments_t *arguments, bool full_a
1539715440
PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->previous, PM_ERR_EXPECT_ARGUMENT, pm_token_str(parser->current.type));
1539815441
}
1539915442

15443+
/* Symmetrically, if the command arguments are followed by a brace block
15444+
* (`m args { }`), the lexer has already pushed that block's frame. Pop
15445+
* it, pop the command-args frame beneath it, and restore the block
15446+
* frame so the block's `}` still pops it. This mirrors the `tLBRACE_ARG`
15447+
* lookahead handling in parse.y's `command_args` rule. */
15448+
bool lookahead_brace = match1(parser, PM_TOKEN_BRACE_LEFT);
15449+
if (lookahead_brace) pm_accepts_block_stack_pop(parser);
1540015450
pm_accepts_block_stack_pop(parser);
15451+
if (lookahead_brace) pm_accepts_block_stack_push(parser, true);
1540115452
}
1540215453

1540315454
// If we're at the end of the arguments, we can now check if there is a block
@@ -15961,9 +16012,7 @@ parse_string_part(pm_parser_t *parser, uint16_t depth) {
1596116012
pm_statements_node_t *statements = NULL;
1596216013

1596316014
if (!match3(parser, PM_TOKEN_EMBEXPR_END, PM_TOKEN_HEREDOC_END, PM_TOKEN_EOF)) {
15964-
pm_accepts_block_stack_push(parser, true);
1596516015
statements = parse_statements(parser, PM_CONTEXT_EMBEXPR, (uint16_t) (depth + 1));
15966-
pm_accepts_block_stack_pop(parser);
1596716016
}
1596816017

1596916018
parser->brace_nesting = brace_nesting;
@@ -19028,7 +19077,6 @@ parse_parentheses(pm_parser_t *parser, pm_binding_power_t binding_power, uint8_t
1902819077

1902919078
/* Otherwise, we're going to parse the first statement in the list of
1903019079
* statements within the parentheses. */
19031-
pm_accepts_block_stack_push(parser, true);
1903219080
context_push(parser, PM_CONTEXT_PARENS);
1903319081
pm_node_t *statement = parse_expression(parser, PM_BINDING_POWER_STATEMENT, PM_PARSE_ACCEPTS_COMMAND_CALL | PM_PARSE_ACCEPTS_DO_BLOCK, PM_ERR_CANNOT_PARSE_EXPRESSION, (uint16_t) (depth + 1));
1903419082
context_pop(parser);
@@ -19062,10 +19110,6 @@ parse_parentheses(pm_parser_t *parser, pm_binding_power_t binding_power, uint8_t
1906219110
lex_state_set(parser, PM_LEX_STATE_ENDARG);
1906319111
}
1906419112

19065-
/* Pop before consuming the closing `)` so the following token (e.g. a
19066-
* `do`) is lexed in the enclosing context rather than as a block
19067-
* belonging to the parenthesized expression. */
19068-
pm_accepts_block_stack_pop(parser);
1906919113
parser_lex(parser);
1907019114
pop_block_exits(parser, previous_block_exits);
1907119115

@@ -19177,7 +19221,6 @@ parse_parentheses(pm_parser_t *parser, pm_binding_power_t binding_power, uint8_t
1917719221
}
1917819222

1917919223
context_pop(parser);
19180-
pm_accepts_block_stack_pop(parser);
1918119224
expect1(parser, PM_TOKEN_PARENTHESIS_RIGHT, PM_ERR_EXPECT_RPAREN);
1918219225

1918319226
/* When we're parsing multi targets, we allow them to be followed by a right
@@ -19239,7 +19282,6 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, u
1923919282
parser_lex(parser);
1924019283

1924119284
pm_array_node_t *array = pm_array_node_create(parser, &parser->previous);
19242-
pm_accepts_block_stack_push(parser, true);
1924319285
bool parsed_bare_hash = false;
1924419286

1924519287
while (!match2(parser, PM_TOKEN_BRACKET_RIGHT, PM_TOKEN_EOF)) {
@@ -19337,13 +19379,6 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, u
1933719379

1933819380
accept1(parser, PM_TOKEN_NEWLINE);
1933919381

19340-
/* Pop before consuming the closing `]` so the following token (e.g.
19341-
* a `do`) is lexed in the enclosing context rather than as a block
19342-
* belonging to the array's interior. Otherwise a `do` block would
19343-
* wrongly bind to a command with an array argument, as in
19344-
* `foo(m [] do end)`. */
19345-
pm_accepts_block_stack_pop(parser);
19346-
1934719382
if (!accept1(parser, PM_TOKEN_BRACKET_RIGHT)) {
1934819383
PM_PARSER_ERR_TOKEN_FORMAT(parser, &parser->current, PM_ERR_ARRAY_TERM, pm_token_str(parser->current.type));
1934919384
parser->previous.start = parser->previous.end;
@@ -19368,7 +19403,6 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, u
1936819403
pm_static_literals_t *current_hash_keys = parser->current_hash_keys;
1936919404
parser->current_hash_keys = NULL;
1937019405

19371-
pm_accepts_block_stack_push(parser, true);
1937219406
parser_lex(parser);
1937319407

1937419408
pm_token_t opening = parser->previous;
@@ -19386,7 +19420,6 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, u
1938619420
accept1(parser, PM_TOKEN_NEWLINE);
1938719421
}
1938819422

19389-
pm_accepts_block_stack_pop(parser);
1939019423
expect1_opening(parser, PM_TOKEN_BRACE_RIGHT, PM_ERR_HASH_TERM, &opening);
1939119424
pm_hash_node_closing_loc_set(parser, node, &parser->previous);
1939219425

@@ -20620,7 +20653,6 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, u
2062020653
parser->lambda_enclosure_nesting = parser->enclosure_nesting;
2062120654

2062220655
size_t opening_newline_index = token_newline_index(parser);
20623-
pm_accepts_block_stack_push(parser, true);
2062420656
parser_lex(parser);
2062520657

2062620658
pm_token_t operator = parser->previous;
@@ -20669,16 +20701,16 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, u
2066920701
}
2067020702

2067120703
parser_warn_indentation_mismatch(parser, opening_newline_index, &operator, false, false);
20672-
20673-
/* Pop before consuming the closing `}` so the following token
20674-
* (e.g. a `do`) is lexed in the enclosing context rather than
20675-
* as a block belonging to the lambda's interior. */
20676-
pm_accepts_block_stack_pop(parser);
2067720704
expect1_opening(parser, PM_TOKEN_BRACE_RIGHT, PM_ERR_LAMBDA_TERM_BRACE, &opening);
2067820705
} else {
2067920706
expect1(parser, PM_TOKEN_KEYWORD_DO, PM_ERR_LAMBDA_OPEN);
2068020707
opening = parser->previous;
2068120708

20709+
/* A `-> { }` body is delimited by `{`/`}`, whose block-accepting
20710+
* frame the lexer manages. A `-> do end` body is delimited by
20711+
* keywords, so push the frame here and pop it before `end`. */
20712+
pm_accepts_block_stack_push(parser, true);
20713+
2068220714
if (!match3(parser, PM_TOKEN_KEYWORD_END, PM_TOKEN_KEYWORD_RESCUE, PM_TOKEN_KEYWORD_ENSURE)) {
2068320715
body = UP(parse_statements(parser, PM_CONTEXT_LAMBDA_DO_END, (uint16_t) (depth + 1)));
2068420716
}
@@ -20690,7 +20722,6 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, u
2069020722
parser_warn_indentation_mismatch(parser, opening_newline_index, &operator, false, false);
2069120723
}
2069220724

20693-
/* As with the brace case above, pop before consuming `end`. */
2069420725
pm_accepts_block_stack_pop(parser);
2069520726
expect1_opening(parser, PM_TOKEN_KEYWORD_END, PM_ERR_LAMBDA_TERM_END, &operator);
2069620727
}
@@ -22134,9 +22165,7 @@ parse_expression_infix(pm_parser_t *parser, pm_node_t *node, pm_binding_power_t
2213422165
arguments.opening_loc = TOK2LOC(parser, &parser->previous);
2213522166

2213622167
if (!accept1(parser, PM_TOKEN_BRACKET_RIGHT)) {
22137-
pm_accepts_block_stack_push(parser, true);
2213822168
parse_arguments(parser, &arguments, false, PM_TOKEN_BRACKET_RIGHT, (uint8_t) (flags & ~PM_PARSE_ACCEPTS_DO_BLOCK), (uint16_t) (depth + 1));
22139-
pm_accepts_block_stack_pop(parser);
2214022169
expect1(parser, PM_TOKEN_BRACKET_RIGHT, PM_ERR_EXPECT_RBRACKET);
2214122170
}
2214222171

@@ -22275,9 +22304,9 @@ parse_expression_terminator(pm_parser_t *parser, pm_node_t *node) {
2227522304
if (pm_command_call_value_p(parser, node)) {
2227622305
return left > PM_BINDING_POWER_COMPOSITION;
2227722306
}
22307+
2227822308
/* A super carrying a do-block is a block call, so it may also be
22279-
* followed by call chaining (`.`, `::`, `&.`).
22280-
*/
22309+
* followed by call chaining (`.`, `::`, `&.`). */
2228122310
if (pm_block_call_p(node)) {
2228222311
return left > PM_BINDING_POWER_COMPOSITION && left < PM_BINDING_POWER_CALL;
2228322312
}

‎test/prism/errors/do_block_after_command_paren_call.txt‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
foo(m a(1) do end)
2+
^~ unexpected 'do'; expected a `)` to close the arguments
3+
^~ unexpected 'do', expecting end-of-input
4+
^~ unexpected 'do', ignoring it
5+
^~~ unexpected 'end', ignoring it
6+
^ unexpected ')', ignoring it
7+
8+
foo(m a.b(1) do end)
9+
^~ unexpected 'do'; expected a `)` to close the arguments
10+
^~ unexpected 'do', expecting end-of-input
11+
^~ unexpected 'do', ignoring it
12+
^~~ unexpected 'end', ignoring it
13+
^ unexpected ')', ignoring it
14+

0 commit comments

Comments
 (0)