Skip to content

Commit 483aa89

Browse files
committed
Optimize context_terminator with a lookup table
1 parent c4e388d commit 483aa89

3 files changed

Lines changed: 96 additions & 123 deletions

File tree

config.yml

Lines changed: 37 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -320,13 +320,42 @@ warnings:
320320
- UNUSED_LOCAL_VARIABLE
321321
- VOID_STATEMENT
322322
tokens:
323+
# The order of the tokens at the beginning is important, because we use them
324+
# for a lookup table.
323325
- name: EOF
324326
value: 1
325327
comment: final token in the file
326-
- name: MISSING
327-
comment: "a token that was expected but not found"
328-
- name: NOT_PROVIDED
329-
comment: "a token that was not present but it is okay"
328+
- name: BRACE_RIGHT
329+
comment: "}"
330+
- name: COMMA
331+
comment: ","
332+
- name: EMBEXPR_END
333+
comment: "}"
334+
- name: KEYWORD_DO
335+
comment: "do"
336+
- name: KEYWORD_ELSE
337+
comment: "else"
338+
- name: KEYWORD_ELSIF
339+
comment: "elsif"
340+
- name: KEYWORD_END
341+
comment: "end"
342+
- name: KEYWORD_ENSURE
343+
comment: "ensure"
344+
- name: KEYWORD_IN
345+
comment: "in"
346+
- name: KEYWORD_RESCUE
347+
comment: "rescue"
348+
- name: KEYWORD_THEN
349+
comment: "then"
350+
- name: KEYWORD_WHEN
351+
comment: "when"
352+
- name: NEWLINE
353+
comment: "a newline character outside of other tokens"
354+
- name: PARENTHESIS_RIGHT
355+
comment: ")"
356+
- name: SEMICOLON
357+
comment: ";"
358+
# Tokens from here on are not used for lookup, and can be in any order.
330359
- name: AMPERSAND
331360
comment: "&"
332361
- name: AMPERSAND_AMPERSAND
@@ -349,8 +378,6 @@ tokens:
349378
comment: "!~"
350379
- name: BRACE_LEFT
351380
comment: "{"
352-
- name: BRACE_RIGHT
353-
comment: "}"
354381
- name: BRACKET_LEFT
355382
comment: "["
356383
- name: BRACKET_LEFT_ARRAY
@@ -373,8 +400,6 @@ tokens:
373400
comment: ":"
374401
- name: COLON_COLON
375402
comment: "::"
376-
- name: COMMA
377-
comment: ","
378403
- name: COMMENT
379404
comment: "a comment"
380405
- name: CONSTANT
@@ -393,8 +418,6 @@ tokens:
393418
comment: "a line inside of embedded documentation"
394419
- name: EMBEXPR_BEGIN
395420
comment: "#{"
396-
- name: EMBEXPR_END
397-
comment: "}"
398421
- name: EMBVAR
399422
comment: "#"
400423
- name: EQUAL
@@ -461,20 +484,10 @@ tokens:
461484
comment: "def"
462485
- name: KEYWORD_DEFINED
463486
comment: "defined?"
464-
- name: KEYWORD_DO
465-
comment: "do"
466487
- name: KEYWORD_DO_LOOP
467488
comment: "do keyword for a predicate in a while, until, or for loop"
468-
- name: KEYWORD_ELSE
469-
comment: "else"
470-
- name: KEYWORD_ELSIF
471-
comment: "elsif"
472-
- name: KEYWORD_END
473-
comment: "end"
474489
- name: KEYWORD_END_UPCASE
475490
comment: "END"
476-
- name: KEYWORD_ENSURE
477-
comment: "ensure"
478491
- name: KEYWORD_FALSE
479492
comment: "false"
480493
- name: KEYWORD_FOR
@@ -483,8 +496,6 @@ tokens:
483496
comment: "if"
484497
- name: KEYWORD_IF_MODIFIER
485498
comment: "if in the modifier form"
486-
- name: KEYWORD_IN
487-
comment: "in"
488499
- name: KEYWORD_MODULE
489500
comment: "module"
490501
- name: KEYWORD_NEXT
@@ -497,8 +508,6 @@ tokens:
497508
comment: "or"
498509
- name: KEYWORD_REDO
499510
comment: "redo"
500-
- name: KEYWORD_RESCUE
501-
comment: "rescue"
502511
- name: KEYWORD_RESCUE_MODIFIER
503512
comment: "rescue in the modifier form"
504513
- name: KEYWORD_RETRY
@@ -509,8 +518,6 @@ tokens:
509518
comment: "self"
510519
- name: KEYWORD_SUPER
511520
comment: "super"
512-
- name: KEYWORD_THEN
513-
comment: "then"
514521
- name: KEYWORD_TRUE
515522
comment: "true"
516523
- name: KEYWORD_UNDEF
@@ -523,8 +530,6 @@ tokens:
523530
comment: "until"
524531
- name: KEYWORD_UNTIL_MODIFIER
525532
comment: "until in the modifier form"
526-
- name: KEYWORD_WHEN
527-
comment: "when"
528533
- name: KEYWORD_WHILE
529534
comment: "while"
530535
- name: KEYWORD_WHILE_MODIFIER
@@ -561,16 +566,12 @@ tokens:
561566
comment: "-="
562567
- name: MINUS_GREATER
563568
comment: "->"
564-
- name: NEWLINE
565-
comment: "a newline character outside of other tokens"
566569
- name: NUMBERED_REFERENCE
567570
comment: "a numbered reference to a capture group in the previous regular expression match"
568571
- name: PARENTHESIS_LEFT
569572
comment: "("
570573
- name: PARENTHESIS_LEFT_PARENTHESES
571574
comment: "( for a parentheses node"
572-
- name: PARENTHESIS_RIGHT
573-
comment: ")"
574575
- name: PERCENT
575576
comment: "%"
576577
- name: PERCENT_EQUAL
@@ -603,8 +604,6 @@ tokens:
603604
comment: "the beginning of a regular expression"
604605
- name: REGEXP_END
605606
comment: "the end of a regular expression"
606-
- name: SEMICOLON
607-
comment: ";"
608607
- name: SLASH
609608
comment: "/"
610609
- name: SLASH_EQUAL
@@ -649,6 +648,10 @@ tokens:
649648
comment: "a separator between words in a list"
650649
- name: __END__
651650
comment: "marker for the point in the file at which the parser should stop"
651+
- name: MISSING
652+
comment: "a token that was expected but not found"
653+
- name: NOT_PROVIDED
654+
comment: "a token that was not present but it is okay"
652655
flags:
653656
- name: ArgumentsNodeFlags
654657
values:

rakelib/lint.rake

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -5,17 +5,6 @@ task :lint do
55
require "yaml"
66
config = YAML.safe_load_file(File.expand_path("../config.yml", __dir__))
77

8-
tokens = config.fetch("tokens")[4..-1].map { |token| token.fetch("name") }
9-
if tokens.sort != tokens
10-
warn("Tokens are not sorted alphabetically")
11-
12-
tokens.sort.zip(tokens).each do |(sorted, unsorted)|
13-
warn("Expected #{sorted} got #{unsorted}") if sorted != unsorted
14-
end
15-
16-
exit(1)
17-
end
18-
198
nodes = config.fetch("nodes")
209
names = nodes.map { |node| node.fetch("name") }
2110
if names.sort != names

src/prism.c

Lines changed: 59 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -8582,85 +8582,66 @@ parser_lex_magic_comment(pm_parser_t *parser, bool semantic_token_seen) {
85828582
/* Context manipulations */
85838583
/******************************************************************************/
85848584

8585-
static bool
8586-
context_terminator(pm_context_t context, pm_token_t *token) {
8587-
switch (context) {
8588-
case PM_CONTEXT_MAIN:
8589-
case PM_CONTEXT_DEF_PARAMS:
8590-
case PM_CONTEXT_DEFINED:
8591-
case PM_CONTEXT_MULTI_TARGET:
8592-
case PM_CONTEXT_TERNARY:
8593-
case PM_CONTEXT_RESCUE_MODIFIER:
8594-
return token->type == PM_TOKEN_EOF;
8595-
case PM_CONTEXT_DEFAULT_PARAMS:
8596-
return token->type == PM_TOKEN_COMMA || token->type == PM_TOKEN_PARENTHESIS_RIGHT;
8597-
case PM_CONTEXT_PREEXE:
8598-
case PM_CONTEXT_POSTEXE:
8599-
return token->type == PM_TOKEN_BRACE_RIGHT;
8600-
case PM_CONTEXT_MODULE:
8601-
case PM_CONTEXT_CLASS:
8602-
case PM_CONTEXT_SCLASS:
8603-
case PM_CONTEXT_LAMBDA_DO_END:
8604-
case PM_CONTEXT_DEF:
8605-
case PM_CONTEXT_BLOCK_KEYWORDS:
8606-
return token->type == PM_TOKEN_KEYWORD_END || token->type == PM_TOKEN_KEYWORD_RESCUE || token->type == PM_TOKEN_KEYWORD_ENSURE;
8607-
case PM_CONTEXT_WHILE:
8608-
case PM_CONTEXT_UNTIL:
8609-
case PM_CONTEXT_ELSE:
8610-
case PM_CONTEXT_FOR:
8611-
case PM_CONTEXT_BEGIN_ENSURE:
8612-
case PM_CONTEXT_BLOCK_ENSURE:
8613-
case PM_CONTEXT_CLASS_ENSURE:
8614-
case PM_CONTEXT_DEF_ENSURE:
8615-
case PM_CONTEXT_LAMBDA_ENSURE:
8616-
case PM_CONTEXT_MODULE_ENSURE:
8617-
case PM_CONTEXT_SCLASS_ENSURE:
8618-
return token->type == PM_TOKEN_KEYWORD_END;
8619-
case PM_CONTEXT_LOOP_PREDICATE:
8620-
return token->type == PM_TOKEN_KEYWORD_DO || token->type == PM_TOKEN_KEYWORD_THEN;
8621-
case PM_CONTEXT_FOR_INDEX:
8622-
return token->type == PM_TOKEN_KEYWORD_IN;
8623-
case PM_CONTEXT_CASE_WHEN:
8624-
return token->type == PM_TOKEN_KEYWORD_WHEN || token->type == PM_TOKEN_KEYWORD_END || token->type == PM_TOKEN_KEYWORD_ELSE;
8625-
case PM_CONTEXT_CASE_IN:
8626-
return token->type == PM_TOKEN_KEYWORD_IN || token->type == PM_TOKEN_KEYWORD_END || token->type == PM_TOKEN_KEYWORD_ELSE;
8627-
case PM_CONTEXT_IF:
8628-
case PM_CONTEXT_ELSIF:
8629-
return token->type == PM_TOKEN_KEYWORD_ELSE || token->type == PM_TOKEN_KEYWORD_ELSIF || token->type == PM_TOKEN_KEYWORD_END;
8630-
case PM_CONTEXT_UNLESS:
8631-
return token->type == PM_TOKEN_KEYWORD_ELSE || token->type == PM_TOKEN_KEYWORD_END;
8632-
case PM_CONTEXT_EMBEXPR:
8633-
return token->type == PM_TOKEN_EMBEXPR_END;
8634-
case PM_CONTEXT_BLOCK_BRACES:
8635-
return token->type == PM_TOKEN_BRACE_RIGHT;
8636-
case PM_CONTEXT_PARENS:
8637-
return token->type == PM_TOKEN_PARENTHESIS_RIGHT;
8638-
case PM_CONTEXT_BEGIN:
8639-
case PM_CONTEXT_BEGIN_RESCUE:
8640-
case PM_CONTEXT_BLOCK_RESCUE:
8641-
case PM_CONTEXT_CLASS_RESCUE:
8642-
case PM_CONTEXT_DEF_RESCUE:
8643-
case PM_CONTEXT_LAMBDA_RESCUE:
8644-
case PM_CONTEXT_MODULE_RESCUE:
8645-
case PM_CONTEXT_SCLASS_RESCUE:
8646-
return token->type == PM_TOKEN_KEYWORD_ENSURE || token->type == PM_TOKEN_KEYWORD_RESCUE || token->type == PM_TOKEN_KEYWORD_ELSE || token->type == PM_TOKEN_KEYWORD_END;
8647-
case PM_CONTEXT_BEGIN_ELSE:
8648-
case PM_CONTEXT_BLOCK_ELSE:
8649-
case PM_CONTEXT_CLASS_ELSE:
8650-
case PM_CONTEXT_DEF_ELSE:
8651-
case PM_CONTEXT_LAMBDA_ELSE:
8652-
case PM_CONTEXT_MODULE_ELSE:
8653-
case PM_CONTEXT_SCLASS_ELSE:
8654-
return token->type == PM_TOKEN_KEYWORD_ENSURE || token->type == PM_TOKEN_KEYWORD_END;
8655-
case PM_CONTEXT_LAMBDA_BRACES:
8656-
return token->type == PM_TOKEN_BRACE_RIGHT;
8657-
case PM_CONTEXT_PREDICATE:
8658-
return token->type == PM_TOKEN_KEYWORD_THEN || token->type == PM_TOKEN_NEWLINE || token->type == PM_TOKEN_SEMICOLON;
8659-
case PM_CONTEXT_NONE:
8660-
return false;
8661-
}
8585+
static const uint32_t context_terminators[] = {
8586+
[PM_CONTEXT_NONE] = 0,
8587+
[PM_CONTEXT_BEGIN] = (1 << PM_TOKEN_KEYWORD_ENSURE) | (1 << PM_TOKEN_KEYWORD_RESCUE) | (1 << PM_TOKEN_KEYWORD_ELSE) | (1 << PM_TOKEN_KEYWORD_END),
8588+
[PM_CONTEXT_BEGIN_ENSURE] = (1 << PM_TOKEN_KEYWORD_END),
8589+
[PM_CONTEXT_BEGIN_ELSE] = (1 << PM_TOKEN_KEYWORD_ENSURE) | (1 << PM_TOKEN_KEYWORD_END),
8590+
[PM_CONTEXT_BEGIN_RESCUE] = (1 << PM_TOKEN_KEYWORD_ENSURE) | (1 << PM_TOKEN_KEYWORD_RESCUE) | (1 << PM_TOKEN_KEYWORD_ELSE) | (1 << PM_TOKEN_KEYWORD_END),
8591+
[PM_CONTEXT_BLOCK_BRACES] = (1 << PM_TOKEN_BRACE_RIGHT),
8592+
[PM_CONTEXT_BLOCK_KEYWORDS] = (1 << PM_TOKEN_KEYWORD_END) | (1 << PM_TOKEN_KEYWORD_RESCUE) | (1 << PM_TOKEN_KEYWORD_ENSURE),
8593+
[PM_CONTEXT_BLOCK_ENSURE] = (1 << PM_TOKEN_KEYWORD_END),
8594+
[PM_CONTEXT_BLOCK_ELSE] = (1 << PM_TOKEN_KEYWORD_ENSURE) | (1 << PM_TOKEN_KEYWORD_END),
8595+
[PM_CONTEXT_BLOCK_RESCUE] = (1 << PM_TOKEN_KEYWORD_ENSURE) | (1 << PM_TOKEN_KEYWORD_RESCUE) | (1 << PM_TOKEN_KEYWORD_ELSE) | (1 << PM_TOKEN_KEYWORD_END),
8596+
[PM_CONTEXT_CASE_WHEN] = (1 << PM_TOKEN_KEYWORD_WHEN) | (1 << PM_TOKEN_KEYWORD_END) | (1 << PM_TOKEN_KEYWORD_ELSE),
8597+
[PM_CONTEXT_CASE_IN] = (1 << PM_TOKEN_KEYWORD_IN) | (1 << PM_TOKEN_KEYWORD_END) | (1 << PM_TOKEN_KEYWORD_ELSE),
8598+
[PM_CONTEXT_CLASS] = (1 << PM_TOKEN_KEYWORD_END) | (1 << PM_TOKEN_KEYWORD_RESCUE) | (1 << PM_TOKEN_KEYWORD_ENSURE),
8599+
[PM_CONTEXT_CLASS_ENSURE] = (1 << PM_TOKEN_KEYWORD_END),
8600+
[PM_CONTEXT_CLASS_ELSE] = (1 << PM_TOKEN_KEYWORD_ENSURE) | (1 << PM_TOKEN_KEYWORD_END),
8601+
[PM_CONTEXT_CLASS_RESCUE] = (1 << PM_TOKEN_KEYWORD_ENSURE) | (1 << PM_TOKEN_KEYWORD_RESCUE) | (1 << PM_TOKEN_KEYWORD_ELSE) | (1 << PM_TOKEN_KEYWORD_END),
8602+
[PM_CONTEXT_DEF] = (1 << PM_TOKEN_KEYWORD_END) | (1 << PM_TOKEN_KEYWORD_RESCUE) | (1 << PM_TOKEN_KEYWORD_ENSURE),
8603+
[PM_CONTEXT_DEF_ENSURE] = (1 << PM_TOKEN_KEYWORD_END),
8604+
[PM_CONTEXT_DEF_ELSE] = (1 << PM_TOKEN_KEYWORD_ENSURE) | (1 << PM_TOKEN_KEYWORD_END),
8605+
[PM_CONTEXT_DEF_RESCUE] = (1 << PM_TOKEN_KEYWORD_ENSURE) | (1 << PM_TOKEN_KEYWORD_RESCUE) | (1 << PM_TOKEN_KEYWORD_ELSE) | (1 << PM_TOKEN_KEYWORD_END),
8606+
[PM_CONTEXT_DEF_PARAMS] = (1 << PM_TOKEN_EOF),
8607+
[PM_CONTEXT_DEFINED] = (1 << PM_TOKEN_EOF),
8608+
[PM_CONTEXT_DEFAULT_PARAMS] = (1 << PM_TOKEN_COMMA) | (1 << PM_TOKEN_PARENTHESIS_RIGHT),
8609+
[PM_CONTEXT_ELSE] = (1 << PM_TOKEN_KEYWORD_END),
8610+
[PM_CONTEXT_ELSIF] = (1 << PM_TOKEN_KEYWORD_ELSE) | (1 << PM_TOKEN_KEYWORD_ELSIF) | (1 << PM_TOKEN_KEYWORD_END),
8611+
[PM_CONTEXT_EMBEXPR] = (1 << PM_TOKEN_EMBEXPR_END),
8612+
[PM_CONTEXT_FOR] = (1 << PM_TOKEN_KEYWORD_END),
8613+
[PM_CONTEXT_FOR_INDEX] = (1 << PM_TOKEN_KEYWORD_IN),
8614+
[PM_CONTEXT_IF] = (1 << PM_TOKEN_KEYWORD_ELSE) | (1 << PM_TOKEN_KEYWORD_ELSIF) | (1 << PM_TOKEN_KEYWORD_END),
8615+
[PM_CONTEXT_LAMBDA_BRACES] = (1 << PM_TOKEN_BRACE_RIGHT),
8616+
[PM_CONTEXT_LAMBDA_DO_END] = (1 << PM_TOKEN_KEYWORD_END) | (1 << PM_TOKEN_KEYWORD_RESCUE) | (1 << PM_TOKEN_KEYWORD_ENSURE),
8617+
[PM_CONTEXT_LAMBDA_ENSURE] = (1 << PM_TOKEN_KEYWORD_END),
8618+
[PM_CONTEXT_LAMBDA_ELSE] = (1 << PM_TOKEN_KEYWORD_ENSURE) | (1 << PM_TOKEN_KEYWORD_END),
8619+
[PM_CONTEXT_LAMBDA_RESCUE] = (1 << PM_TOKEN_KEYWORD_ENSURE) | (1 << PM_TOKEN_KEYWORD_RESCUE) | (1 << PM_TOKEN_KEYWORD_ELSE) | (1 << PM_TOKEN_KEYWORD_END),
8620+
[PM_CONTEXT_LOOP_PREDICATE] = (1 << PM_TOKEN_KEYWORD_DO) | (1 << PM_TOKEN_KEYWORD_THEN),
8621+
[PM_CONTEXT_MAIN] = (1 << PM_TOKEN_EOF),
8622+
[PM_CONTEXT_MODULE] = (1 << PM_TOKEN_KEYWORD_END) | (1 << PM_TOKEN_KEYWORD_RESCUE) | (1 << PM_TOKEN_KEYWORD_ENSURE),
8623+
[PM_CONTEXT_MODULE_ENSURE] = (1 << PM_TOKEN_KEYWORD_END),
8624+
[PM_CONTEXT_MODULE_ELSE] = (1 << PM_TOKEN_KEYWORD_ENSURE) | (1 << PM_TOKEN_KEYWORD_END),
8625+
[PM_CONTEXT_MODULE_RESCUE] = (1 << PM_TOKEN_KEYWORD_ENSURE) | (1 << PM_TOKEN_KEYWORD_RESCUE) | (1 << PM_TOKEN_KEYWORD_ELSE) | (1 << PM_TOKEN_KEYWORD_END),
8626+
[PM_CONTEXT_MULTI_TARGET] = (1 << PM_TOKEN_EOF),
8627+
[PM_CONTEXT_PARENS] = (1 << PM_TOKEN_PARENTHESIS_RIGHT),
8628+
[PM_CONTEXT_POSTEXE] = (1 << PM_TOKEN_BRACE_RIGHT),
8629+
[PM_CONTEXT_PREDICATE] = (1 << PM_TOKEN_KEYWORD_THEN) | (1 << PM_TOKEN_NEWLINE) | (1 << PM_TOKEN_SEMICOLON),
8630+
[PM_CONTEXT_PREEXE] = (1 << PM_TOKEN_BRACE_RIGHT),
8631+
[PM_CONTEXT_RESCUE_MODIFIER] = (1 << PM_TOKEN_EOF),
8632+
[PM_CONTEXT_SCLASS] = (1 << PM_TOKEN_KEYWORD_END) | (1 << PM_TOKEN_KEYWORD_RESCUE) | (1 << PM_TOKEN_KEYWORD_ENSURE),
8633+
[PM_CONTEXT_SCLASS_ENSURE] = (1 << PM_TOKEN_KEYWORD_END),
8634+
[PM_CONTEXT_SCLASS_ELSE] = (1 << PM_TOKEN_KEYWORD_ENSURE) | (1 << PM_TOKEN_KEYWORD_END),
8635+
[PM_CONTEXT_SCLASS_RESCUE] = (1 << PM_TOKEN_KEYWORD_ENSURE) | (1 << PM_TOKEN_KEYWORD_RESCUE) | (1 << PM_TOKEN_KEYWORD_ELSE) | (1 << PM_TOKEN_KEYWORD_END),
8636+
[PM_CONTEXT_TERNARY] = (1 << PM_TOKEN_EOF),
8637+
[PM_CONTEXT_UNLESS] = (1 << PM_TOKEN_KEYWORD_ELSE) | (1 << PM_TOKEN_KEYWORD_END),
8638+
[PM_CONTEXT_UNTIL] = (1 << PM_TOKEN_KEYWORD_END),
8639+
[PM_CONTEXT_WHILE] = (1 << PM_TOKEN_KEYWORD_END),
8640+
};
86628641

8663-
return false;
8642+
static inline bool
8643+
context_terminator(pm_context_t context, pm_token_t *token) {
8644+
return token->type < 32 && (context_terminators[context] & (1 << token->type));
86648645
}
86658646

86668647
/**

0 commit comments

Comments
 (0)