Skip to content

Commit 20e626a

Browse files
kddnewtonkddnewton
committed
Lex simple integer values as we are lexing
1 parent fbcd3fc commit 20e626a

2 files changed

Lines changed: 90 additions & 26 deletions

File tree

‎include/prism/parser.h‎

Lines changed: 19 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -793,12 +793,26 @@ struct pm_parser {
793793
pm_line_offset_list_t line_offsets;
794794

795795
/**
796-
* We want to add a flag to integer nodes that indicates their base. We only
797-
* want to parse these once, but we don't have space on the token itself to
798-
* communicate this information. So we store it here and pass it through
799-
* when we find tokens that we need it for.
796+
* State communicated from the lexer to the parser for integer tokens.
800797
*/
801-
pm_node_flags_t integer_base;
798+
struct {
799+
/**
800+
* A flag indicating the base of the integer (binary, octal, decimal,
801+
* hexadecimal). Set during lexing and read during node creation.
802+
*/
803+
pm_node_flags_t base;
804+
805+
/**
806+
* When lexing a decimal integer that fits in a uint32_t, we compute
807+
* the value during lexing to avoid re-scanning the digits during
808+
* parsing. If lexed is true, this holds the result and
809+
* pm_integer_parse can be skipped.
810+
*/
811+
uint32_t value;
812+
813+
/** Whether value holds a valid pre-computed integer. */
814+
bool lexed;
815+
} integer;
802816

803817
/**
804818
* This string is used to pass information from the lexer to the parser. It

‎src/prism.c‎

Lines changed: 71 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -4710,17 +4710,24 @@ pm_integer_node_create(pm_parser_t *parser, pm_node_flags_t base, const pm_token
47104710
((pm_integer_t) { 0 })
47114711
);
47124712

4713-
pm_integer_base_t integer_base = PM_INTEGER_BASE_DECIMAL;
4714-
switch (base) {
4715-
case PM_INTEGER_BASE_FLAGS_BINARY: integer_base = PM_INTEGER_BASE_BINARY; break;
4716-
case PM_INTEGER_BASE_FLAGS_OCTAL: integer_base = PM_INTEGER_BASE_OCTAL; break;
4717-
case PM_INTEGER_BASE_FLAGS_DECIMAL: break;
4718-
case PM_INTEGER_BASE_FLAGS_HEXADECIMAL: integer_base = PM_INTEGER_BASE_HEXADECIMAL; break;
4719-
default: assert(false && "unreachable"); break;
4713+
if (parser->integer.lexed) {
4714+
// The value was already computed during lexing.
4715+
node->value.value = parser->integer.value;
4716+
parser->integer.lexed = false;
4717+
} else {
4718+
pm_integer_base_t integer_base = PM_INTEGER_BASE_DECIMAL;
4719+
switch (base) {
4720+
case PM_INTEGER_BASE_FLAGS_BINARY: integer_base = PM_INTEGER_BASE_BINARY; break;
4721+
case PM_INTEGER_BASE_FLAGS_OCTAL: integer_base = PM_INTEGER_BASE_OCTAL; break;
4722+
case PM_INTEGER_BASE_FLAGS_DECIMAL: break;
4723+
case PM_INTEGER_BASE_FLAGS_HEXADECIMAL: integer_base = PM_INTEGER_BASE_HEXADECIMAL; break;
4724+
default: assert(false && "unreachable"); break;
4725+
}
4726+
4727+
pm_integer_parse(&node->value, integer_base, token->start, token->end);
4728+
pm_integer_arena_move(parser->arena, &node->value);
47204729
}
47214730

4722-
pm_integer_parse(&node->value, integer_base, token->start, token->end);
4723-
pm_integer_arena_move(parser->arena, &node->value);
47244731
return node;
47254732
}
47264733

@@ -8112,7 +8119,7 @@ lex_numeric_prefix(pm_parser_t *parser, bool* seen_e) {
81128119
pm_parser_err_current(parser, PM_ERR_INVALID_NUMBER_BINARY);
81138120
}
81148121

8115-
parser->integer_base = PM_INTEGER_BASE_FLAGS_BINARY;
8122+
parser->integer.base = PM_INTEGER_BASE_FLAGS_BINARY;
81168123
break;
81178124

81188125
// 0o1111 is an octal number
@@ -8126,7 +8133,7 @@ lex_numeric_prefix(pm_parser_t *parser, bool* seen_e) {
81268133
pm_parser_err_current(parser, PM_ERR_INVALID_NUMBER_OCTAL);
81278134
}
81288135

8129-
parser->integer_base = PM_INTEGER_BASE_FLAGS_OCTAL;
8136+
parser->integer.base = PM_INTEGER_BASE_FLAGS_OCTAL;
81308137
break;
81318138

81328139
// 01111 is an octal number
@@ -8140,7 +8147,7 @@ lex_numeric_prefix(pm_parser_t *parser, bool* seen_e) {
81408147
case '6':
81418148
case '7':
81428149
parser->current.end += pm_strspn_octal_number_validate(parser, parser->current.end);
8143-
parser->integer_base = PM_INTEGER_BASE_FLAGS_OCTAL;
8150+
parser->integer.base = PM_INTEGER_BASE_FLAGS_OCTAL;
81448151
break;
81458152

81468153
// 0x1111 is a hexadecimal number
@@ -8154,7 +8161,7 @@ lex_numeric_prefix(pm_parser_t *parser, bool* seen_e) {
81548161
pm_parser_err_current(parser, PM_ERR_INVALID_NUMBER_HEXADECIMAL);
81558162
}
81568163

8157-
parser->integer_base = PM_INTEGER_BASE_FLAGS_HEXADECIMAL;
8164+
parser->integer.base = PM_INTEGER_BASE_FLAGS_HEXADECIMAL;
81588165
break;
81598166

81608167
// 0.xxx is a float
@@ -8172,11 +8179,53 @@ lex_numeric_prefix(pm_parser_t *parser, bool* seen_e) {
81728179
}
81738180
} else {
81748181
// If it didn't start with a 0, then we'll lex as far as we can into a
8175-
// decimal number.
8176-
parser->current.end += pm_strspn_decimal_number_validate(parser, parser->current.end);
8182+
// decimal number. We compute the integer value inline to avoid
8183+
// re-scanning the digits later in pm_integer_parse.
8184+
{
8185+
const uint8_t *cursor = parser->current.end;
8186+
const uint8_t *end = parser->end;
8187+
uint64_t value = (uint64_t) (cursor[-1] - '0');
8188+
8189+
bool has_underscore = false;
8190+
bool prev_underscore = false;
8191+
const uint8_t *invalid = NULL;
8192+
8193+
while (cursor < end) {
8194+
uint8_t c = *cursor;
8195+
if (c >= '0' && c <= '9') {
8196+
if (value <= UINT32_MAX) value = value * 10 + (uint64_t) (c - '0');
8197+
prev_underscore = false;
8198+
cursor++;
8199+
} else if (c == '_') {
8200+
has_underscore = true;
8201+
if (prev_underscore && invalid == NULL) invalid = cursor;
8202+
prev_underscore = true;
8203+
cursor++;
8204+
} else {
8205+
break;
8206+
}
8207+
}
8208+
8209+
if (has_underscore) {
8210+
if (prev_underscore && invalid == NULL) invalid = cursor - 1;
8211+
pm_strspn_number_validate(parser, parser->current.end, (size_t) (cursor - parser->current.end), invalid);
8212+
}
8213+
8214+
if (value <= UINT32_MAX) {
8215+
parser->integer.value = (uint32_t) value;
8216+
parser->integer.lexed = true;
8217+
}
8218+
8219+
parser->current.end = cursor;
8220+
}
81778221

81788222
// Afterward, we'll lex as far as we can into an optional float suffix.
81798223
type = lex_optional_float_suffix(parser, seen_e);
8224+
8225+
// If it turned out to be a float, the cached integer value is invalid.
8226+
if (type != PM_TOKEN_INTEGER) {
8227+
parser->integer.lexed = false;
8228+
}
81808229
}
81818230

81828231
// At this point we have a completed number, but we want to provide the user
@@ -8195,7 +8244,8 @@ lex_numeric_prefix(pm_parser_t *parser, bool* seen_e) {
81958244
static pm_token_type_t
81968245
lex_numeric(pm_parser_t *parser) {
81978246
pm_token_type_t type = PM_TOKEN_INTEGER;
8198-
parser->integer_base = PM_INTEGER_BASE_FLAGS_DECIMAL;
8247+
parser->integer.base = PM_INTEGER_BASE_FLAGS_DECIMAL;
8248+
parser->integer.lexed = false;
81998249

82008250
if (parser->current.end < parser->end) {
82018251
bool seen_e = false;
@@ -18302,22 +18352,22 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, u
1830218352
return node;
1830318353
}
1830418354
case PM_TOKEN_INTEGER: {
18305-
pm_node_flags_t base = parser->integer_base;
18355+
pm_node_flags_t base = parser->integer.base;
1830618356
parser_lex(parser);
1830718357
return UP(pm_integer_node_create(parser, base, &parser->previous));
1830818358
}
1830918359
case PM_TOKEN_INTEGER_IMAGINARY: {
18310-
pm_node_flags_t base = parser->integer_base;
18360+
pm_node_flags_t base = parser->integer.base;
1831118361
parser_lex(parser);
1831218362
return UP(pm_integer_node_imaginary_create(parser, base, &parser->previous));
1831318363
}
1831418364
case PM_TOKEN_INTEGER_RATIONAL: {
18315-
pm_node_flags_t base = parser->integer_base;
18365+
pm_node_flags_t base = parser->integer.base;
1831618366
parser_lex(parser);
1831718367
return UP(pm_integer_node_rational_create(parser, base, &parser->previous));
1831818368
}
1831918369
case PM_TOKEN_INTEGER_RATIONAL_IMAGINARY: {
18320-
pm_node_flags_t base = parser->integer_base;
18370+
pm_node_flags_t base = parser->integer.base;
1832118371
parser_lex(parser);
1832218372
return UP(pm_integer_node_rational_imaginary_create(parser, base, &parser->previous));
1832318373
}
@@ -22154,7 +22204,7 @@ pm_parser_init(pm_arena_t *arena, pm_parser_t *parser, const uint8_t *source, si
2215422204
.filepath = { 0 },
2215522205
.constant_pool = { 0 },
2215622206
.line_offsets = { 0 },
22157-
.integer_base = 0,
22207+
.integer = { 0 },
2215822208
.current_string = PM_STRING_EMPTY,
2215922209
.start_line = 1,
2216022210
.explicit_encoding = NULL,

0 commit comments

Comments
 (0)