Skip to content

Commit c3f738b

Browse files
braindigitalisbraindigitalis
committed
tokenizer optimisations
1 parent 22f80c8 commit c3f738b

5 files changed

Lines changed: 37 additions & 101 deletions

File tree

‎include/basic/tokenizer.h‎

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -295,17 +295,6 @@ void tokenizer_fnum(struct basic_ctx* ctx, enum token_t token, double* f);
295295
*/
296296
const char* tokenizer_variable_name(struct basic_ctx* ctx, size_t* count);
297297

298-
/**
299-
* @brief Get a string constant as the next token
300-
* (do not advance the pointer)
301-
*
302-
* @param dest destination string buffer
303-
* @param len length of destination buffer
304-
* @param ctx context
305-
* @return true if succesfully found a string constant
306-
*/
307-
bool tokenizer_string(char *dest, int len, struct basic_ctx* ctx);
308-
309298
/**
310299
* @brief Returns true if the program is finished
311300
* (does not advance the pointer)

‎include/basic/unified_expression.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,8 @@
1010
* ### Tokeniser interaction
1111
* - **Parentheses**: content inside `(` .. `)` is parsed as a relation-aware
1212
* expression, so forms such as `(in<128)` are valid in IF/WHILE.
13-
* - **String literals**: obtained via `tokenizer_string(...)` and then
14-
* `accept(STRING)`; the tokeniser produces the literal and we consume it.
13+
* - **String literals**: obtained via `gc_from_tokenizer_string(...)`.
14+
* This advances the tokenizer, we then call `tokenizer_next()` to update the token.
1515
* - **Variables and built-ins**:
1616
* - Names ending with `$` - STRING.
1717
* - Names ending with `#` - REAL.

‎include/io.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -150,7 +150,7 @@ static inline void interrupts_off()
150150
* Loops indefinitely, executing the `hlt` instruction on each iteration.
151151
* Used to stop execution in unrecoverable conditions.
152152
*/
153-
static inline void wait_forever()
153+
_Noreturn static inline void wait_forever()
154154
{
155155
for (;;) __asm__ volatile("hlt");
156156
}

‎include/string.h‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ static inline char __attribute__((always_inline)) toupper(char low) {
7171
* @param x Character to check
7272
* @return int Non-zero if uppercase, 0 otherwise
7373
*/
74-
static inline int __attribute__((always_inline)) isupper(const char x)
74+
static inline bool __attribute__((always_inline)) isupper(const char x)
7575
{
7676
return (unsigned)(x - 'A') <= ('Z' - 'A');
7777
}
@@ -105,7 +105,7 @@ static inline bool __attribute__((always_inline)) isalpha(const char x)
105105
* @param x Character to check
106106
* @return unsigned char Non-zero if digit, 0 otherwise
107107
*/
108-
static inline unsigned char __attribute__((always_inline)) isdigit(const char x)
108+
static inline bool __attribute__((always_inline)) isdigit(const char x)
109109
{
110110
return (unsigned)(x - '0') <= 9;
111111
}
@@ -116,9 +116,9 @@ static inline unsigned char __attribute__((always_inline)) isdigit(const char x)
116116
* @param x Character to check
117117
* @return unsigned char Non-zero if hex digit, 0 otherwise
118118
*/
119-
static inline unsigned char __attribute__((always_inline)) isxdigit(const char x)
119+
static inline bool __attribute__((always_inline)) isxdigit(const char x)
120120
{
121-
return (x >= '0' && x <= '9') || (x >= 'A' && x <= 'F');
121+
return isdigit(x) || (unsigned)(((unsigned const char)x | 0x20) - 'a') <= ('f' - 'a');
122122
}
123123

124124
/**
@@ -127,7 +127,7 @@ static inline unsigned char __attribute__((always_inline)) isxdigit(const char x
127127
* @param x Character to check
128128
* @return int Non-zero if alphanumeric, 0 otherwise
129129
*/
130-
static inline int __attribute__((always_inline)) isalnum(const char x)
130+
static inline bool __attribute__((always_inline)) isalnum(const char x)
131131
{
132132
return isdigit(x) || isalpha(x);
133133
}

‎src/basic/tokenizer.c‎

Lines changed: 29 additions & 82 deletions
Original file line numberDiff line numberDiff line change
@@ -147,7 +147,7 @@ static const char *intern_variable_name(const char *name, size_t len)
147147

148148
existing = hashmap_get(
149149
interned_variable_names,
150-
&(struct interned_name) { .name = name }
150+
&(struct interned_name) { .name = name, .name_length = len }
151151
);
152152

153153
if (existing) {
@@ -170,7 +170,7 @@ static const char *intern_variable_name(const char *name, size_t len)
170170

171171
existing = hashmap_get(
172172
interned_variable_names,
173-
&(struct interned_name) { .name = name }
173+
&(struct interned_name) { .name = name, .name_length = len }
174174
);
175175

176176
return existing ? existing->name : copy;
@@ -267,46 +267,33 @@ void build_keyword_prefix_offsets(void)
267267
keyword_prefix_ready = true;
268268
}
269269

270-
static int singlechar(struct basic_ctx* ctx)
271-
{
272-
switch (*ctx->ptr) {
273-
case '\n':
274-
return NEWLINE;
275-
case ',':
276-
return COMMA;
277-
case ';':
278-
return SEMICOLON;
279-
case '+':
280-
return PLUS;
281-
case '-':
282-
return MINUS;
283-
case '*':
284-
return ASTERISK;
285-
case '/':
286-
return SLASH;
287-
case '%':
288-
return MOD;
289-
case '(':
290-
return OPENBRACKET;
291-
case ')':
292-
return CLOSEBRACKET;
293-
case '<':
294-
return LESSTHAN;
295-
case '>':
296-
return GREATERTHAN;
297-
case '=':
298-
return EQUALS;
299-
case '~':
300-
return TILDE;
301-
case ' ':
302-
case '\t':
303-
return SPACE;
304-
}
305-
return 0;
270+
static const enum token_t singlechar_tokens[256] = {
271+
['\n'] = NEWLINE,
272+
[','] = COMMA,
273+
[';'] = SEMICOLON,
274+
['+'] = PLUS,
275+
['-'] = MINUS,
276+
['*'] = ASTERISK,
277+
['/'] = SLASH,
278+
['%'] = MOD,
279+
['('] = OPENBRACKET,
280+
[')'] = CLOSEBRACKET,
281+
['<'] = LESSTHAN,
282+
['>'] = GREATERTHAN,
283+
['='] = EQUALS,
284+
['~'] = TILDE,
285+
[' '] = SPACE,
286+
['\t'] = SPACE,
287+
};
288+
289+
static inline __attribute__((always_inline)) enum token_t singlechar(const char *p) {
290+
return singlechar_tokens[(unsigned char)*p];
306291
}
307292

308293
int get_next_token(struct basic_ctx* ctx)
309294
{
295+
enum token_t tok;
296+
310297
if (*ctx->ptr == 0) {
311298
return ENDOFINPUT;
312299
}
@@ -324,10 +311,6 @@ int get_next_token(struct basic_ctx* ctx)
324311
return NO_TOKEN;
325312
}
326313
}
327-
if (!isxdigit(ctx->ptr[i])) {
328-
tokenizer_error_print(ctx, "Malformed hexadecimal number");
329-
return NO_TOKEN;
330-
}
331314
}
332315
} else {
333316
/* Scan forwards up to MAX_NUMLEN characters */
@@ -342,17 +325,13 @@ int get_next_token(struct basic_ctx* ctx)
342325
return NO_TOKEN;
343326
}
344327
}
345-
if (!isdigit(ctx->ptr[i]) && ctx->ptr[i] != '.') {
346-
tokenizer_error_print(ctx, "Malformed number");
347-
return NO_TOKEN;
348-
}
349328
}
350329
}
351330
tokenizer_error_print(ctx, "Number too long");
352331
return NO_TOKEN;
353-
} else if (singlechar(ctx)) {
332+
} else if ((tok = singlechar(ctx->ptr)) != NO_TOKEN) {
354333
ctx->nextptr = ctx->ptr + 1;
355-
return singlechar(ctx);
334+
return tok;
356335
} else if (*ctx->ptr == '"') {
357336
ctx->nextptr = ctx->ptr;
358337
int strl = 0;
@@ -376,7 +355,7 @@ int get_next_token(struct basic_ctx* ctx)
376355
int start = keyword_prefix_offsets[key];
377356
int end = keyword_prefix_offsets[key + 1];
378357
for (int kt = start; kt < end; ++kt) {
379-
int tok = keywords[kt];
358+
tok = keywords[kt];
380359
size_t len = token_name_lengths[tok];
381360
/* First two characters already matched by prefix16 bucket.
382361
* Compare from byte 2 onwards; <=2 length is already a full match.
@@ -479,35 +458,6 @@ void tokenizer_fnum(struct basic_ctx* ctx, enum token_t token, double* f)
479458
atof(ctx->ptr, f);
480459
}
481460

482-
bool tokenizer_string(char *dest, int len, struct basic_ctx* ctx)
483-
{
484-
char *string_end;
485-
int string_len;
486-
487-
if (tokenizer_token(ctx) != STRING) {
488-
return true;
489-
}
490-
string_end = strchr(ctx->ptr + 1, '"');
491-
if (string_end == NULL) {
492-
tokenizer_error_print(ctx, "Unterminated \"");
493-
*dest = 0;
494-
return false;
495-
}
496-
string_len = string_end - ctx->ptr - 1;
497-
if (len < string_len) {
498-
string_len = len;
499-
}
500-
if (ctx->ptr == ctx->program_ptr) {
501-
tokenizer_error_print(ctx, "Unterminated \"");
502-
*dest = 0;
503-
return false;
504-
}
505-
memcpy(dest, ctx->ptr + 1, string_len);
506-
dest[string_len] = 0;
507-
return true;
508-
}
509-
510-
511461
void tokenizer_error_printf(struct basic_ctx* ctx, const char* fmt, ...)
512462
{
513463
char error[MAX_STRINGLEN];
@@ -518,7 +468,6 @@ void tokenizer_error_printf(struct basic_ctx* ctx, const char* fmt, ...)
518468
tokenizer_error_print(ctx, error);
519469
}
520470

521-
522471
void tokenizer_error_print(struct basic_ctx* ctx, const char* error)
523472
{
524473
dprintf("tokenizer_error_print: %s\n", error);
@@ -612,8 +561,6 @@ bool tokenizer_finished(struct basic_ctx* ctx)
612561
const char* tokenizer_variable_name(struct basic_ctx* ctx, size_t* count)
613562
{
614563
char varname[MAX_VARNAME];
615-
const char *interned;
616-
617564
*count = 0;
618565

619566
while (*count < MAX_VARNAME && *ctx->ptr != 0) {
@@ -648,7 +595,7 @@ const char* tokenizer_variable_name(struct basic_ctx* ctx, size_t* count)
648595

649596
varname[*count] = 0;
650597

651-
interned = intern_variable_name(varname, *count);
598+
const char* interned = intern_variable_name(varname, *count);
652599
if (!interned) {
653600
tokenizer_error_printf(ctx, "Out of memory interning variable name '%s'", varname);
654601
*count = 0;

0 commit comments

Comments
 (0)