From a480d7b17636f9a449d597f2ac6b9068eeb261d5 Mon Sep 17 00:00:00 2001 From: rahil627 <1132053+rahil627@users.noreply.github.com> Date: Sun, 18 Jan 2026 03:51:00 -0800 Subject: [PATCH 1/5] a/ ruby lexer WARNING: used python.jai as base then ai gen'd a few differences --- src/langs/ruby.jai | 714 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 714 insertions(+) create mode 100644 src/langs/ruby.jai diff --git a/src/langs/ruby.jai b/src/langs/ruby.jai new file mode 100644 index 00000000..18546fb0 --- /dev/null +++ b/src/langs/ruby.jai @@ -0,0 +1,714 @@ +tokenize_ruby :: (using buffer: *Buffer, start_offset := -1, count := -1) -> [] Buffer_Region { + tokenizer := get_tokenizer(buffer, start_offset, count); + + last_token: Token; + + while true { + token := get_next_token(*tokenizer); + if token.type == .eof break; + + using tokenizer; + + // Maybe highlight a function + if token.type == .punctuation && token.punctuation == .l_paren && last_token.type == .identifier { + memset(tokens.data + last_token.start, xx Token_Type.function, last_token.len); + } + + last_token = token; + + highlight_token(buffer, token); + } + + return .[]; +} + +#scope_file + +get_next_token :: (using tokenizer: *Tokenizer) -> Token { + eat_white_space(tokenizer); + + token: Token; + token.start = cast(s32) (t - buf.data); + token.type = .eof; + if t >= max_t return token; + + start_t = t; + + // Assume ASCII, unless we're in the middle of a string. + // UTF-8 characters elsewhere are a syntax error. + char := t.*; + + if ascii_is_alpha(char) || char == #char "_" || char == #char "@" || char == #char "$" { + parse_identifier(tokenizer, *token); + } else if ascii_is_digit(char) { + parse_number(tokenizer, *token); + } else if char == { + case #char ":"; parse_colon (tokenizer, *token); + case #char "="; parse_equal (tokenizer, *token); + case #char "-"; parse_minus (tokenizer, *token); + case #char "+"; parse_plus (tokenizer, *token); + case #char "*"; parse_asterisk (tokenizer, *token); + case #char "<"; parse_less_than (tokenizer, *token); + case #char ">"; parse_greater_than (tokenizer, *token); + case #char "!"; parse_bang (tokenizer, *token); + case #char "/"; parse_slash (tokenizer, *token); + case #char "\""; parse_double_quote (tokenizer, *token); + case #char "'"; parse_single_quote (tokenizer, *token); + case #char "\t"; parse_tab (tokenizer, *token); + case #char "#"; parse_comment (tokenizer, *token); + case #char "&"; parse_ampersand (tokenizer, *token); + case #char "|"; parse_pipe (tokenizer, *token); + case #char "%"; parse_percent (tokenizer, *token); + case #char "^"; parse_caret (tokenizer, *token); + case #char "\\"; parse_back_slash (tokenizer, *token); + + case #char ";"; token.type = .punctuation; token.punctuation = .semicolon; t += 1; + case #char ","; token.type = .punctuation; token.punctuation = .comma; t += 1; + case #char "."; token.type = .punctuation; token.punctuation = .period; t += 1; + case #char "{"; token.type = .punctuation; token.punctuation = .l_brace; t += 1; + case #char "}"; token.type = .punctuation; token.punctuation = .r_brace; t += 1; + case #char "("; token.type = .punctuation; token.punctuation = .l_paren; t += 1; + case #char ")"; token.type = .punctuation; token.punctuation = .r_paren; t += 1; + case #char "["; token.type = .punctuation; token.punctuation = .l_bracket; t += 1; + case #char "]"; token.type = .punctuation; token.punctuation = .r_bracket; t += 1; + case #char "\\"; token.type = .punctuation; token.punctuation = .back_slash; t += 1; + + case #char "~"; token.type = .operation; token.operation = .tilde; t += 1; + case #char "`"; token.type = .operation; token.operation = .backtick; t += 1; + + case; token.type = .invalid; t += 1; + } + + if t >= max_t then t = max_t; + token.len = cast(s32) (t - start_t); + return token; +} + +// NOTE: replaces read_identifier_string in common.jai +// WARNING: may miss out on updates to the original common function +read_ruby_identifier_string :: (using tokenizer: *Tokenizer) -> string { + identifier: string; + identifier.data = t; + + // prefixes (@, @@, $) + if t < max_t && (t.* == #char "@" || t.* == #char "$") { + t += 1; + if t < max_t && t.* == #char "@" then t += 1; + } + + // standard alphanumeric + underscore + while t < max_t && (ascii_is_alnum(t.*) || t.* == #char "_") { + t += 1; + } + + // suffixes (!, ?) + if t < max_t && (t.* == #char "!" || t.* == #char "?") { + t += 1; + } + + if t >= max_t then t = max_t; + identifier.count = t - identifier.data; + + return identifier; +} + +parse_identifier :: (using tokenizer: *Tokenizer, token: *Token) { + token.type = .identifier; + + identifier_str := read_ruby_identifier_string(tokenizer); // read_identifier_string + + // globals + if t.* == #char "$" { + t += 1; + if t < max_t { + // 1. Numeric Globals (Regex captures: $1, $2, etc) + if ascii_is_digit(t.*) { + t += 1; + } + // 2. Standard Globals ($LOAD_PATH, $global_var) + else if ascii_is_alpha(t.*) || t.* == #char "_" { + read_identifier_string(tokenizer); + } + // 3. Special Globals ($!, $?, $/, $~, $*) + else { + t += 1; + } + } + token.type = .string; // Globals are often colored like Symbols + return; + } + + // 1. Check for Labels (key:) + // Do this early so that "if:" is a symbol, not a keyword. + if t < max_t && t.* == #char ":" { + // Peek to ensure it's not the first part of a '::' + if (t + 1) < max_t && (t + 1).* != #char ":" { + t += 1; // Consume the ':' + token.type = .string; // Highlight as a Ruby symbol + return; + } + } + + // 2. Check for Ruby Constants (Starts with A-Z) + // In Ruby, capitalization is a semantic rule for Types/Classes. + first_char := identifier_str[0]; + if ascii_is_upper(first_char) { + token.type = .type; + // Note: Don't return yet! Some constants might be in the keyword map + // (though rare in Ruby, like BEGIN/END). + } + + // Maybe it's a keyword + if identifier_str.count <= MAX_KEYWORD_LENGTH { + ok, kw_token := table_find(*KEYWORD_MAP, identifier_str); + if ok { token.type = kw_token.type; token.keyword = kw_token.keyword; return; } + } + +} + +parse_number :: (using tokenizer: *Tokenizer, token: *Token) { + token.type = .number; + + t += 1; + if t >= max_t return; + + if ascii_is_digit(t.*) || t.* == #char "_" { + // Decimal + t += 1; + seen_decimal_point := false; + while t < max_t && (ascii_is_digit(t.*) || t.* == #char "_" || t.* == #char ".") { + if t.* == #char "." { + if seen_decimal_point break; + seen_decimal_point = true; + } + t += 1; + } + } else if t.* == #char "x" || t.* == #char "h" { + // Hex + t += 1; + while t < max_t && (is_hex(t.*) || t.* == #char "_") t += 1; + } else if t.* == #char "b" { + // Binary + t += 1; + while t < max_t && (t.* == #char "1" || t.* == #char "0" || t.* == #char "_") t += 1; + } +} + +parse_colon :: (using tokenizer: *Tokenizer, token: *Token) { + t += 1; // Consume the first ':' + + // If we hit the end of the file right after the colon + if t >= max_t { + token.type = .operation; + token.operation = .colon; + return; + } + + char := t.*; + + + + // CASE 1: Double Colon (::) + if char == #char ":" { + t += 1; // Consume the second ':' + token.type = .operation; + token.operation = .double_colon; + return; + } + + + // CASE 2: Standard Symbol (:symbol) + if ascii_is_alpha(char) || char == #char "_" { + // Use the original reader to grab the alphanumeric part + read_identifier_string(tokenizer); + + // Ruby symbols can end in ! or ? (e.g. :valid?) + if t < max_t && (t.* == #char "!" || t.* == #char "?") { + t += 1; + } + + token.type = .string; // Color it as a symbol/string + return; + } + + // CASE 3: Quoted Symbol (:"symbol") + if char == #char "\"" || char == #char "'" { + // This re-uses the existing string literal logic + parse_string_literal(tokenizer, token, char); + token.type = .string; + return; + } + + // Default: Just a plain colon (used in ternary a ? b : c) + token.type = .operation; + token.operation = .colon; +} + +parse_equals :: (using tokenizer: *Tokenizer, token: *Token) { + // handle =begin/=end + // Check if we are at the start of a line (t == line_start) + // and if the next characters are "begin" + if is_at_start_of_line(tokenizer) && peek_string(tokenizer, "begin") { + token.type = .comment; + + // Skip "=begin" + t += 6; + + // Loop until we find "=end" at the start of a line + while t < max_t { + if is_at_start_of_line(tokenizer) && peek_string(tokenizer, "=end") { + t += 4; // consume "=end" + break; + } + t += 1; + } + return; + } + + t += 1; + if t >= max_t return; + + if t.* == { + case #char "="; + token.operation = .equal_equal; + t += 1; + + // Ruby specific: Check for === + if t < max_t && t.* == #char "=" { + token.operation = .triple_equal; + t += 1; + } + + case #char ">"; + token.operation = .hash_rocket; // For { :a => 1 } + t += 1; + + case #char "~"; + token.operation = .equal_tilde; // The =~ operator + t += 1; + + } +} + +parse_minus :: (using tokenizer: *Tokenizer, token: *Token) { + token.type = .operation; + token.operation = .minus; + + t += 1; + if t >= max_t return; + + if t.* == { + case #char "="; + token.operation = .minus_equal; + t += 1; + case #char ">"; + token.operation = .arrow; + t += 1; + case #char "-"; + t += 1; + if t < max_t && t.* == #char "-" { + token.operation = .triple_dash; + t += 1; + } else { + token.operation = .unknown; // -- is not a valid token + } + case; + if ascii_is_digit(t.*) parse_number(tokenizer, token); + } +} + +parse_plus :: (using tokenizer: *Tokenizer, token: *Token) { + token.type = .operation; + token.operation = .plus; + + t += 1; + if t >= max_t return; + + if t.* == { + case #char "="; + token.operation = .plus_equal; + t += 1; + } +} + +parse_asterisk :: (using tokenizer: *Tokenizer, token: *Token) { + token.type = .operation; + token.operation = .asterisk; + + t += 1; + if t >= max_t return; + + if t.* == { + case #char "="; + token.operation = .asterisk_equal; + t += 1; + } +} + +parse_bang :: (using tokenizer: *Tokenizer, token: *Token) { + token.type = .operation; + token.operation = .bang; + + t += 1; + if t >= max_t return; + + if t.* == { + case #char "="; + token.operation = .bang_equal; + t += 1; + + case #char "~"; + token.operation = .bang_tilde; // The !~ operator + t += 1; + } +} + +parse_percent :: (using tokenizer: *Tokenizer, token: *Token) { + + t += 1; // Consume '%' + + if t < max_t && ascii_is_alpha(t.*) { + // It's a literal like %w, %q, %i, %r + t += 1; // Consume the letter + + if t < max_t { + open_delim := t.*; + close_delim: uint8; + + // Set the matching pair + if open_delim == #char "(" close_delim = #char ")"; + else if open_delim == #char "[" close_delim = #char "]"; + else if open_delim == #char "{" close_delim = #char "}"; + else if open_delim == #char "<" close_delim = #char ">"; + else close_delim = open_delim; + + t += 1; // Consume opening delimiter + while t < max_t { + if t.* == #char "\\" { t += 2; continue; } + if t.* == close_delim { t += 1; break; } + t += 1; + } + token.type = .string; + token.count = t - start_pos; + return; + } + } + + // Default: Modulo (%) or Modulo Assignment (%=) + token.type = .operation; + token.operation = .percent; + + t += 1; + if t >= max_t return; + + if t.* == { + case #char "="; + token.operation = .percent_equal; + t += 1; + } +} + +parse_caret :: (using tokenizer: *Tokenizer, token: *Token) { + token.type = .operation; + token.operation = .caret; + + t += 1; + if t >= max_t return; + + if t.* == { + case #char "="; + token.operation = .caret_equal; + t += 1; + } +} + +parse_ampersand :: (using tokenizer: *Tokenizer, token: *Token) { + token.type = .operation; + token.operation = .ampersand; + + t += 1; + if t >= max_t return; + + if t.* == { + case #char "="; + token.operation = .ampersand_equal; + t += 1; + case #char "&"; + token.operation = .double_ampersand; + t += 1; + } +} + +parse_pipe :: (using tokenizer: *Tokenizer, token: *Token) { + token.type = .operation; + token.operation = .pipe; + + t += 1; + if t >= max_t return; + + if t.* == { + case #char "="; + token.operation = .pipe_equal; + t += 1; + case #char "|"; + token.operation = .double_pipe; + t += 1; + } +} + +parse_slash :: (using tokenizer: *Tokenizer, token: *Token) { + token.type = .operation; + token.operation = .slash; + + t += 1; // Consume the first '/' + if t >= max_t return; + + // CASE 1: Division Assignment (/=) + if t.* == #char "=" { + token.operation = .slash_equal; + t += 1; + return; + } + + // CASE 2: Regular Expression (/regex/) + // Logic: If the slash is NOT followed by a space, it's likely a Regex. + // In Ruby, division / usually has spaces around it: x / y. + if t.* != #char " " && t.* != #char "\t" && t.* != #char "\n" { + token.type = .string; // Color regex as a string literal + + while t < max_t { + if t.* == #char "\\" { + t += 2; // Skip escaped characters like \/ + continue; + } + + if t.* == #char "/" { + t += 1; // Consume closing '/' + + // Consume optional regex modifiers: /abc/ix + while t < max_t && (t.* == #char "i" || t.* == #char "m" || + t.* == #char "x" || t.* == #char "o") { + t += 1; + } + break; + } + + if t.* == #char "\n" break; // Safety: stop at end of line + t += 1; + } + } +} + +parse_less_than :: (using tokenizer: *Tokenizer, token: *Token) { + token.type = .operation; + token.operation = .less_than; + + t += 1; + if t >= max_t return; + + if t.* == { + case #char "="; + // Check if it's <= or <=> + if (t + 1) < max_t && (t + 1).* == #char ">" { + token.operation = .spaceship; // <=> + t += 2; + } else { + token.operation = .less_than_equal; // <= + t += 1; + } + case #char "<"; + token.operation = .double_less_than; // << (Also used for appending to arrays) + t += 1; + } +} + +parse_greater_than :: (using tokenizer: *Tokenizer, token: *Token) { + token.type = .operation; + token.operation = .greater_than; + + t += 1; + if t >= max_t return; + + if t.* == { + case #char "="; + token.operation = .greater_than_equal; + t += 1; + } +} + +parse_tab :: (using tokenizer: *Tokenizer, token: *Token) { + token.type = .comment; + t += 1; + while t < max_t && t.* == #char "\t" t += 1; +} + +parse_comment :: (using tokenizer: *Tokenizer, token: *Token) { + token.type = .comment; + t += 1; + while t < max_t && t.* != #char "\n" t += 1; +} + +parse_double_quote :: (using tokenizer: *Tokenizer, token: *Token) { + parse_string_literal(tokenizer, token, #char "\""); +} + +parse_single_quote :: (using tokenizer: *Tokenizer, token: *Token) { + parse_string_literal(tokenizer, token, #char "'"); +} + +parse_string_literal :: (using tokenizer: *Tokenizer, token: *Token, quote_type: uint8) { + token.type = .string; + t += 1; // Consume the opening quote (' or ") + + while t < max_t { + // Handle escaped characters (like \" or \\) + if t.* == #char "\\" { + t += 2; + continue; + } + + // If we hit the matching closing quote, we are done + if t.* == quote_type { + t += 1; + break; + } + + // Ruby specific: Peek for interpolation start #{ + // We stay in the string for now to avoid breaking the lexer + if quote_type == #char "\"" && t.* == #char "#" { + if (t + 1) < max_t && (t + 1).* == #char "{" { + // For a basic lexer, we just continue. + // Advanced lexers would break here to color the code inside. + } + } + + t += 1; + } +} + +parse_back_slash :: (using tokenizer: *Tokenizer, token: *Token) { + token.type = .punctuation; + token.punctuation = .back_slash; + t += 1; // Consume the '\' + + // Ruby logic: If a backslash is followed immediately by a newline, + // it's a line continuation. We consume the newline so the lexer + // treats the next line as part of the current expression. + if t < max_t && t.* == #char "\n" { + t += 1; + // Optional: If you want to skip highlighting line continuations entirely: + // token.type = .none; + } +} + +Token :: struct { + using #as base: Base_Token; + + // Additional info to distinguish between keywords/punctuation + union { + keyword: Keyword; + punctuation: Punctuation; + operation: Operation; + } +} + +PUNCTUATION :: string.[ + "semicolon", "l_paren", "r_paren", "l_brace", "r_brace", "l_bracket", "r_bracket", "period", "comma", "back_slash" +]; + +OPERATIONS :: string.[ + "arrow", "bang", "backtick", "pipe", "double_pipe", "pipe_equal", "equal", "equal_equal", "bang_equal", + "percent", "percent_equal", "less_than", "double_less_than", "less_than_equal", "greater_than", "greater_than_equal", + "minus", "minus_equal", "asterisk", "asterisk_equal", "colon", "double_colon", "slash", + "plus", "plus_equal", "slash_equal", "ampersand", "double_ampersand", "ampersand_equal", "tilde", + "caret", "caret_equal", + // NOTE: added: hash_rocket (=>), spaceship (<=>), equal_tilde (=~), bang_tilde (!~), and triple_equal (===) + "hash_rocket", "spaceship", "equal_tilde", "bang_tilde", "triple_equal" + // TODO: unimpl: "double_asterisk" (exponent), "asterisk_equal" (exponent equal?), "double_dot", "triple_dot" +]; + + +KEYWORDS :: string.[ + "alias", "and", "begin", "break", "case", "class", "def", "defined?", + "do", "else", "elsif", "end", "ensure", "false", "for", "if", "in", + "module", "next", "nil", "not", "or", "redo", "rescue", "retry", + "return", "self", "super", "then", "true", "undef", "unless", "until", + "when", "while", "yield", "it", + "__FILE__", "__LINE__", "__ENCODING__", "BEGIN", "END" +]; + +TYPE_KEYWORDS :: string.[ + // alt: could just leave this blank and treat them as classes.. + "Array", "Hash", "String", "Integer", "Float", "Numeric", + "Symbol", "Range", "Regexp", "TrueClass", "FalseClass", "NilClass", + "Object", "Module", "Class", "Struct", "Enumerable" +]; + +BUILTIN_FUNCTIONS :: string.[ + // Kernal methods as global functions + "puts", "print", "p", "gets", "chomp", "require", "require_relative", + "include", "extend", "attr_accessor", "attr_reader", "attr_writer", + "raise", "fail", "catch", "throw", "loop", "proc", "lambda" +]; + +BUILTIN_EXCEPTIONS :: string.[ + "Exception", "StandardError", "RuntimeError", "ArgumentError", + "NoMethodError", "NameError", "TypeError", "ZeroDivisionError", + "SystemExit", "StopIteration", "IOError", "LoadError", "SecurityError" +]; + +VALUE_KEYWORDS :: string.[ + "true", "false", "nil", "self" +]; + +#insert -> string { + b: String_Builder; + init_string_builder(*b); + + define_enum :: (b: *String_Builder, enum_name: string, prefix: string, value_lists: [][] string) { + print_to_builder(b, "% :: enum u16 {\n", enum_name); + for values : value_lists { + for v : values print_to_builder(b, " %0%;\n", prefix, v); + } + print_to_builder(b, "}\n"); + } + + define_enum(*b, "Punctuation", "", .[PUNCTUATION]); + define_enum(*b, "Operation", "", .[OPERATIONS]); + define_enum(*b, "Keyword", "kw_", .[KEYWORDS, TYPE_KEYWORDS, VALUE_KEYWORDS, BUILTIN_FUNCTIONS, BUILTIN_EXCEPTIONS]); + + return builder_to_string(*b); +} + +Keyword_Token :: struct { + type: Token_Type; + keyword: Keyword; +} + +KEYWORD_MAP :: #run -> Table(string, Keyword_Token) { + table: Table(string, Keyword_Token); + size := 10 * (KEYWORDS.count + TYPE_KEYWORDS.count + VALUE_KEYWORDS.count); + init(*table, size); + + #insert -> string { + b: String_Builder; + for KEYWORDS append(*b, sprint("table_add(*table, \"%\", Keyword_Token.{ type = .keyword, keyword = .kw_% });\n", it, it)); + for TYPE_KEYWORDS append(*b, sprint("table_add(*table, \"%\", Keyword_Token.{ type = .type, keyword = .kw_% });\n", it, it)); + for VALUE_KEYWORDS append(*b, sprint("table_add(*table, \"%\", Keyword_Token.{ type = .value, keyword = .kw_% });\n", it, it)); + for BUILTIN_FUNCTIONS append(*b, sprint("table_add(*table, \"%\", Keyword_Token.{ type = .builtin_function, keyword = .kw_% });\n", it, it)); + for BUILTIN_EXCEPTIONS append(*b, sprint("table_add(*table, \"%\", Keyword_Token.{ type = .builtin_exception, keyword = .kw_% });\n", it, it)); + return builder_to_string(*b); + } + + return table; +} + +MAX_KEYWORD_LENGTH :: #run -> s32 { + result: s64; + for KEYWORDS { if it.count > result then result = it.count; } + for TYPE_KEYWORDS { if it.count > result then result = it.count; } + for VALUE_KEYWORDS { if it.count > result then result = it.count; } + for BUILTIN_FUNCTIONS { if it.count > result then result = it.count; } + for BUILTIN_EXCEPTIONS { if it.count > result then result = it.count; } + return xx result; +} From a67f85f6191502430907435dffe1c7ee9f46de75 Mon Sep 17 00:00:00 2001 From: rahil627 <1132053+rahil627@users.noreply.github.com> Date: Mon, 22 Jun 2026 01:51:05 -0700 Subject: [PATCH 2/5] u/ ruby lexer: bugfixes --- src/langs/ruby.jai | 45 +++++++++++++++++++++++++++++++-------------- 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/src/langs/ruby.jai b/src/langs/ruby.jai index 18546fb0..d8c924c8 100644 --- a/src/langs/ruby.jai +++ b/src/langs/ruby.jai @@ -117,6 +117,8 @@ parse_identifier :: (using tokenizer: *Tokenizer, token: *Token) { identifier_str := read_ruby_identifier_string(tokenizer); // read_identifier_string + // if identifier_str.count > 0 && identifier_str[0] == #char "$" { + // globals if t.* == #char "$" { t += 1; @@ -151,11 +153,12 @@ parse_identifier :: (using tokenizer: *Tokenizer, token: *Token) { // 2. Check for Ruby Constants (Starts with A-Z) // In Ruby, capitalization is a semantic rule for Types/Classes. - first_char := identifier_str[0]; - if ascii_is_upper(first_char) { - token.type = .type; - // Note: Don't return yet! Some constants might be in the keyword map - // (though rare in Ruby, like BEGIN/END). + if identifier_str.count > 0 { + if ascii_is_upper(identifier_str[0]) { + token.type = .type; + // Note: Don't return yet! Some constants might be in the keyword map + // (though rare in Ruby, like BEGIN/END). + } } // Maybe it's a keyword @@ -244,13 +247,25 @@ parse_colon :: (using tokenizer: *Tokenizer, token: *Token) { token.operation = .colon; } -parse_equals :: (using tokenizer: *Tokenizer, token: *Token) { +is_at_start_of_line :: (using tokenizer: *Tokenizer) -> bool #inline { + if t == buf.data return true; + return (t - 1).* == #char "\n"; +} + +peek_string :: (using tokenizer: *Tokenizer, s: string) -> bool { // TODO: #inline? + if t + s.count > max_t return false; + for 0..s.count-1 { + if (t + it).* != s[it] return false; + } + return true; +} + +parse_equal :: (using tokenizer: *Tokenizer, token: *Token) { // handle =begin/=end // Check if we are at the start of a line (t == line_start) // and if the next characters are "begin" if is_at_start_of_line(tokenizer) && peek_string(tokenizer, "begin") { token.type = .comment; - // Skip "=begin" t += 6; @@ -389,7 +404,8 @@ parse_percent :: (using tokenizer: *Tokenizer, token: *Token) { t += 1; } token.type = .string; - token.count = t - start_pos; + token.len = cast(s32) (t - start_t); // TODO: check ai's work + // token.count = t - start_pos; return; } } @@ -563,7 +579,8 @@ parse_string_literal :: (using tokenizer: *Tokenizer, token: *Token, quote_type: while t < max_t { // Handle escaped characters (like \" or \\) if t.* == #char "\\" { - t += 2; + t += 1; + if t < max_t t += 1; continue; } @@ -629,10 +646,10 @@ OPERATIONS :: string.[ KEYWORDS :: string.[ - "alias", "and", "begin", "break", "case", "class", "def", "defined?", - "do", "else", "elsif", "end", "ensure", "false", "for", "if", "in", - "module", "next", "nil", "not", "or", "redo", "rescue", "retry", - "return", "self", "super", "then", "true", "undef", "unless", "until", + "alias", "and", "begin", "break", "case", "class", "def", "defined?", + "do", "else", "elsif", "end", "ensure", "for", "if", "in", + "module", "next", "not", "or", "redo", "rescue", "retry", + "return", "self", "super", "then", "undef", "unless", "until", "when", "while", "yield", "it", "__FILE__", "__LINE__", "__ENCODING__", "BEGIN", "END" ]; @@ -688,7 +705,7 @@ Keyword_Token :: struct { KEYWORD_MAP :: #run -> Table(string, Keyword_Token) { table: Table(string, Keyword_Token); size := 10 * (KEYWORDS.count + TYPE_KEYWORDS.count + VALUE_KEYWORDS.count); - init(*table, size); + table_resize(*table, size); #insert -> string { b: String_Builder; From f11d619b92923b28cc13ffd69e60d57016d5605e Mon Sep 17 00:00:00 2001 From: rahil627 <1132053+rahil627@users.noreply.github.com> Date: Mon, 22 Jun 2026 01:51:15 -0700 Subject: [PATCH 3/5] f/ ruby lexer: fmt --- src/langs/ruby.jai | 71 +++++++++++++++++++++++----------------------- 1 file changed, 36 insertions(+), 35 deletions(-) diff --git a/src/langs/ruby.jai b/src/langs/ruby.jai index d8c924c8..1164d049 100644 --- a/src/langs/ruby.jai +++ b/src/langs/ruby.jai @@ -60,7 +60,7 @@ get_next_token :: (using tokenizer: *Tokenizer) -> Token { case #char "|"; parse_pipe (tokenizer, *token); case #char "%"; parse_percent (tokenizer, *token); case #char "^"; parse_caret (tokenizer, *token); - case #char "\\"; parse_back_slash (tokenizer, *token); + case #char "\\"; parse_back_slash (tokenizer, *token); case #char ";"; token.type = .punctuation; token.punctuation = .semicolon; t += 1; case #char ","; token.type = .punctuation; token.punctuation = .comma; t += 1; @@ -125,29 +125,29 @@ parse_identifier :: (using tokenizer: *Tokenizer, token: *Token) { if t < max_t { // 1. Numeric Globals (Regex captures: $1, $2, etc) if ascii_is_digit(t.*) { - t += 1; - } + t += 1; + } // 2. Standard Globals ($LOAD_PATH, $global_var) else if ascii_is_alpha(t.*) || t.* == #char "_" { - read_identifier_string(tokenizer); - } + read_identifier_string(tokenizer); + } // 3. Special Globals ($!, $?, $/, $~, $*) else { - t += 1; + t += 1; } } token.type = .string; // Globals are often colored like Symbols return; } - // 1. Check for Labels (key:) + // 1. Check for Labels (key:) // Do this early so that "if:" is a symbol, not a keyword. if t < max_t && t.* == #char ":" { // Peek to ensure it's not the first part of a '::' if (t + 1) < max_t && (t + 1).* != #char ":" { t += 1; // Consume the ':' token.type = .string; // Highlight as a Ruby symbol - return; + return; } } @@ -166,7 +166,7 @@ parse_identifier :: (using tokenizer: *Tokenizer, token: *Token) { ok, kw_token := table_find(*KEYWORD_MAP, identifier_str); if ok { token.type = kw_token.type; token.keyword = kw_token.keyword; return; } } - + } parse_number :: (using tokenizer: *Tokenizer, token: *Token) { @@ -215,7 +215,7 @@ parse_colon :: (using tokenizer: *Tokenizer, token: *Token) { if char == #char ":" { t += 1; // Consume the second ':' token.type = .operation; - token.operation = .double_colon; + token.operation = .double_colon; return; } @@ -223,8 +223,8 @@ parse_colon :: (using tokenizer: *Tokenizer, token: *Token) { // CASE 2: Standard Symbol (:symbol) if ascii_is_alpha(char) || char == #char "_" { // Use the original reader to grab the alphanumeric part - read_identifier_string(tokenizer); - + read_identifier_string(tokenizer); + // Ruby symbols can end in ! or ? (e.g. :valid?) if t < max_t && (t.* == #char "!" || t.* == #char "?") { t += 1; @@ -238,7 +238,7 @@ parse_colon :: (using tokenizer: *Tokenizer, token: *Token) { if char == #char "\"" || char == #char "'" { // This re-uses the existing string literal logic parse_string_literal(tokenizer, token, char); - token.type = .string; + token.type = .string; return; } @@ -266,9 +266,10 @@ parse_equal :: (using tokenizer: *Tokenizer, token: *Token) { // and if the next characters are "begin" if is_at_start_of_line(tokenizer) && peek_string(tokenizer, "begin") { token.type = .comment; + // Skip "=begin" - t += 6; - + t += 6; + // Loop until we find "=end" at the start of a line while t < max_t { if is_at_start_of_line(tokenizer) && peek_string(tokenizer, "=end") { @@ -287,7 +288,7 @@ parse_equal :: (using tokenizer: *Tokenizer, token: *Token) { case #char "="; token.operation = .equal_equal; t += 1; - + // Ruby specific: Check for === if t < max_t && t.* == #char "=" { token.operation = .triple_equal; @@ -301,7 +302,7 @@ parse_equal :: (using tokenizer: *Tokenizer, token: *Token) { case #char "~"; token.operation = .equal_tilde; // The =~ operator t += 1; - + } } @@ -385,11 +386,11 @@ parse_percent :: (using tokenizer: *Tokenizer, token: *Token) { if t < max_t && ascii_is_alpha(t.*) { // It's a literal like %w, %q, %i, %r t += 1; // Consume the letter - + if t < max_t { open_delim := t.*; close_delim: uint8; - + // Set the matching pair if open_delim == #char "(" close_delim = #char ")"; else if open_delim == #char "[" close_delim = #char "]"; @@ -474,7 +475,7 @@ parse_pipe :: (using tokenizer: *Tokenizer, token: *Token) { parse_slash :: (using tokenizer: *Tokenizer, token: *Token) { token.type = .operation; - token.operation = .slash; + token.operation = .slash; t += 1; // Consume the first '/' if t >= max_t return; @@ -491,24 +492,24 @@ parse_slash :: (using tokenizer: *Tokenizer, token: *Token) { // In Ruby, division / usually has spaces around it: x / y. if t.* != #char " " && t.* != #char "\t" && t.* != #char "\n" { token.type = .string; // Color regex as a string literal - + while t < max_t { if t.* == #char "\\" { t += 2; // Skip escaped characters like \/ continue; } - + if t.* == #char "/" { t += 1; // Consume closing '/' - + // Consume optional regex modifiers: /abc/ix - while t < max_t && (t.* == #char "i" || t.* == #char "m" || + while t < max_t && (t.* == #char "i" || t.* == #char "m" || t.* == #char "x" || t.* == #char "o") { t += 1; } break; } - + if t.* == #char "\n" break; // Safety: stop at end of line t += 1; } @@ -586,7 +587,7 @@ parse_string_literal :: (using tokenizer: *Tokenizer, token: *Token, quote_type: // If we hit the matching closing quote, we are done if t.* == quote_type { - t += 1; + t += 1; break; } @@ -594,7 +595,7 @@ parse_string_literal :: (using tokenizer: *Tokenizer, token: *Token, quote_type: // We stay in the string for now to avoid breaking the lexer if quote_type == #char "\"" && t.* == #char "#" { if (t + 1) < max_t && (t + 1).* == #char "{" { - // For a basic lexer, we just continue. + // For a basic lexer, we just continue. // Advanced lexers would break here to color the code inside. } } @@ -608,13 +609,13 @@ parse_back_slash :: (using tokenizer: *Tokenizer, token: *Token) { token.punctuation = .back_slash; t += 1; // Consume the '\' - // Ruby logic: If a backslash is followed immediately by a newline, - // it's a line continuation. We consume the newline so the lexer + // Ruby logic: If a backslash is followed immediately by a newline, + // it's a line continuation. We consume the newline so the lexer // treats the next line as part of the current expression. if t < max_t && t.* == #char "\n" { t += 1; // Optional: If you want to skip highlighting line continuations entirely: - // token.type = .none; + // token.type = .none; } } @@ -656,26 +657,26 @@ KEYWORDS :: string.[ TYPE_KEYWORDS :: string.[ // alt: could just leave this blank and treat them as classes.. - "Array", "Hash", "String", "Integer", "Float", "Numeric", + "Array", "Hash", "String", "Integer", "Float", "Numeric", "Symbol", "Range", "Regexp", "TrueClass", "FalseClass", "NilClass", "Object", "Module", "Class", "Struct", "Enumerable" ]; BUILTIN_FUNCTIONS :: string.[ // Kernal methods as global functions - "puts", "print", "p", "gets", "chomp", "require", "require_relative", + "puts", "print", "p", "gets", "chomp", "require", "require_relative", "include", "extend", "attr_accessor", "attr_reader", "attr_writer", "raise", "fail", "catch", "throw", "loop", "proc", "lambda" ]; BUILTIN_EXCEPTIONS :: string.[ - "Exception", "StandardError", "RuntimeError", "ArgumentError", - "NoMethodError", "NameError", "TypeError", "ZeroDivisionError", + "Exception", "StandardError", "RuntimeError", "ArgumentError", + "NoMethodError", "NameError", "TypeError", "ZeroDivisionError", "SystemExit", "StopIteration", "IOError", "LoadError", "SecurityError" ]; VALUE_KEYWORDS :: string.[ - "true", "false", "nil", "self" + "true", "false", "nil", "self" ]; #insert -> string { From 6575504adb3da8a287b8a7bec169e8e01e3092b2 Mon Sep 17 00:00:00 2001 From: rahil627 <1132053+rahil627@users.noreply.github.com> Date: Mon, 22 Jun 2026 14:39:15 -0700 Subject: [PATCH 4/5] u/ operators: .. ... ** --- src/langs/ruby.jai | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/src/langs/ruby.jai b/src/langs/ruby.jai index 1164d049..2b3d1add 100644 --- a/src/langs/ruby.jai +++ b/src/langs/ruby.jai @@ -61,10 +61,10 @@ get_next_token :: (using tokenizer: *Tokenizer) -> Token { case #char "%"; parse_percent (tokenizer, *token); case #char "^"; parse_caret (tokenizer, *token); case #char "\\"; parse_back_slash (tokenizer, *token); + case #char "."; parse_dot (tokenizer, *token); case #char ";"; token.type = .punctuation; token.punctuation = .semicolon; t += 1; case #char ","; token.type = .punctuation; token.punctuation = .comma; t += 1; - case #char "."; token.type = .punctuation; token.punctuation = .period; t += 1; case #char "{"; token.type = .punctuation; token.punctuation = .l_brace; t += 1; case #char "}"; token.type = .punctuation; token.punctuation = .r_brace; t += 1; case #char "("; token.type = .punctuation; token.punctuation = .l_paren; t += 1; @@ -197,6 +197,23 @@ parse_number :: (using tokenizer: *Tokenizer, token: *Token) { } } +parse_dot :: (using tokenizer: *Tokenizer, token: *Token) { + token.type = .operation; + token.operation = .period; + t += 1; + + if t >= max_t { + return; // skip single dot + else if t < max_t && t.* == #char "." { + t += 1; + token.operation = .double_dot; + if t < max_t && t.* == #char "." { + t += 1; + token.operation = .triple_dot; + } + } +} + parse_colon :: (using tokenizer: *Tokenizer, token: *Token) { t += 1; // Consume the first ':' @@ -355,6 +372,9 @@ parse_asterisk :: (using tokenizer: *Tokenizer, token: *Token) { if t >= max_t return; if t.* == { + case #char "*"; + token.operation = .double_asterisk; + t += 1; case #char "="; token.operation = .asterisk_equal; t += 1; @@ -640,9 +660,11 @@ OPERATIONS :: string.[ "minus", "minus_equal", "asterisk", "asterisk_equal", "colon", "double_colon", "slash", "plus", "plus_equal", "slash_equal", "ampersand", "double_ampersand", "ampersand_equal", "tilde", "caret", "caret_equal", - // NOTE: added: hash_rocket (=>), spaceship (<=>), equal_tilde (=~), bang_tilde (!~), and triple_equal (===) - "hash_rocket", "spaceship", "equal_tilde", "bang_tilde", "triple_equal" - // TODO: unimpl: "double_asterisk" (exponent), "asterisk_equal" (exponent equal?), "double_dot", "triple_dot" + + "hash_rocket", "spaceship", // =>, <=> + "equal_tilde", "bang_tilde", "triple_equal", + "double_dot", "triple_dot", + "double_asterisk" ]; From 5fb57d677f65dea88eeb7afe7e710bc644196a9c Mon Sep 17 00:00:00 2001 From: rahil627 <1132053+rahil627@users.noreply.github.com> Date: Mon, 22 Jun 2026 14:41:13 -0700 Subject: [PATCH 5/5] u/ tiny note: special type classes or just classes? --- src/langs/ruby.jai | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/langs/ruby.jai b/src/langs/ruby.jai index 2b3d1add..40693353 100644 --- a/src/langs/ruby.jai +++ b/src/langs/ruby.jai @@ -678,7 +678,7 @@ KEYWORDS :: string.[ ]; TYPE_KEYWORDS :: string.[ - // alt: could just leave this blank and treat them as classes.. + // TODO: DECIDE: could just skip types and just treat them as classes.. "Array", "Hash", "String", "Integer", "Float", "Numeric", "Symbol", "Range", "Regexp", "TrueClass", "FalseClass", "NilClass", "Object", "Module", "Class", "Struct", "Enumerable"