Skip to content

Commit 391e4e6

Browse files
hyperpolymathclaude
andcommitted
feat(lexer): underscore-prefix idents _key/_unused (Refs gitbot-fleet#148)
Extends `lower_ident_start` from a single `lower` to `lower | ('_', alphanum)` — a lowercase letter, OR underscore followed by at least one alphanum. This is the conventional Rust-idiom spelling for "intentionally unused" bindings and parameters. Bare `_` (the wildcard pattern) is preserved as the distinct UNDERSCORE token: sedlex applies maximal munch, and the underscore branch of `lower_ident_start` requires at least one trailing alphanum, so `_` standalone falls through to the existing UNDERSCORE rule. Updates `test/test_lexer.ml::test_identifiers` to assert the new tokenisation — `_test` is now a single `LOWER_IDENT "_test"`, and the test additionally asserts that bare `_` still emits `UNDERSCORE`. 327/327 tests pass. Required by sustainabot hand-port (gitbot-fleet#148) — `GitHubApp.affine` declares unused cache-stub bindings as `let _key = ...; let _value = ...;`. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
1 parent 4b83c1a commit 391e4e6

2 files changed

Lines changed: 18 additions & 4 deletions

File tree

lib/lexer.ml

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -98,7 +98,15 @@ let upper = [%sedlex.regexp? 'A'..'Z']
9898
let alpha = [%sedlex.regexp? lower | upper]
9999
let alphanum = [%sedlex.regexp? alpha | digit | '_']
100100

101-
let lower_ident = [%sedlex.regexp? lower, Star alphanum]
101+
(* `lower_ident_start` accepts the conventional Rust-idiom underscore-prefix
102+
form `_unused` for parameters and bindings that are intentionally unused.
103+
The two-alternative form `lower | ('_', alphanum)` (a lowercase letter, OR
104+
underscore-then-at-least-one-alphanum) ensures bare `_` (the wildcard
105+
pattern) still lexes as the distinct UNDERSCORE token under sedlex's
106+
maximal-munch rule. Added 2026-05-26 (Refs gitbot-fleet#148 sustainabot
107+
hand-port: `_key`, `_value` cache stubs in GitHubApp.affine). *)
108+
let lower_ident_start = [%sedlex.regexp? lower | ('_', alphanum)]
109+
let lower_ident = [%sedlex.regexp? lower_ident_start, Star alphanum]
102110
let upper_ident = [%sedlex.regexp? upper, Star alphanum]
103111

104112
let int_lit = [%sedlex.regexp? Opt '-', Plus digit]

test/test_lexer.ml

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,11 +23,17 @@ let test_keywords () =
2323
tokens
2424

2525
let test_identifiers () =
26-
let tokens = lex_all "foo Bar _test test123" in
26+
(* Underscore-prefix policy (Refs 2026-05-26 gitbot-fleet#148 sustainabot
27+
hand-port): `_test` (and `_anything`) is the conventional Rust-idiom
28+
unused-binding spelling; the lexer treats it as a single LOWER_IDENT.
29+
Bare `_` is still tokenised as the distinct UNDERSCORE (wildcard) token
30+
— sedlex applies maximal munch and the underscore branch of
31+
`lower_ident_start` requires at least one trailing alphanum. *)
32+
let tokens = lex_all "foo Bar _test test123 _" in
2733
Alcotest.(check (list token_testable)) "identifiers"
2834
[Token.LOWER_IDENT "foo"; Token.UPPER_IDENT "Bar";
29-
Token.UNDERSCORE; Token.LOWER_IDENT "test";
30-
Token.LOWER_IDENT "test123"; Token.EOF]
35+
Token.LOWER_IDENT "_test"; Token.LOWER_IDENT "test123";
36+
Token.UNDERSCORE; Token.EOF]
3137
tokens
3238

3339
let test_literals () =

0 commit comments

Comments
 (0)