Skip to content

Commit b645ec8

Browse files
nystromnystrom
committed
Add formatted and non-formatted literal tokens
1 parent 992405f commit b645ec8

17 files changed

Lines changed: 12962 additions & 11575 deletions

File tree

‎meta/src/meta/cli.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -179,14 +179,16 @@ def run(args) -> int:
179179
return 1
180180
grammar = Grammar(
181181
start=start,
182+
token_aliases=grammar_config.token_aliases,
182183
ignored_completeness=grammar_config.ignored_completeness,
183184
function_defs=grammar_config.function_defs,
184185
)
185186
for _, rules in grammar_config.rules.items():
186187
for rule in rules:
187188
grammar.add_rule(rule)
188189

189-
# Add tokens with patterns from terminal declarations in grammar file
190+
# Add tokens with patterns from terminal declarations in grammar file.
191+
# Skip aliases — they share the base token's pattern and shouldn't be in the lexer.
190192
from .grammar import Token
191193

192194
for terminal_name, terminal_def in grammar_config.terminal_patterns.items():

‎meta/src/meta/codegen_templates.py‎

Lines changed: 45 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,17 @@ class BuiltinTemplate:
9393
"format_decimal": BuiltinTemplate("self.format_decimal({0})"),
9494
"format_int128": BuiltinTemplate("self.format_int128({0})"),
9595
"format_uint128": BuiltinTemplate("self.format_uint128({0})"),
96+
# Formatted variants (same as base in Python)
97+
"format_int64_formatted": BuiltinTemplate("str({0})"),
98+
"format_int32_formatted": BuiltinTemplate("(str({0}) + 'i32')"),
99+
"format_float32_formatted": BuiltinTemplate(
100+
"(self.format_float32_value({0}) + 'f32')"
101+
),
102+
"format_float64_formatted": BuiltinTemplate("str({0})"),
103+
"format_string_formatted": BuiltinTemplate("self.format_string_value({0})"),
104+
"format_decimal_formatted": BuiltinTemplate("self.format_decimal({0})"),
105+
"format_int128_formatted": BuiltinTemplate("self.format_int128({0})"),
106+
"format_uint128_formatted": BuiltinTemplate("self.format_uint128({0})"),
96107
"greater": BuiltinTemplate("({0} > {1})"),
97108
"to_string": BuiltinTemplate("str({0})"),
98109
# Type conversions used by pretty printer
@@ -176,16 +187,33 @@ class BuiltinTemplate:
176187
"indent_sexp_io": BuiltinTemplate("nothing", ["indent_sexp!(pp)"]),
177188
"dedent_io": BuiltinTemplate("nothing", ["dedent!(pp)"]),
178189
"try_flat_io": BuiltinTemplate("try_flat(pp, {0}, {1})"),
179-
"format_int64": BuiltinTemplate("format_int(pp, {0})"),
190+
"format_int64": BuiltinTemplate("string({0})"),
180191
"format_int32": BuiltinTemplate('(string(Int64({0})) * "i32")'),
181192
"format_float32": BuiltinTemplate('(lowercase(string({0})) * "f32")'),
182-
"format_float64": BuiltinTemplate("format_float(pp, {0})"),
183-
"format_string": BuiltinTemplate("format_string(pp, {0})"),
193+
"format_float64": BuiltinTemplate("lowercase(string({0}))"),
194+
"format_string": BuiltinTemplate(
195+
"format_string(DEFAULT_CONSTANT_FORMATTER, pp, {0})"
196+
),
184197
"format_symbol": BuiltinTemplate("{0}"),
185198
"format_bool": BuiltinTemplate("format_bool(pp, {0})"),
186-
"format_decimal": BuiltinTemplate("format_decimal(pp, {0})"),
187-
"format_int128": BuiltinTemplate("format_int128(pp, {0})"),
188-
"format_uint128": BuiltinTemplate("format_uint128(pp, {0})"),
199+
"format_decimal": BuiltinTemplate(
200+
"format_decimal(DEFAULT_CONSTANT_FORMATTER, pp, {0})"
201+
),
202+
"format_int128": BuiltinTemplate(
203+
"format_int128(DEFAULT_CONSTANT_FORMATTER, pp, {0})"
204+
),
205+
"format_uint128": BuiltinTemplate(
206+
"format_uint128(DEFAULT_CONSTANT_FORMATTER, pp, {0})"
207+
),
208+
# Formatted variants (use constant_formatter hooks in Julia)
209+
"format_int64_formatted": BuiltinTemplate("format_int(pp, {0})"),
210+
"format_int32_formatted": BuiltinTemplate("format_int32(pp, {0})"),
211+
"format_float32_formatted": BuiltinTemplate("format_float32(pp, {0})"),
212+
"format_float64_formatted": BuiltinTemplate("format_float(pp, {0})"),
213+
"format_string_formatted": BuiltinTemplate("format_string(pp, {0})"),
214+
"format_decimal_formatted": BuiltinTemplate("format_decimal(pp, {0})"),
215+
"format_int128_formatted": BuiltinTemplate("format_int128(pp, {0})"),
216+
"format_uint128_formatted": BuiltinTemplate("format_uint128(pp, {0})"),
189217
"greater": BuiltinTemplate("({0} > {1})"),
190218
"to_string": BuiltinTemplate("string({0})"),
191219
# Type conversions used by pretty printer
@@ -283,6 +311,17 @@ class BuiltinTemplate:
283311
"format_decimal": BuiltinTemplate("p.formatDecimal({0})"),
284312
"format_int128": BuiltinTemplate("p.formatInt128({0})"),
285313
"format_uint128": BuiltinTemplate("p.formatUint128({0})"),
314+
# Formatted variants (same as base in Go)
315+
"format_int64_formatted": BuiltinTemplate('fmt.Sprintf("%d", {0})'),
316+
"format_int32_formatted": BuiltinTemplate('fmt.Sprintf("%di32", {0})'),
317+
"format_float32_formatted": BuiltinTemplate(
318+
"fmt.Sprintf(\"%sf32\", strconv.FormatFloat(float64({0}), 'g', -1, 32))"
319+
),
320+
"format_float64_formatted": BuiltinTemplate("formatFloat64({0})"),
321+
"format_string_formatted": BuiltinTemplate("p.formatStringValue({0})"),
322+
"format_decimal_formatted": BuiltinTemplate("p.formatDecimal({0})"),
323+
"format_int128_formatted": BuiltinTemplate("p.formatInt128({0})"),
324+
"format_uint128_formatted": BuiltinTemplate("p.formatUint128({0})"),
286325
"greater": BuiltinTemplate("({0} > {1})"),
287326
"to_string": BuiltinTemplate('fmt.Sprintf("%v", {0})'),
288327
# Type conversions used by pretty printer

‎meta/src/meta/extra_pretty_gen.py‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,10 +32,11 @@
3232
from .target_builtins import INT64, STRING, VOID, make_builtin
3333

3434
# Token-type messages that have format_* helpers in the templates.
35+
# Use the _formatted variants to go through the constant_formatter hooks.
3536
TOKEN_MESSAGES: dict[str, str] = {
36-
"DecimalValue": "format_decimal",
37-
"Int128Value": "format_int128",
38-
"UInt128Value": "format_uint128",
37+
"DecimalValue": "format_decimal_formatted",
38+
"Int128Value": "format_int128_formatted",
39+
"UInt128Value": "format_uint128_formatted",
3940
}
4041

4142
# Proto scalar type -> target IR base type

‎meta/src/meta/grammar.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -281,6 +281,9 @@ class Grammar:
281281
start: Nonterminal
282282
rules: dict[Nonterminal, list[Rule]] = field(default_factory=dict)
283283
tokens: list[Token] = field(default_factory=list)
284+
token_aliases: dict[str, str] = field(
285+
default_factory=dict
286+
) # alias name -> base token name
284287
ignored_completeness: list[str] = field(
285288
default_factory=list
286289
) # Message names to ignore in completeness checks
@@ -446,6 +449,9 @@ class GrammarConfig:
446449
rules: dict[Nonterminal, list[Rule]] = field(default_factory=dict)
447450
ignored_completeness: list[str] = field(default_factory=list)
448451
function_defs: dict[str, FunDef] = field(default_factory=dict)
452+
token_aliases: dict[str, str] = field(
453+
default_factory=dict
454+
) # alias name -> base name
449455

450456

451457
# Import traversal utilities here to avoid circular imports

‎meta/src/meta/grammar.y‎

Lines changed: 93 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,16 @@
4545
%token SYMBOL String r'[a-zA-Z_][a-zA-Z0-9_./#-]*'
4646
%token UINT128 logic.UInt128Value r'0x[0-9a-fA-F]+'
4747

48+
# Token aliases for formula constants (use hookable formatting in pretty printer)
49+
%token_alias FORMATTED_INT INT
50+
%token_alias FORMATTED_FLOAT FLOAT
51+
%token_alias FORMATTED_STRING STRING
52+
%token_alias FORMATTED_INT32 INT32
53+
%token_alias FORMATTED_FLOAT32 FLOAT32
54+
%token_alias FORMATTED_DECIMAL DECIMAL
55+
%token_alias FORMATTED_INT128 INT128
56+
%token_alias FORMATTED_UINT128 UINT128
57+
4858
# Type declarations for rules
4959
%nonterm abstraction logic.Abstraction
5060
%nonterm abstraction_with_arity Tuple[logic.Abstraction, Int64]
@@ -68,7 +78,6 @@
6878
%nonterm config_key_value Tuple[String, logic.Value]
6979
%nonterm configure transactions.Configure
7080
%nonterm conjunction logic.Conjunction
71-
%nonterm constant logic.Value
7281
%nonterm constraint logic.Constraint
7382
%nonterm construct logic.Construct
7483
%nonterm context transactions.Context
@@ -140,6 +149,9 @@
140149
%nonterm output transactions.Output
141150
%nonterm pragma logic.Pragma
142151
%nonterm primitive logic.Primitive
152+
%nonterm raw_date logic.DateValue
153+
%nonterm raw_datetime logic.DateTimeValue
154+
%nonterm raw_value logic.Value
143155
%nonterm read transactions.Read
144156
%nonterm reduce logic.Reduce
145157
%nonterm rel_atom logic.RelAtom
@@ -204,7 +216,7 @@ config_dict
204216
: "{" config_key_value* "}"
205217

206218
config_key_value
207-
: ":" SYMBOL value
219+
: ":" SYMBOL raw_value
208220
construct: $$ = builtin.tuple($2, $3)
209221
deconstruct:
210222
$2: String = $$[0]
@@ -219,27 +231,35 @@ value
219231
construct: $$ = logic.Value(datetime_value=$1)
220232
deconstruct if builtin.has_proto_field($$, 'datetime_value'):
221233
$1: logic.DateTimeValue = $$.datetime_value
222-
| STRING
234+
| FORMATTED_STRING
223235
construct: $$ = logic.Value(string_value=$1)
224236
deconstruct if builtin.has_proto_field($$, 'string_value'):
225237
$1: String = $$.string_value
226-
| INT
238+
| FORMATTED_INT32
239+
construct: $$ = logic.Value(int32_value=$1)
240+
deconstruct if builtin.has_proto_field($$, 'int32_value'):
241+
$1: Int32 = $$.int32_value
242+
| FORMATTED_INT
227243
construct: $$ = logic.Value(int_value=$1)
228244
deconstruct if builtin.has_proto_field($$, 'int_value'):
229245
$1: Int64 = $$.int_value
230-
| FLOAT
246+
| FORMATTED_FLOAT32
247+
construct: $$ = logic.Value(float32_value=$1)
248+
deconstruct if builtin.has_proto_field($$, 'float32_value'):
249+
$1: Float32 = $$.float32_value
250+
| FORMATTED_FLOAT
231251
construct: $$ = logic.Value(float_value=$1)
232252
deconstruct if builtin.has_proto_field($$, 'float_value'):
233253
$1: Float64 = $$.float_value
234-
| UINT128
254+
| FORMATTED_UINT128
235255
construct: $$ = logic.Value(uint128_value=$1)
236256
deconstruct if builtin.has_proto_field($$, 'uint128_value'):
237257
$1: logic.UInt128Value = $$.uint128_value
238-
| INT128
258+
| FORMATTED_INT128
239259
construct: $$ = logic.Value(int128_value=$1)
240260
deconstruct if builtin.has_proto_field($$, 'int128_value'):
241261
$1: logic.Int128Value = $$.int128_value
242-
| DECIMAL
262+
| FORMATTED_DECIMAL
243263
construct: $$ = logic.Value(decimal_value=$1)
244264
deconstruct if builtin.has_proto_field($$, 'decimal_value'):
245265
$1: logic.DecimalValue = $$.decimal_value
@@ -249,24 +269,72 @@ value
249269
construct: $$ = logic.Value(boolean_value=$1)
250270
deconstruct if builtin.has_proto_field($$, 'boolean_value'):
251271
$1: Boolean = $$.boolean_value
272+
273+
raw_value
274+
: raw_date
275+
construct: $$ = logic.Value(date_value=$1)
276+
deconstruct if builtin.has_proto_field($$, 'date_value'):
277+
$1: logic.DateValue = $$.date_value
278+
| raw_datetime
279+
construct: $$ = logic.Value(datetime_value=$1)
280+
deconstruct if builtin.has_proto_field($$, 'datetime_value'):
281+
$1: logic.DateTimeValue = $$.datetime_value
282+
| STRING
283+
construct: $$ = logic.Value(string_value=$1)
284+
deconstruct if builtin.has_proto_field($$, 'string_value'):
285+
$1: String = $$.string_value
252286
| INT32
253287
construct: $$ = logic.Value(int32_value=$1)
254288
deconstruct if builtin.has_proto_field($$, 'int32_value'):
255289
$1: Int32 = $$.int32_value
290+
| INT
291+
construct: $$ = logic.Value(int_value=$1)
292+
deconstruct if builtin.has_proto_field($$, 'int_value'):
293+
$1: Int64 = $$.int_value
256294
| FLOAT32
257295
construct: $$ = logic.Value(float32_value=$1)
258296
deconstruct if builtin.has_proto_field($$, 'float32_value'):
259297
$1: Float32 = $$.float32_value
298+
| FLOAT
299+
construct: $$ = logic.Value(float_value=$1)
300+
deconstruct if builtin.has_proto_field($$, 'float_value'):
301+
$1: Float64 = $$.float_value
302+
| UINT128
303+
construct: $$ = logic.Value(uint128_value=$1)
304+
deconstruct if builtin.has_proto_field($$, 'uint128_value'):
305+
$1: logic.UInt128Value = $$.uint128_value
306+
| INT128
307+
construct: $$ = logic.Value(int128_value=$1)
308+
deconstruct if builtin.has_proto_field($$, 'int128_value'):
309+
$1: logic.Int128Value = $$.int128_value
310+
| DECIMAL
311+
construct: $$ = logic.Value(decimal_value=$1)
312+
deconstruct if builtin.has_proto_field($$, 'decimal_value'):
313+
$1: logic.DecimalValue = $$.decimal_value
314+
| "missing"
315+
construct: $$ = logic.Value(missing_value=logic.MissingValue())
316+
| boolean_value
317+
construct: $$ = logic.Value(boolean_value=$1)
318+
deconstruct if builtin.has_proto_field($$, 'boolean_value'):
319+
$1: Boolean = $$.boolean_value
260320

261-
date
321+
raw_date
262322
: "(" "date" INT INT INT ")"
263323
construct: $$ = logic.DateValue(year=builtin.int64_to_int32($3), month=builtin.int64_to_int32($4), day=builtin.int64_to_int32($5))
264324
deconstruct:
265325
$3: Int64 = builtin.int32_to_int64($$.year)
266326
$4: Int64 = builtin.int32_to_int64($$.month)
267327
$5: Int64 = builtin.int32_to_int64($$.day)
268328

269-
datetime
329+
date
330+
: "(" "date" FORMATTED_INT FORMATTED_INT FORMATTED_INT ")"
331+
construct: $$ = logic.DateValue(year=builtin.int64_to_int32($3), month=builtin.int64_to_int32($4), day=builtin.int64_to_int32($5))
332+
deconstruct:
333+
$3: Int64 = builtin.int32_to_int64($$.year)
334+
$4: Int64 = builtin.int32_to_int64($$.month)
335+
$5: Int64 = builtin.int32_to_int64($$.day)
336+
337+
raw_datetime
270338
: "(" "datetime" INT INT INT INT INT INT INT? ")"
271339
construct: $$ = logic.DateTimeValue(year=builtin.int64_to_int32($3), month=builtin.int64_to_int32($4), day=builtin.int64_to_int32($5), hour=builtin.int64_to_int32($6), minute=builtin.int64_to_int32($7), second=builtin.int64_to_int32($8), microsecond=builtin.int64_to_int32(builtin.unwrap_option_or($9, 0)))
272340
deconstruct:
@@ -278,6 +346,18 @@ datetime
278346
$8: Int64 = builtin.int32_to_int64($$.second)
279347
$9: Optional[Int64] = builtin.some(builtin.int32_to_int64($$.microsecond))
280348

349+
datetime
350+
: "(" "datetime" FORMATTED_INT FORMATTED_INT FORMATTED_INT FORMATTED_INT FORMATTED_INT FORMATTED_INT FORMATTED_INT? ")"
351+
construct: $$ = logic.DateTimeValue(year=builtin.int64_to_int32($3), month=builtin.int64_to_int32($4), day=builtin.int64_to_int32($5), hour=builtin.int64_to_int32($6), minute=builtin.int64_to_int32($7), second=builtin.int64_to_int32($8), microsecond=builtin.int64_to_int32(builtin.unwrap_option_or($9, 0)))
352+
deconstruct:
353+
$3: Int64 = builtin.int32_to_int64($$.year)
354+
$4: Int64 = builtin.int32_to_int64($$.month)
355+
$5: Int64 = builtin.int32_to_int64($$.day)
356+
$6: Int64 = builtin.int32_to_int64($$.hour)
357+
$7: Int64 = builtin.int32_to_int64($$.minute)
358+
$8: Int64 = builtin.int32_to_int64($$.second)
359+
$9: Optional[Int64] = builtin.some(builtin.int32_to_int64($$.microsecond))
360+
281361
boolean_value
282362
: "true"
283363
construct: $$ = True
@@ -596,7 +676,7 @@ term
596676
construct: $$ = logic.Term(var=$1)
597677
deconstruct if builtin.has_proto_field($$, 'var'):
598678
$1: logic.Var = $$.var
599-
| constant
679+
| value
600680
construct: $$ = logic.Term(constant=$1)
601681
deconstruct if builtin.has_proto_field($$, 'constant'):
602682
$1: logic.Value = $$.constant
@@ -606,9 +686,6 @@ var
606686
construct: $$ = logic.Var(name=$1)
607687
deconstruct: $1: String = $$.name
608688

609-
constant
610-
: value
611-
612689
conjunction
613690
: "(" "and" formula* ")"
614691
construct: $$ = logic.Conjunction(args=$3)
@@ -749,7 +826,7 @@ rel_term
749826
$1: logic.Term = $$.term
750827

751828
specialized_value
752-
: "#" value
829+
: "#" raw_value
753830

754831
rel_atom
755832
: "(" "relatom" name rel_term* ")"
@@ -769,7 +846,7 @@ attrs
769846
: "(" "attrs" attribute* ")"
770847

771848
attribute
772-
: "(" "attribute" name value* ")"
849+
: "(" "attribute" name raw_value* ")"
773850
construct: $$ = logic.Attribute(name=$3, args=$4)
774851
deconstruct:
775852
$3: String = $$.name

0 commit comments

Comments
 (0)