Skip to content

Commit 1bd2fcf

Browse files
nystromnystrom
authored
[LQP Meta] Add formatted and non-formatted literal tokens (#228)
In the Julia pretty-printer, we want to obscure some constants (e.g., by hashing them so they don't appear in logs) but not all. For instance, literal values inside formula fragments should go through a ConstantFormatter hook, while values in config relations and attributes should be printed as-is. This PR introduces formatted literal tokens (FORMATTED_INT, FORMATTED_STRING, etc.) into the grammar as aliases of their unformatted counterparts (INT, STRING, etc.):
1 parent fc217ea commit 1bd2fcf

33 files changed

Lines changed: 14161 additions & 11747 deletions

‎meta/src/meta/cli.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -179,14 +179,16 @@ def run(args) -> int:
179179
return 1
180180
grammar = Grammar(
181181
start=start,
182+
token_aliases=grammar_config.token_aliases,
182183
ignored_completeness=grammar_config.ignored_completeness,
183184
function_defs=grammar_config.function_defs,
184185
)
185186
for _, rules in grammar_config.rules.items():
186187
for rule in rules:
187188
grammar.add_rule(rule)
188189

189-
# Add tokens with patterns from terminal declarations in grammar file
190+
# Add tokens with patterns from terminal declarations in grammar file.
191+
# Skip aliases — they share the base token's pattern and shouldn't be in the lexer.
190192
from .grammar import Token
191193

192194
for terminal_name, terminal_def in grammar_config.terminal_patterns.items():

‎meta/src/meta/codegen_templates.py‎

Lines changed: 47 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -87,14 +87,24 @@ class BuiltinTemplate:
8787
"format_int64": BuiltinTemplate("str({0})"),
8888
"format_int32": BuiltinTemplate("(str({0}) + 'i32')"),
8989
"format_uint32": BuiltinTemplate("(str({0}) + 'u32')"),
90-
"format_float32": BuiltinTemplate("(self.format_float32_value({0}) + 'f32')"),
90+
"format_float32": BuiltinTemplate("self.format_float32_literal({0})"),
9191
"format_float64": BuiltinTemplate("str({0})"),
9292
"format_string": BuiltinTemplate("self.format_string_value({0})"),
9393
"format_symbol": BuiltinTemplate("{0}"),
9494
"format_bool": BuiltinTemplate('("true" if {0} else "false")'),
9595
"format_decimal": BuiltinTemplate("self.format_decimal({0})"),
9696
"format_int128": BuiltinTemplate("self.format_int128({0})"),
9797
"format_uint128": BuiltinTemplate("self.format_uint128({0})"),
98+
# Formatted variants (same as base in Python)
99+
"format_int64_formatted": BuiltinTemplate("str({0})"),
100+
"format_int32_formatted": BuiltinTemplate("(str({0}) + 'i32')"),
101+
"format_uint32_formatted": BuiltinTemplate("(str({0}) + 'u32')"),
102+
"format_float32_formatted": BuiltinTemplate("self.format_float32_literal({0})"),
103+
"format_float64_formatted": BuiltinTemplate("str({0})"),
104+
"format_string_formatted": BuiltinTemplate("self.format_string_value({0})"),
105+
"format_decimal_formatted": BuiltinTemplate("self.format_decimal({0})"),
106+
"format_int128_formatted": BuiltinTemplate("self.format_int128({0})"),
107+
"format_uint128_formatted": BuiltinTemplate("self.format_uint128({0})"),
98108
"greater": BuiltinTemplate("({0} > {1})"),
99109
"to_string": BuiltinTemplate("str({0})"),
100110
# Type conversions used by pretty printer
@@ -180,17 +190,35 @@ class BuiltinTemplate:
180190
"indent_sexp_io": BuiltinTemplate("nothing", ["indent_sexp!(pp)"]),
181191
"dedent_io": BuiltinTemplate("nothing", ["dedent!(pp)"]),
182192
"try_flat_io": BuiltinTemplate("try_flat(pp, {0}, {1})"),
183-
"format_int64": BuiltinTemplate("format_int(pp, {0})"),
193+
"format_int64": BuiltinTemplate("string({0})"),
184194
"format_int32": BuiltinTemplate('(string(Int64({0})) * "i32")'),
185195
"format_uint32": BuiltinTemplate('(string(Int64({0})) * "u32")'),
186-
"format_float32": BuiltinTemplate('(lowercase(string({0})) * "f32")'),
187-
"format_float64": BuiltinTemplate("format_float(pp, {0})"),
188-
"format_string": BuiltinTemplate("format_string(pp, {0})"),
196+
"format_float32": BuiltinTemplate("format_float32_literal({0})"),
197+
"format_float64": BuiltinTemplate("lowercase(string({0}))"),
198+
"format_string": BuiltinTemplate(
199+
"format_string(DEFAULT_CONSTANT_FORMATTER, pp, {0})"
200+
),
189201
"format_symbol": BuiltinTemplate("{0}"),
190202
"format_bool": BuiltinTemplate("format_bool(pp, {0})"),
191-
"format_decimal": BuiltinTemplate("format_decimal(pp, {0})"),
192-
"format_int128": BuiltinTemplate("format_int128(pp, {0})"),
193-
"format_uint128": BuiltinTemplate("format_uint128(pp, {0})"),
203+
"format_decimal": BuiltinTemplate(
204+
"format_decimal(DEFAULT_CONSTANT_FORMATTER, pp, {0})"
205+
),
206+
"format_int128": BuiltinTemplate(
207+
"format_int128(DEFAULT_CONSTANT_FORMATTER, pp, {0})"
208+
),
209+
"format_uint128": BuiltinTemplate(
210+
"format_uint128(DEFAULT_CONSTANT_FORMATTER, pp, {0})"
211+
),
212+
# Formatted variants (use constant_formatter hooks in Julia)
213+
"format_int64_formatted": BuiltinTemplate("format_int(pp, {0})"),
214+
"format_int32_formatted": BuiltinTemplate("format_int32(pp, {0})"),
215+
"format_uint32_formatted": BuiltinTemplate("format_uint32(pp, {0})"),
216+
"format_float32_formatted": BuiltinTemplate("format_float32(pp, {0})"),
217+
"format_float64_formatted": BuiltinTemplate("format_float(pp, {0})"),
218+
"format_string_formatted": BuiltinTemplate("format_string(pp, {0})"),
219+
"format_decimal_formatted": BuiltinTemplate("format_decimal(pp, {0})"),
220+
"format_int128_formatted": BuiltinTemplate("format_int128(pp, {0})"),
221+
"format_uint128_formatted": BuiltinTemplate("format_uint128(pp, {0})"),
194222
"greater": BuiltinTemplate("({0} > {1})"),
195223
"to_string": BuiltinTemplate("string({0})"),
196224
# Type conversions used by pretty printer
@@ -281,16 +309,24 @@ class BuiltinTemplate:
281309
"format_int64": BuiltinTemplate('fmt.Sprintf("%d", {0})'),
282310
"format_int32": BuiltinTemplate('fmt.Sprintf("%di32", {0})'),
283311
"format_uint32": BuiltinTemplate('fmt.Sprintf("%du32", {0})'),
284-
"format_float32": BuiltinTemplate(
285-
"fmt.Sprintf(\"%sf32\", strconv.FormatFloat(float64({0}), 'g', -1, 32))"
286-
),
312+
"format_float32": BuiltinTemplate("formatFloat32({0})"),
287313
"format_float64": BuiltinTemplate("formatFloat64({0})"),
288314
"format_string": BuiltinTemplate("p.formatStringValue({0})"),
289315
"format_symbol": BuiltinTemplate("{0}"),
290316
"format_bool": BuiltinTemplate("formatBool({0})"),
291317
"format_decimal": BuiltinTemplate("p.formatDecimal({0})"),
292318
"format_int128": BuiltinTemplate("p.formatInt128({0})"),
293319
"format_uint128": BuiltinTemplate("p.formatUint128({0})"),
320+
# Formatted variants (same as base in Go)
321+
"format_int64_formatted": BuiltinTemplate('fmt.Sprintf("%d", {0})'),
322+
"format_int32_formatted": BuiltinTemplate('fmt.Sprintf("%di32", {0})'),
323+
"format_uint32_formatted": BuiltinTemplate('fmt.Sprintf("%du32", {0})'),
324+
"format_float32_formatted": BuiltinTemplate("formatFloat32({0})"),
325+
"format_float64_formatted": BuiltinTemplate("formatFloat64({0})"),
326+
"format_string_formatted": BuiltinTemplate("p.formatStringValue({0})"),
327+
"format_decimal_formatted": BuiltinTemplate("p.formatDecimal({0})"),
328+
"format_int128_formatted": BuiltinTemplate("p.formatInt128({0})"),
329+
"format_uint128_formatted": BuiltinTemplate("p.formatUint128({0})"),
294330
"greater": BuiltinTemplate("({0} > {1})"),
295331
"to_string": BuiltinTemplate('fmt.Sprintf("%v", {0})'),
296332
# Type conversions used by pretty printer

‎meta/src/meta/extra_pretty_gen.py‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,10 +32,11 @@
3232
from .target_builtins import INT64, STRING, VOID, make_builtin
3333

3434
# Token-type messages that have format_* helpers in the templates.
35+
# Use the _formatted variants to go through the constant_formatter hooks.
3536
TOKEN_MESSAGES: dict[str, str] = {
36-
"DecimalValue": "format_decimal",
37-
"Int128Value": "format_int128",
38-
"UInt128Value": "format_uint128",
37+
"DecimalValue": "format_decimal_formatted",
38+
"Int128Value": "format_int128_formatted",
39+
"UInt128Value": "format_uint128_formatted",
3940
}
4041

4142
# Proto scalar type -> target IR base type

‎meta/src/meta/grammar.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -281,6 +281,9 @@ class Grammar:
281281
start: Nonterminal
282282
rules: dict[Nonterminal, list[Rule]] = field(default_factory=dict)
283283
tokens: list[Token] = field(default_factory=list)
284+
token_aliases: dict[str, str] = field(
285+
default_factory=dict
286+
) # alias name -> base token name
284287
ignored_completeness: list[str] = field(
285288
default_factory=list
286289
) # Message names to ignore in completeness checks
@@ -446,6 +449,9 @@ class GrammarConfig:
446449
rules: dict[Nonterminal, list[Rule]] = field(default_factory=dict)
447450
ignored_completeness: list[str] = field(default_factory=list)
448451
function_defs: dict[str, FunDef] = field(default_factory=dict)
452+
token_aliases: dict[str, str] = field(
453+
default_factory=dict
454+
) # alias name -> base name
449455

450456

451457
# Import traversal utilities here to avoid circular imports

0 commit comments

Comments
 (0)