Get more tests working

Author: nystrom

meta/src/meta/codegen_templates.py

@@ -108,7 +108,7 @@ class BuiltinTemplate:
     "get_at": BuiltinTemplate("{0}[{1}]"),
     # Provenance tracking
     "span_start": BuiltinTemplate("self.span_start()"),
-    "record_span": BuiltinTemplate("None", ["self.record_span({0})"]),
+    "record_span": BuiltinTemplate("None", ["self.record_span({0}, {1})"]),
 }
 
 
@@ -196,7 +196,7 @@ class BuiltinTemplate:
     "get_at": BuiltinTemplate("{0}[{1} + 1]"),
     # Provenance tracking
     "span_start": BuiltinTemplate("span_start(parser)"),
-    "record_span": BuiltinTemplate("nothing", ["record_span!(parser, {0})"]),
+    "record_span": BuiltinTemplate("nothing", ["record_span!(parser, {0}, {1})"]),
 }
 
 
@@ -288,7 +288,7 @@ class BuiltinTemplate:
     "get_at": BuiltinTemplate("{0}[{1}]"),
     # Provenance tracking
     "span_start": BuiltinTemplate("int64(p.spanStart())"),
-    "record_span": BuiltinTemplate("nil", ["p.recordSpan(int({0}))"]),
+    "record_span": BuiltinTemplate("nil", ["p.recordSpan(int({0}), {1})"]),
 }
 
 __all__ = [

meta/src/meta/parser_gen.py

@@ -101,6 +101,7 @@
     ListExpr,
     ListType,
     Lit,
+    MessageType,
     ParseNonterminal,
     ParseNonterminalDef,
     Seq,
@@ -150,6 +151,7 @@ def _wrap_with_span(body: TargetExpr, return_type) -> TargetExpr:
     """Wrap a nonterminal body with span_start/record_span."""
     span_var = Var(gensym("span_start"), BaseType("Int64"))
     result_var = Var(gensym("result"), return_type)
+    type_name = return_type.name if isinstance(return_type, MessageType) else ""
     return Let(
         span_var,
         Call(make_builtin("span_start"), []),
@@ -158,7 +160,7 @@ def _wrap_with_span(body: TargetExpr, return_type) -> TargetExpr:
             body,
             Seq(
                 [
-                    Call(make_builtin("record_span"), [span_var]),
+                    Call(make_builtin("record_span"), [span_var, Lit(type_name)]),
                     result_var,
                 ]
             ),
@@ -220,7 +222,8 @@ def _generate_parse_method(
             )
         rhs = Let(Var(prediction, BaseType("Int64")), predictor, tail)
     assert return_type is not None
-    rhs = _wrap_with_span(rhs, return_type)
+    if isinstance(return_type, MessageType):
+        rhs = _wrap_with_span(rhs, return_type)
     return ParseNonterminalDef(lhs, [], return_type, rhs, indent)
 
 

meta/src/meta/target_builtins.py

@@ -251,7 +251,7 @@ def is_builtin(name: str) -> bool:
 
 # === Provenance tracking ===
 register_builtin("span_start", [], INT64)
-register_builtin("record_span", [INT64], VOID)
+register_builtin("record_span", [INT64, STRING], VOID)
 
 
 # === Validation functions ===

meta/src/meta/templates/parser.go.template

@@ -31,8 +31,9 @@ type Location struct {{
 
 // Span represents a source span from start to stop location.
 type Span struct {{
-	Start Location
-	Stop  Location
+	Start    Location
+	Stop     Location
+	TypeName string
 }}
 
 // ParseError represents a parse error
@@ -329,7 +330,7 @@ type Parser struct {{
 	pos               int
 	idToDebugInfo     map[string]map[relationIdKey]string
 	currentFragmentID []byte
-	Provenance        map[string]Span
+	Provenance        map[int]Span
 	lineStarts        []int
 }}
 
@@ -340,7 +341,7 @@ func NewParser(tokens []Token, input string) *Parser {{
 		pos:               0,
 		idToDebugInfo:     make(map[string]map[relationIdKey]string),
 		currentFragmentID: nil,
-		Provenance:        make(map[string]Span),
+		Provenance:        make(map[int]Span),
 		lineStarts:        computeLineStarts(input),
 	}}
 }}
@@ -364,16 +365,22 @@ func (p *Parser) spanStart() int {{
 	return p.lookahead(0).StartPos
 }}
 
-func (p *Parser) recordSpan(startOffset int) {{
+func (p *Parser) recordSpan(startOffset int, typeName string) {{
+	// First-wins: innermost parse function records first; outer wrappers
+	// that share the same offset do not overwrite.
+	if _, exists := p.Provenance[startOffset]; exists {{
+		return
+	}}
 	endOffset := startOffset
 	if p.pos > 0 {{
 		endOffset = p.tokens[p.pos-1].EndPos
 	}}
 	s := Span{{
-		Start: p.makeLocation(startOffset),
-		Stop:  p.makeLocation(endOffset),
+		Start:    p.makeLocation(startOffset),
+		Stop:     p.makeLocation(endOffset),
+		TypeName: typeName,
 	}}
-	p.Provenance[""] = s
+	p.Provenance[startOffset] = s
 }}
 
 func (p *Parser) lookahead(k int) Token {{
@@ -565,7 +572,7 @@ func toPascalCase(s string) string {{
 {parse_nonterminal_defns}
 
 // Parse parses the input string and returns (result, provenance, error).
-func Parse(input string) (result *pb.Transaction, provenance map[string]Span, err error) {{
+func Parse(input string) (result *pb.Transaction, provenance map[int]Span, err error) {{
 	defer func() {{
 		if r := recover(); r != nil {{
 			if pe, ok := r.(ParseError); ok {{

meta/src/meta/templates/parser.jl.template

@@ -37,6 +37,7 @@ end
 struct Span
     start::Location
     stop::Location
+    type_name::String
 end
 
 struct Token
@@ -189,7 +190,7 @@ mutable struct ParserState
     id_to_debuginfo::Dict{{Vector{{UInt8}},Vector{{Pair{{Tuple{{UInt64,UInt64}},String}}}}}}
     _current_fragment_id::Union{{Nothing,Vector{{UInt8}}}}
     _relation_id_to_name::Dict{{Tuple{{UInt64,UInt64}},String}}
-    provenance::Dict{{Tuple{{Vararg{{Int}}}},Span}}
+    provenance::Dict{{Int,Span}}
     _line_starts::Vector{{Int}}
 
     function ParserState(tokens::Vector{{Token}}, input_str::String)
@@ -208,14 +209,17 @@ function span_start(parser::ParserState)::Int
     return lookahead(parser, 0).start_pos
 end
 
-function record_span!(parser::ParserState, start_offset::Int)
+function record_span!(parser::ParserState, start_offset::Int, type_name::String="")
+    # First-wins: innermost parse function records first; outer wrappers
+    # that share the same offset do not overwrite.
+    haskey(parser.provenance, start_offset) && return nothing
     if parser.pos > 1
         end_offset = parser.tokens[parser.pos - 1].end_pos
     else
         end_offset = start_offset
     end
-    s = Span(_make_location(parser, start_offset), _make_location(parser, end_offset))
-    parser.provenance[tuple()] = s
+    s = Span(_make_location(parser, start_offset), _make_location(parser, end_offset), type_name)
+    parser.provenance[start_offset] = s
     return nothing
 end
 

meta/src/meta/templates/parser.py.template

@@ -48,11 +48,12 @@ class Location:
 class Span:
     """Source span from start to stop location."""
 
-    __slots__ = ("start", "stop")
+    __slots__ = ("start", "stop", "type_name")
 
-    def __init__(self, start: Location, stop: Location):
+    def __init__(self, start: Location, stop: Location, type_name: str = ""):
         self.start = start
         self.stop = stop
+        self.type_name = type_name
 
     def __repr__(self) -> str:
         return f"Span({{self.start}}, {{self.stop}})"
@@ -217,7 +218,7 @@ class Parser:
         self.id_to_debuginfo = {{}}
         self._current_fragment_id: bytes | None = None
         self._relation_id_to_name = {{}}
-        self.provenance: dict[tuple[int, ...], Span] = {{}}
+        self.provenance: dict[int, Span] = {{}}
         self._line_starts = _compute_line_starts(input_str)
 
     def _make_location(self, offset: int) -> Location:
@@ -230,14 +231,20 @@ class Parser:
         """Return the start offset of the current token."""
         return self.lookahead(0).start_pos
 
-    def record_span(self, start_offset: int) -> None:
-        """Record a span from start_offset to the previous token's end."""
+    def record_span(self, start_offset: int, type_name: str = "") -> None:
+        """Record a span from start_offset to the previous token's end.
+
+        Uses first-wins semantics: the innermost parse function records first,
+        and outer wrappers that share the same offset do not overwrite.
+        """
+        if start_offset in self.provenance:
+            return
         if self.pos > 0:
             end_offset = self.tokens[self.pos - 1].end_pos
         else:
             end_offset = start_offset
-        span = Span(self._make_location(start_offset), self._make_location(end_offset))
-        self.provenance[tuple()] = span
+        span = Span(self._make_location(start_offset), self._make_location(end_offset), type_name)
+        self.provenance[start_offset] = span
 
     def lookahead(self, k: int = 0) -> Token:
         """Get lookahead token at offset k."""
@@ -349,7 +356,7 @@ class Parser:
     # --- Parse methods ---
 {parse_nonterminal_defns}
 
-def parse(input_str: str) -> tuple[Any, dict[tuple[int, ...], Span]]:
+def parse(input_str: str) -> tuple[Any, dict[int, Span]]:
     """Parse input string and return (result, provenance) tuple."""
     lexer = Lexer(input_str)
     parser = Parser(lexer.tokens, input_str)