feat(selfhost): Step 3 spans — lexer tier-1 (line/col) + tier-2 (length) complete

The rss tokenizer already tracked per-char line/col (used by statement_end); added
a `len` field to Tok (= the consumed source span j-i, matching the Rust lexer's
index-start) and taught lexer.rss to emit the real `<line>:<col>:<len>` prefix
instead of the `0:0:0` placeholder.

Result: lexer_parity_corpus is byte-exact at ALL THREE tiers over 576 files —
tier 0 (kind+payload), tier 1 (+line/col), tier 2 (+length), token-mismatches 0
each. The lexer span ladder is fully closed; string/char/multiline/interp
delimiter spans and the max(1) cases all match. Tok.len is additive (only lexer.rss
reads it); parser/checker/astdump parity unaffected (all samples green).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Author: olwang

selfhost/lexer.rss

@@ -24,6 +24,11 @@ fn kind_name(k: read Int) -> String {
     return "UnknownKind"
 }
 
+// Source char length of a token (span length), tier-2 field.
+fn tok_len(t: read Tok) -> Int {
+    return t.len
+}
+
 fn main() -> Unit {
     let source = Args.get_or_default(index: 0, default: read "")
     let toks = tokenize(source: read source)
@@ -33,9 +38,14 @@ fn main() -> Unit {
         let t = List.get(list: read toks, index: read i)
         let kn = kind_name(k: read t.kind)
         let esc = escape(s: read t.text)
-        let a = String.concat(left: read "0:0:0\t", right: read kn)
-        let b = String.concat(left: read a, right: read "\t")
-        let line = String.concat(left: read b, right: read esc)
+        // <line>:<col>:<len>  — line/col are real (tier 1); len is the token's
+        // char length (tier 2), 0 for the synthetic EOF.
+        let pos = String.concat(left: read String.concat(left: read String.concat(left: read String.from_int(value: t.line), right: read ":"), right: read String.from_int(value: t.col)), right: read ":")
+        let plen = String.concat(left: read pos, right: read String.from_int(value: tok_len(t: read t)))
+        let a = String.concat(left: read plen, right: read "\t")
+        let b = String.concat(left: read a, right: read kn)
+        let c = String.concat(left: read b, right: read "\t")
+        let line = String.concat(left: read c, right: read esc)
         Log.write(message: read line)
         i = i + 1
     }

selfhost/scan.rss

@@ -32,6 +32,7 @@ struct Tok {
     val: Int
     line: Int
     col: Int
+    len: Int
     text: String
 }
 
@@ -559,7 +560,7 @@ fn tokenize(source: read String) -> fresh List<Tok> {
             }
         }
         if kind != 0 {
-            let t = Tok(kind: kind, val: val, line: startLine, col: startCol, text: text)
+            let t = Tok(kind: kind, val: val, line: startLine, col: startCol, len: (j - i), text: text)
             List.push(list: mut toks, value: read t)
         }
         // Advance line/col over the consumed span [i, j).
@@ -576,7 +577,7 @@ fn tokenize(source: read String) -> fresh List<Tok> {
         }
         i = j
     }
-    let eof = Tok(kind: TOK_EOF, val: 0, line: curLine, col: curCol, text: "")
+    let eof = Tok(kind: TOK_EOF, val: 0, line: curLine, col: curCol, len: 0, text: "")
     List.push(list: mut toks, value: read eof)
     return toks
 }