fix(compiler): rustc-style diagnostics and five interpreter correctness bugs

Author: dylan-sutton-chavez

compiler/README.md

@@ -20,7 +20,7 @@ What this leaves is a small, fast, deterministic core: arithmetic with arbitrary
 * **Lexer**: Hand-written, LUT-driven scanner over CPython 3.13 token kinds. Tokens are `(start, end, kind)` offsets into the source buffer; no string copies during lexing.
 * **Parser**: Single-pass, Pratt precedence climbing. Emits SSA-versioned bytecode directly (`x` -> `x_1`, `x_2`) with explicit `Phi` opcodes at control-flow joins. No intermediate AST.
 * **Optimizer**: One peephole pass: constant folding over adjacent literal operands, plus dead-code compaction with jump remapping. Does not propagate through `LoadName`.
-* **VM**: Stack-based interpreter over a pre-compiled `Vec<ThreadedOp>` where operands are baked into typed enum variants. Dispatch is a flat `match` over the variant. One LoadAttr+Call superinstruction (`CallMethod`).
+* **VM**: Stack-based interpreter over `Vec<Instruction>` where each `Instruction` is `(opcode: OpCode, operand: u16)`. Dispatch is a flat `match` on the opcode (Rust lowers it to a jump table). One LoadAttr+Call superinstruction (`CallMethod` + `CallMethodArgs`), fused once per chunk and cached in `cache.fused_ref()`.
 * **Inline Caching**: Per-instruction type-recording cache for arithmetic and comparisons. After 4 stable hits the IC stores a `FastOp` (`AddInt`, `LtFloat`, ...) used as a speculative fast path with type-guard deopt.
 * **Template Memoization**: Pure functions called repeatedly with the same arguments return cached results after 2 hits, bypassing full execution.
 * **Memory**: NaN-boxed 64-bit `Val` (48-bit signed int, IEEE-754 float, bool, None, 28-bit heap index). Mark-and-sweep GC. Arbitrary-precision `BigInt` fallback for integers outside the 48-bit range.

compiler/src/main.rs

@@ -59,16 +59,18 @@ fn parse_args() -> (String, usize, bool, bool) {
 }
 
 fn run(path: &str, sandbox: bool, verbosity: usize, quiet: bool) -> Result<(), String> {
-    let src = if path.ends_with(".py") {
+    let is_file = path.ends_with(".py");
+    let src = if is_file {
         fs::read_to_string(path).map_err(|_| s!("io: cannot access '", str path, "'"))?
     } else {
         path.to_string()
     };
+    let diag_path = if is_file { Some(path) } else { None };
 
     let (mut chunk, errs) = Parser::new(&src, lexer(&src)).parse();
     if !errs.is_empty() {
         for e in &errs {
-            eprint_msg(&s!("syntax: ", str &e.render_with_path(path)));
+            eprint_msg(&e.render(&src, diag_path));
         }
         exit(1);
     }

compiler/src/modules/parser/control.rs

@@ -106,6 +106,7 @@ impl<'src, I: Iterator<Item = Token>> Parser<'src, I> {
         let loop_start = self.chunk.instructions.len() as u16;
         self.loop_starts.push(loop_start);
         self.loop_breaks.push(vec![]);
+        self.loop_kinds.push(false);
 
         self.expr();
         self.chunk.emit(OpCode::JumpIfFalse, 0);
@@ -123,6 +124,7 @@ impl<'src, I: Iterator<Item = Token>> Parser<'src, I> {
         }
 
         self.loop_starts.pop();
+        self.loop_kinds.pop();
         for pos in self.loop_breaks.pop().unwrap_or_default() {
             self.patch(pos);
         }
@@ -157,6 +159,7 @@ impl<'src, I: Iterator<Item = Token>> Parser<'src, I> {
         let loop_start = self.chunk.instructions.len() as u16;
         self.loop_starts.push(loop_start);
         self.loop_breaks.push(vec![]);
+        self.loop_kinds.push(true);
 
         self.chunk.emit(OpCode::ForIter, 0);
         let fi = self.chunk.instructions.len() - 1;
@@ -186,6 +189,7 @@ impl<'src, I: Iterator<Item = Token>> Parser<'src, I> {
         }
 
         self.loop_starts.pop();
+        self.loop_kinds.pop();
         for pos in self.loop_breaks.pop().unwrap_or_default() {
             self.patch(pos);
         }

compiler/src/modules/parser/expr.rs

@@ -222,9 +222,9 @@ impl<'src, I: Iterator<Item = Token>> Parser<'src, I> {
                 }
             }
             TokenType::Lambda => self.parse_lambda(),
-            _ => {
-                if t.kind != TokenType::Endmarker { self.error("unexpected token"); }
-            }
+            // Anchor at the consumed token so the caret points at the
+            // structural marker (newline, dedent, etc.) the user actually wrote.
+            _ => self.error_at(t.start, t.end, "expected expression"),
         }
         self.postfix_tail();
     }

compiler/src/modules/parser/literals.rs

@@ -435,6 +435,7 @@ impl<'src, I: Iterator<Item = Token>> Parser<'src, I> {
             i.opcode,
             OpCode::CallPrint
             | OpCode::StoreItem
+            | OpCode::DelItem
             | OpCode::StoreAttr
             | OpCode::CallInput
             | OpCode::Global

compiler/src/modules/parser/mod.rs

@@ -23,6 +23,10 @@ pub struct Parser<'src, I: Iterator<Item = Token>> {
     pub(super) loop_starts: Vec<u16>,
     pub(super) last_line: usize,
     pub(super) loop_breaks: Vec<Vec<usize>>,
+    // Parallel to loop_starts/loop_breaks: true for `for` loops (which push
+    // an iter on iter_stack), false for `while`. Lets `break` emit PopIter
+    // only when escaping a for-loop, so nested for/while combinations work.
+    pub(super) loop_kinds: Vec<bool>,
     pub(super) expr_depth: usize,
     pub(super) saw_newline: bool,
     pub errors: Vec<Diagnostic>,
@@ -161,19 +165,22 @@ impl<'src, I: Iterator<Item = Token>> Parser<'src, I> {
         tok
     }
 
+    /* Push a Diagnostic anchored at the next token's span (or at end-of-source
+       if we ran past EOF) and panic-mode sync to the next statement boundary
+       so we can keep reporting downstream errors. */
     pub(super) fn error(&mut self, msg: &str) {
-        let (line, byte_offset, end) = self
-            .tokens
-            .peek()
-            .map(|t| (t.line, t.start, t.end))
-            .unwrap_or((self.last_line, 0, 0));
-
-        let col = self.source[..byte_offset]
-            .rfind('\n')
-            .map(|line_start| byte_offset - line_start - 1)
-            .unwrap_or(byte_offset);
-
-        self.errors.push(Diagnostic { line, col, end, msg: msg.to_string() });
+        let n = self.source.len();
+        let (start, end) = self.tokens.peek()
+            .map(|t| (t.start, t.end))
+            .unwrap_or((n, n));
+        self.error_at(start, end, msg);
+    }
+
+    /* Same as `error` but anchored at the caller-provided span. Use when
+       a parser has already consumed the offending token and wants the
+       diagnostic to point at it (not at whatever comes next). */
+    pub(super) fn error_at(&mut self, start: usize, end: usize, msg: &str) {
+        self.errors.push(Diagnostic { start, end, msg: msg.to_string() });
         loop {
             match self.tokens.peek().map(|t| t.kind) {
                 None | Some(TokenType::Newline | TokenType::Dedent | TokenType::Endmarker) => break,
@@ -192,6 +199,11 @@ impl<'src, I: Iterator<Item = Token>> Parser<'src, I> {
         self.lexeme(&t).to_string()
     }
 
+    /* Surface user-visible tokens. Skips Newline (latching `saw_newline`),
+       Nl, Comment. Treats Endmarker as None so `at_end()` and "loop while
+       not closer/None" patterns terminate cleanly without explicit Endmarker
+       checks at every site. The raw iterator (`self.tokens.peek()`) still
+       sees Endmarker for diagnostic anchoring in `error()` / `eat()`. */
     pub(super) fn peek(&mut self) -> Option<TokenType> {
         loop {
             match self.tokens.peek().map(|t| t.kind) {
@@ -200,8 +212,8 @@ impl<'src, I: Iterator<Item = Token>> Parser<'src, I> {
                     self.tokens.next();
                 }
                 Some(TokenType::Nl | TokenType::Comment) => { self.tokens.next(); }
+                Some(TokenType::Endmarker) | None => return None,
                 Some(k) => return Some(k),
-                None => return None,
             }
         }
     }
@@ -210,16 +222,27 @@ impl<'src, I: Iterator<Item = Token>> Parser<'src, I> {
         self.chunk.instructions[pos].operand = self.chunk.instructions.len() as u16;
     }
 
+    /* Consume `kind` or push a diagnostic with a friendly description of
+       what was actually found (lexeme for normal tokens, kind label for
+       synthetic Endmarker / structural tokens, and "EOF" past end). */
     pub(super) fn eat(&mut self, kind: TokenType) {
         if matches!(self.peek(), Some(k) if k == kind) {
             self.advance();
-        } else {
-            let token_text = match self.tokens.peek() {
-                Some(t) => &self.source[t.start..t.end],
-                None => "EOF",
-            };
-            self.error(&s!("expected ", str kind.as_str(), ", got '", str token_text, "'"));
+            return;
         }
+        let label: alloc::string::String = match self.tokens.peek() {
+            Some(t) if t.kind == TokenType::Endmarker => "EOF".to_string(),
+            Some(t) if t.kind == TokenType::Newline || t.kind == TokenType::Nl => "newline".to_string(),
+            Some(t) if t.kind == TokenType::Indent => "indent".to_string(),
+            Some(t) if t.kind == TokenType::Dedent => "dedent".to_string(),
+            Some(t) if t.start == t.end => t.kind.as_str().to_string(),
+            Some(t) => {
+                let mut s = alloc::string::String::with_capacity(t.end - t.start + 2);
+                s.push('\''); s.push_str(&self.source[t.start..t.end]); s.push('\''); s
+            }
+            None => "EOF".to_string(),
+        };
+        self.error(&s!("expected ", str kind.as_str(), ", got ", str &label));
     }
 
     pub(super) fn eat_if(&mut self, kind: TokenType) -> bool {
@@ -244,6 +267,7 @@ impl<'src, I: Iterator<Item = Token>> Parser<'src, I> {
             join_stack: Vec::new(),
             loop_starts: Vec::new(),
             loop_breaks: Vec::new(),
+            loop_kinds: Vec::new(),
             saw_newline: false,
             expr_depth: 0,
             last_line: 0,
@@ -257,21 +281,33 @@ impl<'src, I: Iterator<Item = Token>> Parser<'src, I> {
             if self.at_end() { break; }
 
             let produced_value = self.stmt();
-            if !self.at_end() && produced_value { self.chunk.emit(OpCode::PopTop, 0); }
+            // Always pop expression-statement results: the implicit ReturnValue
+            // at chunk end returns Val::none() if the stack is empty.
+            if produced_value { self.chunk.emit(OpCode::PopTop, 0); }
         }
 
         if self.chunk.overflow {
-            let line = self.errors.last().map(|e| e.line).unwrap_or(0);
+            let n = self.source.len();
             self.errors.push(Diagnostic {
-                line, col: 0, end: 0,
+                start: n, end: n,
                 msg: "program too large: exceeded maximum instruction limit".to_string()
             });
         }
 
         if !self.errors.is_empty() {
+            // Wipe ALL bytecode side-state so finalize_prev_slots doesn't
+            // index `canonical` (built from `names`) with stale phi sources.
             self.chunk.instructions.clear();
             self.chunk.constants.clear();
             self.chunk.names.clear();
+            self.chunk.phi_sources.clear();
+            self.chunk.phi_map.clear();
+            self.chunk.functions.clear();
+            self.chunk.classes.clear();
+            self.chunk.name_index.clear();
+            self.chunk.nonlocals.clear();
+            self.chunk.annotations.clear();
+            self.loop_kinds.clear();
         }
 
         self.chunk.emit(OpCode::ReturnValue, 0);

compiler/src/modules/parser/stmt.rs

@@ -50,9 +50,23 @@ impl<'src, I: Iterator<Item = Token>> Parser<'src, I> {
             Some(TokenType::Yield) => {
                 self.advance();
                 if self.eat_if(TokenType::From) {
+                    // Lower `yield from <expr>` into a for-loop that re-yields
+                    // each value: GetIter pushes onto iter_stack, ForIter pulls
+                    // the next item, Yield re-yields it, PopTop discards the
+                    // (None) send value on resume, Jump back. LoadNone at the
+                    // end gives the expression a value (sub-generator return
+                    // value isn't tracked, so always None).
                     self.expr();
-                    self.chunk.emit(OpCode::YieldFrom, 0);
-                } else if matches!(self.peek(), Some(TokenType::Newline | TokenType::Endmarker)) {
+                    self.chunk.emit(OpCode::GetIter, 0);
+                    let loop_start = self.chunk.instructions.len() as u16;
+                    self.chunk.emit(OpCode::ForIter, 0);
+                    let fi = self.chunk.instructions.len() - 1;
+                    self.chunk.emit(OpCode::Yield, 0);
+                    self.chunk.emit(OpCode::PopTop, 0);
+                    self.chunk.emit(OpCode::Jump, loop_start);
+                    self.patch(fi);
+                    self.chunk.emit(OpCode::LoadNone, 0);
+                } else if matches!(self.peek(), Some(TokenType::Newline) | None) {
                     self.chunk.emit(OpCode::LoadNone, 0);
                     self.chunk.emit(OpCode::Yield, 0);
                 } else {
@@ -141,14 +155,27 @@ impl<'src, I: Iterator<Item = Token>> Parser<'src, I> {
             }
             Some(TokenType::Del) => {
                 self.advance();
-                let name = self.advance_text();
-                let idx = self.push_ssa_name(&name, self.current_version(&name));
-                self.chunk.emit(OpCode::Del, idx);
+                loop {
+                    let name = self.advance_text();
+                    if self.eat_if(TokenType::Lsqb) {
+                        // del x[k]: load container + key, mutate in place.
+                        // Slice deletion (x[i:j]) is not supported — `expr()`
+                        // stops at `:` so the eat(Rsqb) below errors cleanly.
+                        self.emit_load_ssa(name);
+                        self.expr();
+                        self.eat(TokenType::Rsqb);
+                        self.chunk.emit(OpCode::DelItem, 0);
+                    } else {
+                        let idx = self.push_ssa_name(&name, self.current_version(&name));
+                        self.chunk.emit(OpCode::Del, idx);
+                    }
+                    if !self.eat_if(TokenType::Comma) { break; }
+                }
                 false
             }
             Some(TokenType::Raise) => {
                 self.advance();
-                if !matches!(self.peek(), Some(TokenType::Newline | TokenType::Endmarker)) {
+                if !matches!(self.peek(), Some(TokenType::Newline) | None) {
                     self.expr();
                     if self.eat_if(TokenType::From) {
                         self.expr();
@@ -166,6 +193,12 @@ impl<'src, I: Iterator<Item = Token>> Parser<'src, I> {
                 if self.loop_breaks.is_empty() {
                     self.error("'break' outside loop");
                 } else {
+                    // For-loops own an iter on iter_stack; pop it before
+                    // jumping out, otherwise the surrounding for-iter would
+                    // read the abandoned iterator.
+                    if let Some(true) = self.loop_kinds.last() {
+                        self.chunk.emit(OpCode::PopIter, 0);
+                    }
                     self.chunk.emit(OpCode::Jump, 0);
                     if let Some(breaks) = self.loop_breaks.last_mut() {
                         breaks.push(self.chunk.instructions.len() - 1);
@@ -251,7 +284,7 @@ impl<'src, I: Iterator<Item = Token>> Parser<'src, I> {
                 break;
             }
             let produced_value = self.stmt();
-            if !self.at_end() && produced_value {
+            if produced_value {
                 self.chunk.emit(OpCode::PopTop, 0);
             }
             if indented { continue; }