Remove raw tokens (#468)

Removes all instances of RawTokens in favour of a widened AST that can represent any valid VHDL (as defined by the LRM), but also invalid VHDL. The latter must be rejected in a subsequent analysis stage.

Author: Schottkyc137

vhdl_syntax/src/builder.rs

@@ -23,10 +23,7 @@
 //! ```
 
 use crate::latin_1::{char_to_latin1, Latin1Str, Latin1String, NonLatin1CharError};
-use crate::parser::builder::NodeBuilder;
-use crate::syntax::node::SyntaxNode;
-use crate::syntax::node_kind::NodeKind;
-use crate::tokens::{Keyword, Token, TokenKind, Tokenize, Trivia, TriviaPiece};
+use crate::tokens::{Keyword, Token, TokenKind, Trivia, TriviaPiece};
 
 fn default_trivia() -> Trivia {
     Trivia::from([TriviaPiece::Spaces(1)])
@@ -451,69 +448,6 @@ fn bit_string_literal_helpers_produce_correct_prefixes_prefixes() {
     assert_eq!(tok.text().to_string(), r#"X"FF""#);
 }
 
-// MARK: RawNodeBuilder
-
-/// Shared builder for `!RawTokens` AST nodes (e.g. `ActualPartSyntax`, `RawTokensSyntax`).
-/// Use the generated per-node wrappers rather than this type directly.
-pub(crate) struct RawNodeBuilder {
-    kind: NodeKind,
-    tokens: Vec<Token>,
-}
-
-impl RawNodeBuilder {
-    pub(crate) fn new(kind: NodeKind) -> Self {
-        Self {
-            kind,
-            tokens: vec![],
-        }
-    }
-
-    pub(crate) fn token(mut self, t: impl Into<Token>) -> Self {
-        self.tokens.push(t.into());
-        self
-    }
-
-    /// Tokenizes `vhdl` and stores the resulting tokens (EOF token excluded).
-    /// Adds one leading space to the first token when it carries no trivia,
-    /// matching the default-trivia convention of all other builders.
-    pub(crate) fn from_vhdl(kind: NodeKind, vhdl: impl Tokenize) -> Self {
-        let mut tokens: Vec<Token> = vhdl
-            .tokenize()
-            .filter(|t| t.kind() != TokenKind::Eof)
-            .collect();
-        if let Some(first) = tokens.first_mut() {
-            if first.leading_trivia().is_empty() {
-                first.set_leading_trivia(Trivia::from([TriviaPiece::Spaces(1)]));
-            }
-        }
-        Self { kind, tokens }
-    }
-
-    pub(crate) fn build(self) -> SyntaxNode {
-        let mut b = NodeBuilder::new();
-        b.start_node(self.kind);
-        for token in self.tokens {
-            b.push(token);
-        }
-        b.end_node();
-        SyntaxNode::new_root(b.end())
-    }
-}
-
-#[test]
-fn raw_node_builder_from_vhdl_adds_leading_space() {
-    let syntax = RawNodeBuilder::from_vhdl(NodeKind::ActualPart, Latin1Str::new(b"clk")).build();
-    assert_eq!(syntax.to_string(), " clk");
-}
-
-/// A programmatically assembled `RawNodeBuilder` holds exactly the pushed tokens.
-#[test]
-fn raw_node_builder_programmatic_token_chain() {
-    let tok = Token::new(TokenKind::Identifier, b"foo".as_slice(), Trivia::default());
-    let syntax = RawNodeBuilder::new(NodeKind::ActualPart).token(tok).build();
-    assert_eq!(syntax.tokens().count(), 1);
-}
-
 // MARK: Canonical tokens
 
 impl TokenKind {

vhdl_syntax/src/parser/productions/composite_types.rs

@@ -4,7 +4,7 @@
 //
 // Copyright (c)  2025, Lukas Scheller lukasscheller@icloud.com
 /// Parsing of composite types (LRM §5.3)
-use crate::parser::{util::LookaheadError, Parser};
+use crate::parser::Parser;
 use crate::syntax::node_kind::NodeKind::*;
 use crate::tokens::token_kind::Keyword as Kw;
 use crate::tokens::TokenKind::*;
@@ -69,42 +69,10 @@ impl Parser {
     pub fn index_constraint(&mut self) {
         self.start_node(IndexConstraint);
         self.expect_token(LeftPar);
-        self.separated_list(Parser::discrete_range, Comma);
+        self.separated_list(Parser::expression, Comma);
         self.expect_token(RightPar);
         self.end_node();
     }
-
-    pub fn discrete_range(&mut self) {
-        // One of the following tokens must follow after a `discrete_range`.
-        //    FOLLOW(discrete_range) := "," | ")" | "|" | "=>" | "generate" | "loop" | ";"
-        let end_of_range = match self.lookahead([
-            Comma,
-            RightPar,
-            RightArrow,
-            Bar,
-            Keyword(Kw::Generate),
-            Keyword(Kw::Loop),
-            SemiColon,
-        ]) {
-            Ok((tok, end_index)) => Some((tok, end_index)),
-            // If EOF is reached, the range cannot be parsed correctly
-            Err((LookaheadError::Eof, _)) => {
-                self.eof_err();
-                None
-            }
-            // Since we use `usize::MAX` as a maximum index, this error is not possible!
-            Err((LookaheadError::MaxIndexReached, _)) => unreachable!(),
-            // This error is only possible, when a `RightPar` is found before any token in `kinds`.
-            // Since `RightPar` is in `kinds` that's not possible!
-            Err((LookaheadError::TokenKindNotFound, _)) => unreachable!(),
-        };
-
-        if let Some((_, end_index)) = end_of_range {
-            self.start_node(RawTokens);
-            self.skip_to(end_index);
-            self.end_node();
-        }
-    }
 }
 
 #[cfg(test)]

vhdl_syntax/src/parser/productions/concurrent_statement.rs

@@ -402,7 +402,7 @@ impl Parser {
         self.start_node(ParameterSpecification);
         self.identifier();
         self.expect_kw(Kw::In);
-        self.discrete_range();
+        self.expression();
         self.end_node();
     }
 

vhdl_syntax/src/parser/productions/expression.rs

@@ -14,8 +14,14 @@ use std::num::NonZeroU8;
 
 fn binary_precedence(token: TokenKind) -> Option<NonZeroU8> {
     Some(match token {
+        // `to`/`downto` were a separate `range` production in the LRM; folding
+        // them into the operator table lets `range_constraint` accept any
+        // expression and reuse precedence climbing.
+        Keyword(Kw::To | Kw::Downto) => nonzero!(1u8),
         Keyword(Kw::And | Kw::Or | Kw::Nand | Kw::Nor | Kw::Xor | Kw::Xnor) => nonzero!(2u8),
-        EQ | NE | LT | LTE | GT | GTE | QueEQ | QueNE | QueLT | QueGT | QueGTE => nonzero!(3u8),
+        EQ | NE | LT | LTE | GT | GTE | QueEQ | QueNE | QueLT | QueLTE | QueGT | QueGTE => {
+            nonzero!(3u8)
+        }
         Keyword(Kw::Sll | Kw::Srl | Kw::Sla | Kw::Sra | Kw::Rol | Kw::Ror) => nonzero!(4u8),
         Plus | Minus | Concat => nonzero!(5u8),
         Times | Div | Keyword(Kw::Mod | Kw::Rem) => nonzero!(7u8),
@@ -39,9 +45,9 @@ impl Parser {
     pub fn primary(&mut self) {
         match_next_token!(self,
             Identifier, LtLt => {
-              self.start_node(NameExpression);
+              let checkpoint = self.checkpoint();
               self.name();
-              self.end_node();
+              self.continue_primary_after_name(checkpoint);
             },
             BitStringLiteral, CharacterLiteral, StringLiteral, Keyword(Kw::Null) => self.skip_into_node(LiteralExpression),
             AbstractLiteral => {
@@ -56,20 +62,43 @@ impl Parser {
                 self.end_node();
             },
             LeftPar => {
-                self.start_node(ParenthesizedExpressionOrAggregate);
-                self.aggregate_inner();
-                self.end_node();
+                self.parenthesized_expression_or_aggregate();
             },
             Keyword(Kw::New) => {
               self.allocator();
             }
         );
     }
 
+    pub(crate) fn parenthesized_expression_or_aggregate(&mut self) {
+        self.start_node(ParenthesizedExpressionOrAggregate);
+        self.aggregate_inner();
+        self.end_node();
+    }
+
+    /// Finalize a primary whose leading `Name` was already parsed and starts
+    /// at `checkpoint`. If a `Tick` follows, the name is the type mark of a
+    /// `QualifiedExpression` and the `'(…)` is consumed here; otherwise the
+    /// name is wrapped in `NameExpression`. Callers that need to continue
+    /// with binary operators should follow up with `expression_from_primary`.
+    pub(crate) fn continue_primary_after_name(
+        &mut self,
+        checkpoint: crate::parser::builder::Checkpoint,
+    ) {
+        if self.next_is(Tick) {
+            self.start_node_at(checkpoint, QualifiedExpression);
+            self.skip();
+            self.parenthesized_expression_or_aggregate();
+        } else {
+            self.start_node_at(checkpoint, NameExpression);
+        }
+        self.end_node();
+    }
+
     pub fn allocator(&mut self) {
-        self.start_node(ExpressionAllocator);
+        self.start_node(Allocator);
         self.expect_kw(Kw::New);
-        self.subtype_indication();
+        self.expression();
         self.end_node();
     }
 
@@ -105,6 +134,22 @@ impl Parser {
         self.expression_inner(0);
     }
 
+    /// Continue an expression parse from an already-emitted primary located at
+    /// `checkpoint`. The caller is responsible for having emitted the leading
+    /// primary node (e.g. `NameExpression`) starting at `checkpoint`.
+    pub(crate) fn expression_from_primary(
+        &mut self,
+        checkpoint: crate::parser::builder::Checkpoint,
+    ) {
+        while let Some(precedence) = binary_precedence(self.peek_token()) {
+            let precedence: u8 = precedence.into();
+            self.start_node_at(checkpoint, BinaryExpression);
+            self.skip();
+            self.expression_inner(precedence);
+            self.end_node();
+        }
+    }
+
     pub fn condition(&mut self) {
         self.expression()
     }