vhdl_syntax: Enable parsing under different VHDL standards and add standards API (#444)

* Introduce VHDL standard API
* Parse keywords selectively based on the tokenizer standard
* Add VHDL standard to the Parser and add toy VHDL2019 example

Author: Schottkyc137

vhdl_syntax/examples/source_refactoring.rs

@@ -14,7 +14,6 @@
 //
 // Copyright (c)  2025, Lukas Scheller lukasscheller@icloud.com
 use vhdl_syntax::parser;
-use vhdl_syntax::parser::Parser;
 use vhdl_syntax::syntax::node::SyntaxElement;
 use vhdl_syntax::syntax::rewrite::RewriteAction;
 use vhdl_syntax::syntax::AstNode;
@@ -38,22 +37,22 @@ end foobar;
         "Did not expect diagnostics for correct VHDL"
     );
 
-    // The target
-    // NOTE: Usage of the parser API will be significantly changed or removed in favor of a better alternative in a future version.
-    let mut parser = Parser::new(
+    // The target: parse a full design file and take the entity declaration node.
+    let (replacement_file, diagnostics) = parser::parse(
         "\
 entity no_longer_foo is
 end no_longer_foo;
 
-"
-        .into(),
+",
     );
-    parser.entity_declaration();
-    let (replacement_entity, diagnostics) = parser.into_root();
     assert!(
         diagnostics.is_empty(),
         "Did not expect diagnostics for correct VHDL"
     );
+    let replacement_entity = replacement_file
+        .design_units()
+        .next()
+        .expect("expected entity declaration in replacement file");
 
     let new_file = file.raw().rewrite(|node| match node {
         SyntaxElement::Node(node) => match EntityDeclarationSyntax::cast(node.clone()) {
@@ -64,7 +63,7 @@ end no_longer_foo;
                     .and_then(|preamble| preamble.name_token())
                     .is_some_and(|tok| tok.text() == "foo") =>
             {
-                RewriteAction::Change(SyntaxElement::Node(replacement_entity.clone()))
+                RewriteAction::Change(SyntaxElement::Node(replacement_entity.raw()))
             }
             // If the syntax node is not an entity, or the name of the entity is not 'foo', leave the node as-is
             _ => RewriteAction::Leave,

vhdl_syntax/src/lib.rs

@@ -9,6 +9,7 @@ pub mod latin_1;
 pub mod parser;
 #[cfg(feature = "serde")]
 pub mod serde;
+pub mod standard;
 pub mod syntax;
 mod token_interning;
 pub mod tokens;

vhdl_syntax/src/parser/mod.rs

@@ -4,9 +4,11 @@
 // You can obtain one at http://mozilla.org/MPL/2.0/.
 //
 // Copyright (c)  2024, Lukas Scheller lukasscheller@icloud.com
+use crate::standard::VHDLStandard;
 use crate::syntax::node::SyntaxNode;
 use crate::syntax::{DesignFileSyntax, NodeKind};
 use crate::tokens::TokenStream;
+use crate::tokens::Tokenizer;
 
 mod builder;
 pub mod diagnostics;
@@ -26,29 +28,50 @@ pub struct Parser {
     builder: builder::NodeBuilder,
     diagnostics: Vec<diagnostics::ParserDiagnostic>,
     unexpected_eof: bool,
+    standard: VHDLStandard,
 }
 
 impl Parser {
-    pub fn new(token_stream: TokenStream) -> Parser {
+    pub(crate) fn new(token_stream: TokenStream, standard: VHDLStandard) -> Parser {
         Parser {
             token_stream,
             builder: builder::NodeBuilder::new(),
             diagnostics: Vec::default(),
             unexpected_eof: false,
+            standard,
         }
     }
 
+    pub fn standard(&self) -> VHDLStandard {
+        self.standard
+    }
+
     pub fn into_root(self) -> (SyntaxNode, Vec<diagnostics::ParserDiagnostic>) {
         let (green, diagnostics) = self.end();
         (SyntaxNode::new_root(green), diagnostics)
     }
 }
 
-/// Parse and return a VHDL file.
+/// Parse and return a VHDL file using the default VHDL standard.
+///
+/// Use [`parse_with_standard`] to use a non-default VHDL standard.
 pub fn parse(
     token_stream: impl Into<TokenStream>,
 ) -> (DesignFileSyntax, Vec<diagnostics::ParserDiagnostic>) {
-    let mut parser: Parser = Parser::new(token_stream.into());
+    let mut parser = Parser::new(token_stream.into(), VHDLStandard::default());
+    parser.design_file();
+    let (syntax_node, diagnostics) = parser.into_root();
+    debug_assert!(syntax_node.kind() == NodeKind::DesignFile);
+    (DesignFileSyntax(syntax_node), diagnostics)
+}
+
+/// Parse and return a VHDL file, tokenizing and parsing under the given `standard`.
+pub fn parse_with_standard(
+    standard: VHDLStandard,
+    input: impl IntoIterator<Item = u8>,
+) -> (DesignFileSyntax, Vec<diagnostics::ParserDiagnostic>) {
+    let token_stream: TokenStream = Tokenizer::with_standard(standard, input.into_iter()).collect();
+    let mut parser = Parser::new(token_stream, standard);
     parser.design_file();
     let (syntax_node, diagnostics) = parser.into_root();
     debug_assert!(syntax_node.kind() == NodeKind::DesignFile);
@@ -60,7 +83,20 @@ pub(crate) fn parse_syntax(
     token_stream: impl Into<TokenStream>,
     parser_fn: impl FnOnce(&mut Parser),
 ) -> (SyntaxNode, Vec<diagnostics::ParserDiagnostic>) {
-    let mut parser: Parser = Parser::new(token_stream.into());
+    let mut parser = Parser::new(token_stream.into(), VHDLStandard::default());
+    parser_fn(&mut parser);
+    let (green, diagnostics) = parser.end();
+    (SyntaxNode::new_root(green), diagnostics)
+}
+
+#[cfg(test)]
+pub(crate) fn parse_syntax_with_standard(
+    standard: VHDLStandard,
+    input: impl IntoIterator<Item = u8>,
+    parser_fn: impl FnOnce(&mut Parser),
+) -> (SyntaxNode, Vec<diagnostics::ParserDiagnostic>) {
+    let token_stream: TokenStream = Tokenizer::with_standard(standard, input.into_iter()).collect();
+    let mut parser = Parser::new(token_stream, standard);
     parser_fn(&mut parser);
     let (green, diagnostics) = parser.end();
     (SyntaxNode::new_root(green), diagnostics)

vhdl_syntax/src/parser/productions/component_declaration.rs

@@ -5,6 +5,7 @@
 // Copyright (c)  2025, Lukas Scheller lukasscheller@icloud.com
 
 use crate::parser::Parser;
+use crate::standard::VHDLStandard;
 use crate::syntax::node_kind::NodeKind::*;
 use crate::tokens::token_kind::Keyword as Kw;
 use crate::tokens::TokenKind::*;
@@ -31,7 +32,12 @@ impl Parser {
 
     pub fn component_declaration_epilogue(&mut self) {
         self.start_node(ComponentDeclarationEpilogue);
-        self.expect_tokens([Keyword(Kw::End), Keyword(Kw::Component)]);
+        self.expect_token(Keyword(Kw::End));
+        if self.standard().is_at_least(VHDLStandard::VHDL2019) {
+            self.opt_token(Keyword(Kw::Component));
+        } else {
+            self.expect_token(Keyword(Kw::Component));
+        }
         self.opt_identifier();
         self.expect_token(SemiColon);
         self.end_node();
@@ -40,8 +46,9 @@ impl Parser {
 
 #[cfg(test)]
 mod tests {
-    use crate::parser::test_utils::to_test_text;
-    use crate::parser::Parser;
+    use crate::parser::test_utils::{to_test_text, to_test_text_with_standard};
+    use crate::parser::{parse_syntax_with_standard, Parser};
+    use crate::standard::VHDLStandard;
 
     #[test]
     fn simple_components() {
@@ -95,4 +102,40 @@ end component;
 ",
         ));
     }
+
+    #[test]
+    fn component_end_without_keyword_vhdl2019() {
+        // `end;` without the trailing `component` keyword is valid from VHDL-2019 onwards.
+        insta::assert_snapshot!(to_test_text_with_standard(
+            VHDLStandard::VHDL2019,
+            Parser::component_declaration,
+            "\
+component foo is
+end;
+"
+        ));
+        // Optional identifier after `end` is also accepted.
+        insta::assert_snapshot!(to_test_text_with_standard(
+            VHDLStandard::VHDL2019,
+            Parser::component_declaration,
+            "\
+component foo is
+end foo;
+"
+        ));
+    }
+
+    #[test]
+    fn component_end_without_keyword_is_error_before_vhdl2019() {
+        // Omitting the trailing `component` keyword must produce a diagnostic under VHDL-2008.
+        let (_, diagnostics) = parse_syntax_with_standard(
+            VHDLStandard::VHDL2008,
+            "component foo is\nend;\n".bytes(),
+            Parser::component_declaration,
+        );
+        assert!(
+            !diagnostics.is_empty(),
+            "expected a diagnostic for missing 'component' keyword under VHDL-2008"
+        );
+    }
 }

vhdl_syntax/src/parser/productions/snapshots/vhdl_syntax__parser__productions__component_declaration__tests__component_end_without_keyword_vhdl2019-2.snap

@@ -0,0 +1,13 @@
+---
+source: vhdl_syntax/src/parser/productions/component_declaration.rs
+expression: "to_test_text_with_standard(VHDLStandard::VHDL2019,\nParser::component_declaration, \"\\\ncomponent foo is\nend foo;\n\")"
+---
+ComponentDeclaration
+  ComponentDeclarationPreamble
+    Keyword(Component)
+    Identifier 'foo'
+    Keyword(Is)
+  ComponentDeclarationEpilogue
+    Keyword(End)
+    Identifier 'foo'
+    SemiColon

vhdl_syntax/src/parser/productions/snapshots/vhdl_syntax__parser__productions__component_declaration__tests__component_end_without_keyword_vhdl2019.snap

@@ -0,0 +1,12 @@
+---
+source: vhdl_syntax/src/parser/productions/component_declaration.rs
+expression: "to_test_text_with_standard(VHDLStandard::VHDL2019,\nParser::component_declaration, \"\\\ncomponent foo is\nend;\n\")"
+---
+ComponentDeclaration
+  ComponentDeclarationPreamble
+    Keyword(Component)
+    Identifier 'foo'
+    Keyword(Is)
+  ComponentDeclarationEpilogue
+    Keyword(End)
+    SemiColon

vhdl_syntax/src/parser/test_utils.rs

@@ -4,11 +4,23 @@
 //
 // Copyright (c)  2025, Lukas Scheller lukasscheller@icloud.com
 
-use crate::parser::{parse_syntax, Parser};
+use crate::parser::{parse_syntax, parse_syntax_with_standard, Parser};
+use crate::standard::VHDLStandard;
 
 /// Returns the AST text for snapshot assertions.
 pub fn to_test_text(func: impl FnOnce(&mut Parser), input: &str) -> String {
     let (entity, diagnostics) = parse_syntax(input, func);
     assert!(diagnostics.is_empty(), "got diagnostics: {:?}", diagnostics);
     entity.test_text()
 }
+
+/// Returns the AST text for snapshot assertions, tokenizing and parsing under `standard`.
+pub fn to_test_text_with_standard(
+    standard: VHDLStandard,
+    func: impl FnOnce(&mut Parser),
+    input: &str,
+) -> String {
+    let (entity, diagnostics) = parse_syntax_with_standard(standard, input.bytes(), func);
+    assert!(diagnostics.is_empty(), "got diagnostics: {:?}", diagnostics);
+    entity.test_text()
+}

vhdl_syntax/src/standard.rs

@@ -0,0 +1,125 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at http://mozilla.org/MPL/2.0/.
+//
+// Copyright (c)  2024, Lukas Scheller lukasscheller@icloud.com
+
+//! VHDL standard version identification and comparison.
+//!
+//! This module provides the [`VHDLStandard`] enum to represent all supported VHDL versions
+//! and methods to work with standards.
+
+use std::fmt;
+
+/// VHDL standard version.
+///
+/// Represents different VHDL versions.
+/// As default, VHDL2008 is chosen.
+#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, Default)]
+#[non_exhaustive]
+pub enum VHDLStandard {
+    VHDL1987,
+    VHDL1993,
+    VHDL2000,
+    VHDL2002,
+    #[default]
+    VHDL2008,
+    VHDL2019,
+}
+
+impl VHDLStandard {
+    /// All supported VHDL standards in order.
+    pub const ALL: [Self; 6] = [
+        Self::VHDL1987,
+        Self::VHDL1993,
+        Self::VHDL2000,
+        Self::VHDL2002,
+        Self::VHDL2008,
+        Self::VHDL2019,
+    ];
+
+    /// Returns the standard as a string.
+    pub const fn as_str(self) -> &'static str {
+        match self {
+            Self::VHDL1987 => "1987",
+            Self::VHDL1993 => "1993",
+            Self::VHDL2000 => "2000",
+            Self::VHDL2002 => "2002",
+            Self::VHDL2008 => "2008",
+            Self::VHDL2019 => "2019",
+        }
+    }
+
+    /// Returns the year of the standard.
+    pub const fn year(self) -> u16 {
+        match self {
+            Self::VHDL1987 => 1987,
+            Self::VHDL1993 => 1993,
+            Self::VHDL2000 => 2000,
+            Self::VHDL2002 => 2002,
+            Self::VHDL2008 => 2008,
+            Self::VHDL2019 => 2019,
+        }
+    }
+
+    /// Returns true if this standard is at or newer than `other`.
+    pub fn is_at_least(self, other: Self) -> bool {
+        self >= other
+    }
+
+    /// Returns true if this standard is older than `other`.
+    pub fn is_before(self, other: Self) -> bool {
+        self < other
+    }
+
+    /// Returns the latest supported VHDL standard.
+    pub fn latest() -> VHDLStandard {
+        Self::VHDL2019
+    }
+}
+
+#[test]
+fn order_of_standards() {
+    assert!(VHDLStandard::VHDL2008 > VHDLStandard::VHDL1993);
+}
+
+/// Error returned when parsing an invalid VHDL standard string.
+#[derive(Debug, Clone)]
+pub struct ParseVHDLStandardError;
+
+impl std::fmt::Display for ParseVHDLStandardError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "invalid VHDL standard")
+    }
+}
+
+impl std::error::Error for ParseVHDLStandardError {}
+
+impl TryFrom<&str> for VHDLStandard {
+    type Error = ParseVHDLStandardError;
+
+    fn try_from(value: &str) -> Result<Self, Self::Error> {
+        use VHDLStandard::*;
+        Ok(match value {
+            "1987" | "87" => VHDL1987,
+            "1993" | "93" => VHDL1993,
+            "2000" | "00" => VHDL2000,
+            "2002" | "02" => VHDL2002,
+            "2008" | "08" => VHDL2008,
+            "2019" | "19" => VHDL2019,
+            _ => return Err(ParseVHDLStandardError),
+        })
+    }
+}
+
+impl AsRef<str> for VHDLStandard {
+    fn as_ref(&self) -> &str {
+        self.as_str()
+    }
+}
+
+impl fmt::Display for VHDLStandard {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.write_str(self.as_str())
+    }
+}