[vhdl-syntax] refactor leading and trailing trivia into single leading trivia

Author: Schottkyc137

Cargo.lock

@@ -39,6 +39,7 @@ ContextItem: !Choice
 DesignFile: !Sequence
   - node: DesignUnit
     repeated: true
+  - token: Eof
 
 DesignUnit: !Sequence
   - node: ContextClause

ast_syntax_gen/src/syntax_definitions/design_units.yaml

@@ -72,6 +72,8 @@ pub enum TokenKind {
     ///
     /// Produced, for example, when there is an unknown char or illegal bit string
     Unknown,
+
+    Eof
 }
 
 impl TokenKind {

ast_syntax_gen/src/token.rs

@@ -17,6 +17,7 @@ serde = { version = "1.0.228", optional = true, features = ["derive"]}
 assert_matches = "1.5.0"
 pretty_assertions = "1.4.1"
 insta = { version = "1.36", features = ["redactions"] }
+similar = { version = "2.7.0", features = ["bytes"] }
 
 [features]
 serde = ["dep:serde"]

vhdl_syntax/Cargo.toml

@@ -41,7 +41,7 @@ end bar;
         if let Some(token) = entity.raw().first_token() {
             // The trivia is where all auxiliary information concerning a token is stored,
             // for example, comments, whitespaces or newlines.
-            let comment = extract_doc_from_trivia(token.all_leading_trivia());
+            let comment = extract_doc_from_trivia(token.leading_trivia());
             // If the entity has a name token, add the extracted documentation to the map
             if let Some(ident) = entity
                 .entity_declaration_preamble()
@@ -62,7 +62,7 @@ end bar;
 /// Note that this is a toy example that simply checks for line comments that with a `-` (i.e.,
 /// a doc comment is written using the `--- doc comment` syntax. A real-world example needs to
 /// include more sophisticated processing.
-fn extract_doc_from_trivia(trivia: Trivia) -> String {
+fn extract_doc_from_trivia(trivia: &Trivia) -> String {
     trivia
         .iter()
         .filter_map(|piece| match piece {

vhdl_syntax/examples/doc_extraction.rs

@@ -69,7 +69,6 @@ impl ToLatin1 for Token {
         let mut buf = Latin1String::new();
         buf.append(&mut self.leading_trivia().to_latin1());
         buf.extend(self.text());
-        buf.append(&mut self.trailing_trivia().to_latin1());
         buf
     }
 }

vhdl_syntax/src/fmt/latin1.rs

@@ -68,7 +68,6 @@ impl fmt::Display for Token {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         write!(f, "{}", self.leading_trivia())?;
         write!(f, "{}", self.text())?;
-        write!(f, "{}", self.trailing_trivia())?;
         Ok(())
     }
 }

vhdl_syntax/src/fmt/utf8.rs

@@ -70,7 +70,7 @@ impl Parser {
     pub fn entity_designator(&mut self) {
         self.start_node(EntityDesignator);
         self.entity_tag();
-        if self.peek_token() == Some(LeftSquare) {
+        if self.peek_token() == LeftSquare {
             self.signature();
         }
         self.end_node();

vhdl_syntax/src/parser/productions/attributes.rs

@@ -69,7 +69,7 @@ impl Parser {
         self.start_node(ConcurrentStatements);
         loop {
             match self.peek_token() {
-                Some(Keyword(Kw::End | Kw::Elsif | Kw::Else | Kw::When)) | None => {
+                Keyword(Kw::End | Kw::Elsif | Kw::Else | Kw::When) | Eof => {
                     break;
                 }
                 _ => self.concurrent_statement(),
@@ -99,7 +99,7 @@ impl Parser {
         self.end_node();
     }
 
-    fn peek_concurrent_statement_kind(&mut self) -> Option<TokenKind> {
+    fn peek_concurrent_statement_kind(&mut self) -> TokenKind {
         // Has label?
         let mut peek_idx = 0usize;
         if self.next_is(Identifier) && self.next_nth_is(Colon, 1) {
@@ -114,8 +114,8 @@ impl Parser {
 
     pub fn instantiated_unit(&mut self) {
         match self.peek_token() {
-            Some(Keyword(Kw::Entity)) => self.entity_instantiated_unit(),
-            Some(Keyword(Kw::Configuration)) => self.configuration_instantiated_unit(),
+            Keyword(Kw::Entity) => self.entity_instantiated_unit(),
+            Keyword(Kw::Configuration) => self.configuration_instantiated_unit(),
             _ => self.component_instantiated_unit(),
         }
     }
@@ -140,24 +140,24 @@ impl Parser {
 
     pub(crate) fn concurrent_statement(&mut self) {
         match self.peek_concurrent_statement_kind() {
-            Some(Keyword(Kw::Block)) => self.block_statement(),
-            Some(Keyword(Kw::Process)) => self.process_statement(),
-            Some(Keyword(Kw::Component | Kw::Configuration | Kw::Entity)) => {
+            Keyword(Kw::Block) => self.block_statement(),
+            Keyword(Kw::Process) => self.process_statement(),
+            Keyword(Kw::Component | Kw::Configuration | Kw::Entity) => {
                 self.component_instantiation_statement()
             }
-            Some(Keyword(Kw::For)) => self.for_generate_statement(),
-            Some(Keyword(Kw::If)) => self.if_generate_statement(),
-            Some(Keyword(Kw::Case)) => self.case_generate_statement(),
-            Some(Keyword(Kw::Assert)) => self.concurrent_assertion_statement(),
-            Some(Keyword(Kw::With)) => self.concurrent_selected_signal_assignment(),
-            Some(Identifier | LtLt | StringLiteral | CharacterLiteral) => {
+            Keyword(Kw::For) => self.for_generate_statement(),
+            Keyword(Kw::If) => self.if_generate_statement(),
+            Keyword(Kw::Case) => self.case_generate_statement(),
+            Keyword(Kw::Assert) => self.concurrent_assertion_statement(),
+            Keyword(Kw::With) => self.concurrent_selected_signal_assignment(),
+            Identifier | LtLt | StringLiteral | CharacterLiteral => {
                 let checkpoint = self.checkpoint();
                 self.opt_label();
                 self.opt_token(Keyword(Kw::Postponed));
                 let checkpoint2 = self.checkpoint();
                 self.name();
                 match self.peek_token() {
-                    Some(LTE) => {
+                    LTE => {
                         self.start_node_at(checkpoint2, NameTarget);
                         self.end_node();
                         self.skip();
@@ -174,7 +174,7 @@ impl Parser {
                             self.start_node_at(checkpoint, ConcurrentSimpleSignalAssignment);
                         }
                     }
-                    Some(Keyword(Kw::Port | Kw::Generic)) => {
+                    Keyword(Kw::Port | Kw::Generic) => {
                         self.start_node_at(checkpoint2, ComponentInstantiatedUnit);
                         self.end_node();
                         self.start_node_at(checkpoint, ComponentInstantiationStatement);

vhdl_syntax/src/parser/productions/concurrent_statement.rs

@@ -101,7 +101,7 @@ impl Parser {
 
     fn configuration_item_known_keyword(&mut self, item_checkpoint: Checkpoint) {
         match self.peek_token() {
-            Some(tok @ Keyword(Kw::All | Kw::Others)) => {
+            tok @ Keyword(Kw::All | Kw::Others) => {
                 self.start_node_at(item_checkpoint, NodeKind::ComponentConfiguration);
                 self.start_node_at(item_checkpoint, NodeKind::ComponentConfigurationPreamble);
                 self.start_node(NodeKind::ComponentSpecification);
@@ -119,7 +119,7 @@ impl Parser {
                 self.component_configuration_known_spec();
                 self.end_node();
             }
-            Some(Identifier) => {
+            Identifier => {
                 if self.next_nth_is(Comma, 1) {
                     self.start_node_at(item_checkpoint, NodeKind::ComponentConfiguration);
                     self.start_node_at(item_checkpoint, NodeKind::ComponentConfigurationPreamble);
@@ -137,7 +137,7 @@ impl Parser {
                     let checkpoint = self.checkpoint();
                     self.name();
                     match self.peek_token() {
-                        Some(Colon) => {
+                        Colon => {
                             self.start_node_at(item_checkpoint, NodeKind::ComponentConfiguration);
                             self.start_node_at(
                                 item_checkpoint,