Merge branch 'master' into codee

Author: daniel-otero

codee/patches/0004-Unhide-end_of_statement.patch

@@ -8,11 +8,11 @@ Subject: Unhide `end_of_statement`
  test/corpus/constructs.txt         | 482 +++++++++++-----
  test/corpus/cudafortran.txt        |  26 +-
  test/corpus/expressions.txt        | 262 +++++++--
- test/corpus/line_continuations.txt |  86 ++-
+ test/corpus/line_continuations.txt |  93 +++-
  test/corpus/preprocessor.txt       | 259 ++++++---
  test/corpus/regressions.txt        |  68 ++-
  test/corpus/statements.txt         | 846 +++++++++++++++++++++++------
- 8 files changed, 1606 insertions(+), 497 deletions(-)
+ 8 files changed, 1611 insertions(+), 499 deletions(-)
 
 diff --git a/grammar.js b/grammar.js
 index 1289710..6d39d36 100644
@@ -2832,7 +2832,7 @@ index cf7f6c4..8545f3d 100644
 +    (end_program_statement
 +      (end_of_statement))))
 diff --git a/test/corpus/line_continuations.txt b/test/corpus/line_continuations.txt
-index c30265a..a3ebedb 100644
+index 952ef1e..5e64143 100644
 --- a/test/corpus/line_continuations.txt
 +++ b/test/corpus/line_continuations.txt
 @@ -13,7 +13,8 @@ end program
@@ -3080,14 +3080,34 @@ index c30265a..a3ebedb 100644
      (variable_declaration
        (intrinsic_type)
        (init_declarator
-@@ -300,5 +340,7 @@ end program test
+@@ -300,8 +340,10 @@ end program test
            (number_literal)
            (number_literal)
            (number_literal))))
 +    (end_of_statement)
      (end_program_statement
 -      (name))))
 +      (name)
++      (end_of_statement))))
+ 
+ ================================================================================
+ Line continuation as part of string
+@@ -316,11 +358,14 @@ end program main
+ (translation_unit
+   (program
+     (program_statement
+-      (name))
++      (name)
++      (end_of_statement))
+     (comment)
+     (print_statement
+       (format_identifier)
+       (output_item_list
+         (string_literal)))
++    (end_of_statement)
+     (end_program_statement
+-      (name))))
++      (name)
 +      (end_of_statement))))
 diff --git a/test/corpus/preprocessor.txt b/test/corpus/preprocessor.txt
 index a42ff39..5f3811a 100644

codee/patches/0005-Consume-end_of_statement.patch

@@ -13,7 +13,7 @@ they can also be consumed as `comment` tokens).
  1 file changed, 4 insertions(+), 4 deletions(-)
 
 diff --git a/src/scanner.c b/src/scanner.c
-index 0b4caef..10a22d7 100644
+index 4a735f7..be574ae 100644
 --- a/src/scanner.c
 +++ b/src/scanner.c
 @@ -226,7 +226,7 @@ static bool scan_end_of_statement(Scanner *scanner, TSLexer *lexer) {

codee/patches/0013-Stop-parsing-continuation-lines-in-number-literals.patch

@@ -14,7 +14,7 @@ explicitly.
  3 files changed, 58 insertions(+), 39 deletions(-)
 
 diff --git a/src/scanner.c b/src/scanner.c
-index 10a22d7..b768d99 100644
+index be574ae..9007f34 100644
 --- a/src/scanner.c
 +++ b/src/scanner.c
 @@ -19,6 +19,16 @@ typedef struct {
@@ -123,7 +123,7 @@ index 10a22d7..b768d99 100644
              }
              lexer->result_symbol = FLOAT_LITERAL;
          }
-@@ -432,8 +444,13 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
+@@ -433,8 +445,13 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
      if (valid_symbols[INTEGER_LITERAL] || valid_symbols[FLOAT_LITERAL] ||
          valid_symbols[BOZ_LITERAL]) {
          // extract out root number from expression
@@ -183,7 +183,7 @@ index 7316d3b..1b23f9a 100644
      (end_of_statement)
      (end_program_statement
 diff --git a/test/corpus/line_continuations.txt b/test/corpus/line_continuations.txt
-index a3ebedb..c086fe8 100644
+index 5e64143..fbf4084 100644
 --- a/test/corpus/line_continuations.txt
 +++ b/test/corpus/line_continuations.txt
 @@ -339,7 +339,8 @@ end program test

codee/patches/0016-Allow-to-parse-macro-identifiers.patch

@@ -45,7 +45,7 @@ index d40361e..9685d86 100644
 
      parenthesized_expression: $ => seq(
 diff --git a/src/scanner.c b/src/scanner.c
-index b768d99..e5a37ee 100644
+index 9007f34..2c348ff 100644
 --- a/src/scanner.c
 +++ b/src/scanner.c
 @@ -1,4 +1,5 @@
@@ -133,7 +133,7 @@ index b768d99..e5a37ee 100644
    return true;
  }
 
-@@ -393,6 +413,33 @@ static bool scan_string_literal(TSLexer *lexer) {
+@@ -394,6 +414,33 @@ static bool scan_string_literal(TSLexer *lexer) {
      return false;
  }
 
@@ -167,7 +167,7 @@ index b768d99..e5a37ee 100644
  /// Need an external scanner to catch '!' before its parsed as a comment
  static bool scan_preproc_unary_operator(TSLexer *lexer) {
    const char next_char = lexer->lookahead;
-@@ -467,19 +514,57 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
+@@ -468,19 +515,57 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
          return true;
      }
 
@@ -229,7 +229,7 @@ index b768d99..e5a37ee 100644
  }
 
  bool tree_sitter_fortran_external_scanner_scan(void *payload, TSLexer *lexer,
-@@ -491,8 +576,9 @@ bool tree_sitter_fortran_external_scanner_scan(void *payload, TSLexer *lexer,
+@@ -492,8 +577,9 @@ bool tree_sitter_fortran_external_scanner_scan(void *payload, TSLexer *lexer,
  unsigned tree_sitter_fortran_external_scanner_serialize(void *payload,
                                                          char *buffer) {
      Scanner *scanner = (Scanner *)payload;
@@ -241,7 +241,7 @@ index b768d99..e5a37ee 100644
  }
 
  void tree_sitter_fortran_external_scanner_deserialize(void *payload,
-@@ -500,11 +586,18 @@ void tree_sitter_fortran_external_scanner_deserialize(void *payload,
+@@ -501,11 +587,18 @@ void tree_sitter_fortran_external_scanner_deserialize(void *payload,
                                                        unsigned length) {
      Scanner *scanner = (Scanner *)payload;
      if (length > 0) {

codee/patches/0017-Unhide-_string_literal.patch

@@ -7,11 +7,11 @@ Subject: Unhide `_string_literal`
  grammar.js                         |  15 +--
  test/corpus/constructs.txt         |  24 +++--
  test/corpus/expressions.txt        | 136 +++++++++++++++++++++++-----
- test/corpus/line_continuations.txt |  33 ++++---
+ test/corpus/line_continuations.txt |  36 +++++---
  test/corpus/preprocessor.txt       |   9 +-
  test/corpus/regressions.txt        |  33 ++++---
  test/corpus/statements.txt         | 141 +++++++++++++++++++----------
- 7 files changed, 282 insertions(+), 109 deletions(-)
+ 7 files changed, 284 insertions(+), 110 deletions(-)
 
 diff --git a/grammar.js b/grammar.js
 index 9685d86..bdfa3d6 100644
@@ -395,7 +395,7 @@ index 29eacb4..870a02e 100644
 +    (end_subroutine_statement
 +      (end_of_statement))))
 diff --git a/test/corpus/line_continuations.txt b/test/corpus/line_continuations.txt
-index c086fe8..ba12587 100644
+index fbf4084..caa7bf2 100644
 --- a/test/corpus/line_continuations.txt
 +++ b/test/corpus/line_continuations.txt
 @@ -18,10 +18,12 @@ end program
@@ -474,6 +474,16 @@ index c086fe8..ba12587 100644
      (end_of_statement)
      (end_program_statement
        (end_of_statement))))
+@@ -365,7 +376,8 @@ end program main
+     (print_statement
+       (format_identifier)
+       (output_item_list
+-        (string_literal)))
++        (prefixed_string_literal
++          (string_literal))))
+     (end_of_statement)
+     (end_program_statement
+       (name)
 diff --git a/test/corpus/preprocessor.txt b/test/corpus/preprocessor.txt
 index b0b35bd..ea589be 100644
 --- a/test/corpus/preprocessor.txt

codee/patches/0019-Parse-whitespaces.patch

@@ -9,11 +9,11 @@ Subject: Parse whitespaces
  test/corpus/constructs.txt         |  810 +++++++++++++-
  test/corpus/cudafortran.txt        |   39 +-
  test/corpus/expressions.txt        |  675 ++++++++++-
- test/corpus/line_continuations.txt |  117 +-
+ test/corpus/line_continuations.txt |  124 ++-
  test/corpus/preprocessor.txt       |  339 +++++-
  test/corpus/regressions.txt        |  140 ++-
  test/corpus/statements.txt         | 1660 +++++++++++++++++++++++++++-
- 9 files changed, 3744 insertions(+), 83 deletions(-)
+ 9 files changed, 3751 insertions(+), 83 deletions(-)
 
 diff --git a/grammar.js b/grammar.js
 index c8ce7d9..a546930 100644
@@ -52,7 +52,7 @@ index c8ce7d9..a546930 100644
        $._external_end_of_statement,
      ),
 diff --git a/src/scanner.c b/src/scanner.c
-index e5a37ee..5fabf79 100644
+index 2c348ff..fc1b6bd 100644
 --- a/src/scanner.c
 +++ b/src/scanner.c
 @@ -15,6 +15,7 @@ enum TokenType {
@@ -63,7 +63,7 @@ index e5a37ee..5fabf79 100644
  };
 
  typedef Array(char *) StringArray;
-@@ -451,10 +452,28 @@ static bool scan_preproc_unary_operator(TSLexer *lexer) {
+@@ -452,10 +453,28 @@ static bool scan_preproc_unary_operator(TSLexer *lexer) {
    return false;
  }
 
@@ -95,7 +95,7 @@ index e5a37ee..5fabf79 100644
      }
 
      // Close the current statement if we can
-@@ -468,6 +487,16 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
+@@ -469,6 +488,16 @@ static bool scan(Scanner *scanner, TSLexer *lexer, const bool *valid_symbols) {
      // statements, so we should eat all whitespace including
      // newlines, until we come to something more interesting
      while (iswspace(lexer->lookahead)) {
@@ -4437,7 +4437,7 @@ index 870a02e..c4b4bfe 100644
          (number_literal)
          (string_literal)))
 diff --git a/test/corpus/line_continuations.txt b/test/corpus/line_continuations.txt
-index ba12587..32739bf 100644
+index caa7bf2..440f58e 100644
 --- a/test/corpus/line_continuations.txt
 +++ b/test/corpus/line_continuations.txt
 @@ -13,23 +13,34 @@ end program
@@ -4760,7 +4760,7 @@ index ba12587..32739bf 100644
        (end_of_statement))))
 
  ================================================================================
-@@ -338,21 +437,35 @@ end program test
+@@ -338,22 +437,36 @@ end program test
  (translation_unit
    (program
      (program_statement
@@ -4794,6 +4794,30 @@ index ba12587..32739bf 100644
      (end_of_statement)
      (end_program_statement
 +      (whitespace)
++      (whitespace)
+       (name)
+       (end_of_statement))))
+ 
+@@ -370,15 +483,22 @@ end program main
+ (translation_unit
+   (program
+     (program_statement
++      (whitespace)
+       (name)
+       (end_of_statement))
++    (whitespace)
+     (comment)
++    (whitespace)
+     (print_statement
++      (whitespace)
+       (format_identifier)
++      (whitespace)
+       (output_item_list
+         (prefixed_string_literal
+           (string_literal))))
+     (end_of_statement)
+     (end_program_statement
++      (whitespace)
 +      (whitespace)
        (name)
        (end_of_statement))))

src/scanner.c

@@ -389,6 +389,7 @@ static bool scan_string_literal(TSLexer *lexer) {
                     advance(lexer);
                 }
             }
+            continue;
         }
 
         // If we hit the same kind of quote that opened this literal,

test/corpus/line_continuations.txt

@@ -469,3 +469,25 @@ end program test
       (whitespace)
       (name)
       (end_of_statement))))
+
+================================================================================
+Line continuation as part of string
+================================================================================
+program main
+  ! This should print "I like fish&chips"
+  print *, "I like fish&&
+    &chips"
+end program main
+--------------------------------------------------------------------------------
+
+(translation_unit
+  (program
+    (program_statement
+      (name))
+    (comment)
+    (print_statement
+      (format_identifier)
+      (output_item_list
+        (string_literal)))
+    (end_program_statement
+      (name))))