Bump Prism to v1.5.2

[Backport #21187]

Author: k0kubun

lib/prism/polyfill/warn.rb

@@ -7,35 +7,29 @@
   Kernel.prepend(
     Module.new {
       def warn(*msgs, uplevel: nil, category: nil) # :nodoc:
-        uplevel =
-          case uplevel
-          when nil
-            1
-          when Integer
-            uplevel + 1
-          else
-            uplevel.to_int + 1
-          end
-
-        super(*msgs, uplevel: uplevel)
+        case uplevel
+        when nil
+          super(*msgs)
+        when Integer
+          super(*msgs, uplevel: uplevel + 1)
+        else
+          super(*msgs, uplevel: uplevel.to_int + 1)
+        end
       end
     }
   )
 
   Object.prepend(
     Module.new {
       def warn(*msgs, uplevel: nil, category: nil) # :nodoc:
-        uplevel =
-          case uplevel
-          when nil
-            1
-          when Integer
-            uplevel + 1
-          else
-            uplevel.to_int + 1
-          end
-
-        super(*msgs, uplevel: uplevel)
+        case uplevel
+        when nil
+          super(*msgs)
+        when Integer
+          super(*msgs, uplevel: uplevel + 1)
+        else
+          super(*msgs, uplevel: uplevel.to_int + 1)
+        end
       end
     }
   )

lib/prism/prism.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |spec|
   spec.name = "prism"
-  spec.version = "1.5.1"
+  spec.version = "1.5.2"
   spec.authors = ["Shopify"]
   spec.email = ["ruby@shopify.com"]
 

lib/prism/translation/parser.rb

@@ -19,6 +19,13 @@ module Translation
     # whitequark/parser gem's syntax tree. It inherits from the base parser for
     # the parser gem, and overrides the parse* methods to parse with prism and
     # then translate.
+    #
+    # Note that this version of the parser always parses using the latest
+    # version of Ruby syntax supported by Prism. If you want specific version
+    # support, use one of the version-specific subclasses, such as
+    # `Prism::Translation::Parser34`. If you want to parse using the same
+    # version of Ruby syntax as the currently running version of Ruby, use
+    # `Prism::Translation::ParserCurrent`.
     class Parser < ::Parser::Base
       Diagnostic = ::Parser::Diagnostic # :nodoc:
       private_constant :Diagnostic
@@ -77,7 +84,7 @@ def initialize(builder = Prism::Translation::Parser::Builder.new, parser: Prism)
       end
 
       def version # :nodoc:
-        34
+        35
       end
 
       # The default encoding for Ruby files is UTF-8.

lib/prism/translation/ruby_parser.rb

@@ -152,7 +152,7 @@ def visit_assoc_splat_node(node)
         # ^^
         # ```
         def visit_back_reference_read_node(node)
-          s(node, :back_ref, node.name.name.delete_prefix("$").to_sym)
+          s(node, :back_ref, node.name.to_s.delete_prefix("$").to_sym)
         end
 
         # ```

prism/config.yml

@@ -60,6 +60,7 @@ errors:
   - CONDITIONAL_WHILE_PREDICATE
   - CONSTANT_PATH_COLON_COLON_CONSTANT
   - DEF_ENDLESS
+  - DEF_ENDLESS_PARAMETERS
   - DEF_ENDLESS_SETTER
   - DEF_NAME
   - DEF_PARAMS_TERM
@@ -1800,7 +1801,7 @@ nodes:
           Represents the predicate of the case statement. This can be either `nil` or any [non-void expressions](https://github.com/ruby/prism/blob/main/docs/parsing_rules.md#non-void-expression).
 
               case true; when false; end
-              ^^^^
+                   ^^^^
       - name: conditions
         type: node[]
         kind: WhenNode

prism/extension.h

@@ -1,7 +1,7 @@
 #ifndef PRISM_EXT_NODE_H
 #define PRISM_EXT_NODE_H
 
-#define EXPECTED_PRISM_VERSION "1.5.1"
+#define EXPECTED_PRISM_VERSION "1.5.2"
 
 #include <ruby.h>
 #include <ruby/encoding.h>

prism/prism.c

@@ -2622,10 +2622,11 @@ pm_break_node_create(pm_parser_t *parser, const pm_token_t *keyword, pm_argument
 // There are certain flags that we want to use internally but don't want to
 // expose because they are not relevant beyond parsing. Therefore we'll define
 // them here and not define them in config.yml/a header file.
-static const pm_node_flags_t PM_WRITE_NODE_FLAGS_IMPLICIT_ARRAY = 0x4;
-static const pm_node_flags_t PM_CALL_NODE_FLAGS_IMPLICIT_ARRAY = 0x40;
-static const pm_node_flags_t PM_CALL_NODE_FLAGS_COMPARISON = 0x80;
-static const pm_node_flags_t PM_CALL_NODE_FLAGS_INDEX = 0x100;
+static const pm_node_flags_t PM_WRITE_NODE_FLAGS_IMPLICIT_ARRAY = (1 << 2);
+
+static const pm_node_flags_t PM_CALL_NODE_FLAGS_IMPLICIT_ARRAY = ((PM_CALL_NODE_FLAGS_LAST - 1) << 1);
+static const pm_node_flags_t PM_CALL_NODE_FLAGS_COMPARISON = ((PM_CALL_NODE_FLAGS_LAST - 1) << 2);
+static const pm_node_flags_t PM_CALL_NODE_FLAGS_INDEX = ((PM_CALL_NODE_FLAGS_LAST - 1) << 3);
 
 /**
  * Allocate and initialize a new CallNode node. This sets everything to NULL or
@@ -5279,6 +5280,12 @@ pm_interpolated_string_node_append(pm_interpolated_string_node_t *node, pm_node_
 
     switch (PM_NODE_TYPE(part)) {
         case PM_STRING_NODE:
+            // If inner string is not frozen, it stops being a static literal. We should *not* clear other flags,
+            // because concatenating two frozen strings (`'foo' 'bar'`) is still frozen. This holds true for
+            // as long as this interpolation only consists of other string literals.
+            if (!PM_NODE_FLAG_P(part, PM_STRING_FLAGS_FROZEN)) {
+                pm_node_flag_unset((pm_node_t *) node, PM_NODE_FLAG_STATIC_LITERAL);
+            }
             part->flags = (pm_node_flags_t) ((part->flags | PM_NODE_FLAG_STATIC_LITERAL | PM_STRING_FLAGS_FROZEN) & ~PM_STRING_FLAGS_MUTABLE);
             break;
         case PM_INTERPOLATED_STRING_NODE:
@@ -14443,6 +14450,17 @@ parse_arguments(pm_parser_t *parser, pm_arguments_t *arguments, bool accepts_for
             if (accepted_newline) {
                 pm_parser_err_previous(parser, PM_ERR_INVALID_COMMA);
             }
+
+            // If this is a command call and an argument takes a block,
+            // there can be no further arguments. For example,
+            // `foo(bar 1 do end, 2)` should be rejected.
+            if (PM_NODE_TYPE_P(argument, PM_CALL_NODE)) {
+                pm_call_node_t *call = (pm_call_node_t *) argument;
+                if (call->opening_loc.start == NULL && call->arguments != NULL && call->block != NULL) {
+                    pm_parser_err_previous(parser, PM_ERR_INVALID_COMMA);
+                    break;
+                }
+            }
         } else {
             // If there is no comma at the end of the argument list then we're
             // done parsing arguments and can break out of this loop.
@@ -14594,6 +14612,18 @@ update_parameter_state(pm_parser_t *parser, pm_token_t *token, pm_parameters_ord
     return true;
 }
 
+/**
+ * Ensures that after parsing a parameter, the next token is not `=`.
+ * Some parameters like `def(* = 1)` cannot become optional. When no parens
+ * are present like in `def * = 1`, this creates ambiguity with endless method definitions.
+ */
+static inline void
+refute_optional_parameter(pm_parser_t *parser) {
+    if (match1(parser, PM_TOKEN_EQUAL)) {
+        pm_parser_err_previous(parser, PM_ERR_DEF_ENDLESS_PARAMETERS);
+    }
+}
+
 /**
  * Parse a list of parameters on a method definition.
  */
@@ -14646,6 +14676,10 @@ parse_parameters(
                     parser->current_scope->parameters |= PM_SCOPE_PARAMETERS_FORWARDING_BLOCK;
                 }
 
+                if (!uses_parentheses) {
+                    refute_optional_parameter(parser);
+                }
+
                 pm_block_parameter_node_t *param = pm_block_parameter_node_create(parser, &name, &operator);
                 if (repeated) {
                     pm_node_flag_set_repeated_parameter((pm_node_t *)param);
@@ -14667,6 +14701,10 @@ parse_parameters(
                 bool succeeded = update_parameter_state(parser, &parser->current, &order);
                 parser_lex(parser);
 
+                if (!uses_parentheses) {
+                    refute_optional_parameter(parser);
+                }
+
                 parser->current_scope->parameters |= PM_SCOPE_PARAMETERS_FORWARDING_ALL;
                 pm_forwarding_parameter_node_t *param = pm_forwarding_parameter_node_create(parser, &parser->previous);
 
@@ -14848,6 +14886,10 @@ parse_parameters(
                         context_pop(parser);
                         pm_parameters_node_keywords_append(params, param);
 
+                        if (!uses_parentheses) {
+                            refute_optional_parameter(parser);
+                        }
+
                         // If parsing the value of the parameter resulted in error recovery,
                         // then we can put a missing node in its place and stop parsing the
                         // parameters entirely now.
@@ -14879,6 +14921,10 @@ parse_parameters(
                     parser->current_scope->parameters |= PM_SCOPE_PARAMETERS_FORWARDING_POSITIONALS;
                 }
 
+                if (!uses_parentheses) {
+                    refute_optional_parameter(parser);
+                }
+
                 pm_node_t *param = (pm_node_t *) pm_rest_parameter_node_create(parser, &operator, &name);
                 if (repeated) {
                     pm_node_flag_set_repeated_parameter(param);
@@ -14927,6 +14973,10 @@ parse_parameters(
                     }
                 }
 
+                if (!uses_parentheses) {
+                    refute_optional_parameter(parser);
+                }
+
                 if (params->keyword_rest == NULL) {
                     pm_parameters_node_keyword_rest_set(params, param);
                 } else {
@@ -18491,20 +18541,28 @@ parse_expression_prefix(pm_parser_t *parser, pm_binding_power_t binding_power, b
             return (pm_node_t *) node;
         }
         case PM_TOKEN_CHARACTER_LITERAL: {
-            parser_lex(parser);
-
-            pm_token_t opening = parser->previous;
-            opening.type = PM_TOKEN_STRING_BEGIN;
-            opening.end = opening.start + 1;
-
-            pm_token_t content = parser->previous;
-            content.type = PM_TOKEN_STRING_CONTENT;
-            content.start = content.start + 1;
-
             pm_token_t closing = not_provided(parser);
-            pm_node_t *node = (pm_node_t *) pm_string_node_create_current_string(parser, &opening, &content, &closing);
+            pm_node_t *node = (pm_node_t *) pm_string_node_create_current_string(
+                parser,
+                &(pm_token_t) {
+                    .type = PM_TOKEN_STRING_BEGIN,
+                    .start = parser->current.start,
+                    .end = parser->current.start + 1
+                },
+                &(pm_token_t) {
+                    .type = PM_TOKEN_STRING_CONTENT,
+                    .start = parser->current.start + 1,
+                    .end = parser->current.end
+                },
+                &closing
+            );
+
             pm_node_flag_set(node, parse_unescaped_encoding(parser));
 
+            // Skip past the character literal here, since now we have handled
+            // parser->explicit_encoding correctly.
+            parser_lex(parser);
+
             // Characters can be followed by strings in which case they are
             // automatically concatenated.
             if (match1(parser, PM_TOKEN_STRING_BEGIN)) {
@@ -20901,7 +20959,7 @@ parse_assignment_values(pm_parser_t *parser, pm_binding_power_t previous_binding
     bool permitted = true;
     if (previous_binding_power != PM_BINDING_POWER_STATEMENT && match1(parser, PM_TOKEN_USTAR)) permitted = false;
 
-    pm_node_t *value = parse_starred_expression(parser, binding_power, previous_binding_power == PM_BINDING_POWER_ASSIGNMENT ? accepts_command_call : previous_binding_power < PM_BINDING_POWER_MATCH, diag_id, (uint16_t) (depth + 1));
+    pm_node_t *value = parse_starred_expression(parser, binding_power, previous_binding_power == PM_BINDING_POWER_ASSIGNMENT ? accepts_command_call : previous_binding_power < PM_BINDING_POWER_MODIFIER, diag_id, (uint16_t) (depth + 1));
     if (!permitted) pm_parser_err_node(parser, value, PM_ERR_UNEXPECTED_MULTI_WRITE);
 
     parse_assignment_value_local(parser, value);
@@ -22498,9 +22556,10 @@ parse_program(pm_parser_t *parser) {
         statements = wrap_statements(parser, statements);
     } else {
         flush_block_exits(parser, previous_block_exits);
-        pm_node_list_free(&current_block_exits);
     }
 
+    pm_node_list_free(&current_block_exits);
+
     // If this is an empty file, then we're still going to parse all of the
     // statements in order to gather up all of the comments and such. Here we'll
     // correct the location information.

prism/templates/include/prism/ast.h.erb

@@ -212,6 +212,8 @@ typedef enum pm_<%= flag.human %> {
     /** <%= value.comment %> */
     PM_<%= flag.human.upcase %>_<%= value.name %> = <%= 1 << (index + Prism::Template::COMMON_FLAGS_COUNT) %>,
     <%- end -%>
+
+    PM_<%= flag.human.upcase %>_LAST,
 } pm_<%= flag.human %>_t;
 <%- end -%>
 

prism/templates/lib/prism/serialize.rb.erb

@@ -14,7 +14,7 @@ module Prism
 
     # The patch version of prism that we are expecting to find in the serialized
     # strings.
-    PATCH_VERSION = 1
+    PATCH_VERSION = 2
 
     # Deserialize the dumped output from a request to parse or parse_file.
     #

prism/templates/src/diagnostic.c.erb

@@ -144,6 +144,7 @@ static const pm_diagnostic_data_t diagnostic_messages[PM_DIAGNOSTIC_ID_MAX] = {
     [PM_ERR_CONDITIONAL_WHILE_PREDICATE]        = { "expected a predicate expression for the `while` statement", PM_ERROR_LEVEL_SYNTAX },
     [PM_ERR_CONSTANT_PATH_COLON_COLON_CONSTANT] = { "expected a constant after the `::` operator", PM_ERROR_LEVEL_SYNTAX },
     [PM_ERR_DEF_ENDLESS]                        = { "could not parse the endless method body", PM_ERROR_LEVEL_SYNTAX },
+    [PM_ERR_DEF_ENDLESS_PARAMETERS]             = { "could not parse the endless method parameters", PM_ERROR_LEVEL_SYNTAX },
     [PM_ERR_DEF_ENDLESS_SETTER]                 = { "invalid method name; a setter method cannot be defined in an endless method definition", PM_ERROR_LEVEL_SYNTAX },
     [PM_ERR_DEF_NAME]                           = { "unexpected %s; expected a method name", PM_ERROR_LEVEL_SYNTAX },
     [PM_ERR_DEF_PARAMS_TERM]                    = { "expected a delimiter to close the parameters", PM_ERROR_LEVEL_SYNTAX },