Merge branch 'main' into topic/javawasm_changes

Author: headius

.github/workflows/main.yml

@@ -79,7 +79,7 @@ jobs:
       shell: bash
 
   build-ibm:
-    if: github.repository == 'ruby/prism'
+    if: github.repository == 'ruby/prism' && false
     strategy:
       fail-fast: false
       matrix:
@@ -105,7 +105,7 @@ jobs:
       matrix:
         target:
           - { ruby: "head", gemfile: "4.1" }
-          - { ruby: "jruby-10.0.0.0", gemfile: ".." } # https://github.com/jruby/jruby/issues/8923
+          - { ruby: "jruby", gemfile: ".." }
           - { ruby: "truffleruby", gemfile: ".." }
     runs-on: ubuntu-latest
     env:
@@ -277,7 +277,7 @@ jobs:
           - { ruby: "3.4", os: "ubuntu-latest", gemfile: "3.4" }
           - { ruby: "4.0", os: "ubuntu-latest", gemfile: "4.0" }
           - { ruby: "head", os: "ubuntu-latest", gemfile: "4.1" }
-          - { ruby: "jruby-10.0.0.0", os: "ubuntu-latest", gemfile: ".." } # https://github.com/jruby/jruby/issues/8923
+          - { ruby: "jruby", os: "ubuntu-latest", gemfile: ".." }
           - { ruby: "truffleruby", os: "ubuntu-latest", gemfile: ".." }
 
           - { ruby: "2.7", os: "macos-latest", gemfile: "2.7" }
@@ -288,7 +288,7 @@ jobs:
           - { ruby: "3.4", os: "macos-latest", gemfile: "3.4" }
           - { ruby: "4.0", os: "macos-latest", gemfile: "4.0" }
           - { ruby: "head", os: "macos-latest", gemfile: "4.1" }
-          - { ruby: "jruby-10.0.0.0", os: "macos-latest", gemfile: ".." } # https://github.com/jruby/jruby/issues/8923
+          - { ruby: "jruby", os: "macos-latest", gemfile: ".." }
           - { ruby: "truffleruby", os: "macos-latest", gemfile: ".." }
 
           - { ruby: "2.7", os: "windows-latest", gemfile: "2.7" }
@@ -298,8 +298,8 @@ jobs:
           - { ruby: "3.3", os: "windows-latest", gemfile: "3.3" }
           - { ruby: "3.4", os: "windows-latest", gemfile: "3.4" }
           - { ruby: "4.0", os: "windows-latest", gemfile: "4.0" }
-          # - { ruby: "head", os: "windows-latest", gemfile: "4.1" } TODO: No windows build yet
-          - { ruby: "jruby-10.0.0.0", os: "windows-latest", gemfile: ".." } # https://github.com/jruby/jruby/issues/8923
+          - { ruby: "head", os: "windows-latest", gemfile: "4.1" }
+          - { ruby: "jruby", os: "windows-latest", gemfile: ".." }
     env:
       BUNDLE_GEMFILE: gemfiles/${{ matrix.target.gemfile }}/Gemfile
     runs-on: ${{ matrix.target.os }}

CHANGELOG.md

@@ -4,6 +4,21 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+
+## [1.9.0] - 2026-01-27
+
+### Added
+
+- Lots of work on the Ripper translation layer to make it more compatible and efficient.
+- Alias `Prism::Node#breadth_first_search` to `Prism::Node#find`.
+- Add `Prism::Node#breadth_first_search_all`/`Prism::Node#find_all` for finding all nodes matching a condition.
+
+### Changed
+
+- Fixed location of opening tokens when invalid syntax is parsed.
+- Fix RBI for parsing options.
+
 ## [1.8.0] - 2026-01-12
 
 ### Added
@@ -19,8 +34,6 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a
 - Decouple ripper translator from ripper library.
 - Sync Prism::Translation::ParserCurrent with Ruby 4.0.
 
-## [Unreleased]
-
 ## [1.7.0] - 2025-12-18
 
 ### Added
@@ -731,7 +744,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) a
 
 - 🎉 Initial release! 🎉
 
-[unreleased]: https://github.com/ruby/prism/compare/v1.8.0...HEAD
+[unreleased]: https://github.com/ruby/prism/compare/v1.9.0...HEAD
+[1.9.0]: https://github.com/ruby/prism/compare/v1.8.0...v1.9.0
 [1.8.0]: https://github.com/ruby/prism/compare/v1.7.0...v1.8.0
 [1.7.0]: https://github.com/ruby/prism/compare/v1.6.0...v1.7.0
 [1.6.0]: https://github.com/ruby/prism/compare/v1.5.2...v1.6.0

Gemfile

@@ -11,7 +11,7 @@ gem "rake-compiler"
 gem "ruby_parser"
 gem "test-unit"
 
-platforms :mri, :mswin, :mingw, :x64_mingw do
+platforms :mri, :windows do
   gem "ffi"
   gem "irb"
   gem "ruby_memcheck"

Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    prism (1.8.0)
+    prism (1.9.0)
 
 GEM
   remote: https://rubygems.org/
@@ -82,4 +82,4 @@ DEPENDENCIES
   test-unit
 
 BUNDLED WITH
-   2.5.16
+  4.0.4

Steepfile

@@ -10,7 +10,6 @@ target :lib do
   # TODO: Type-checking these files is still WIP
   ignore "lib/prism/desugar_compiler.rb"
   ignore "lib/prism/lex_compat.rb"
-  ignore "lib/prism/lex_ripper.rb"
   ignore "lib/prism/serialize.rb"
   ignore "lib/prism/ffi.rb"
   ignore "lib/prism/translation"

bin/prism

@@ -30,7 +30,7 @@ module Prism
             bin/prism console
             bin/prism dot [source]
             bin/prism encoding [encoding]
-            bin/prism error [name] [source]
+            bin/prism error [source]
             bin/prism lex [source]
             bin/prism lex_compat [source]
             bin/prism locals [source]
@@ -161,36 +161,17 @@ module Prism
       unicode_lists(found) if found == Encoding::UTF_8 || found == Encoding::UTF8_MAC
     end
 
-    # bin/prism error [name] [source]
+    # bin/prism error [source]
     def error(argv)
-      name = argv.shift
-
-      source = nil
-      filepath = File.expand_path("../test/prism/errors/#{name}.txt", __dir__)
+      source, filepath = read_source(argv)
+      result = Prism.parse(source)
 
-      if argv.empty?
-        raise "Expected #{filepath} to exist" unless File.file?(filepath)
 
-        source = File.read(filepath, binmode: true, external_encoding: Encoding::UTF_8)
-        source = source.lines.grep_v(/^\s*\^/).join.gsub(/\n*\z/, "")
+      if result.errors.any?
+        puts result.errors_format
       else
-        if File.file?(filepath)
-          counter = 1
-
-          begin
-            current = "#{File.dirname(filepath)}/#{File.basename(filepath, ".txt")}_#{counter += 1}.txt"
-          end while File.file?(current)
-
-          filepath = current
-        end
-
-        source, _ = read_source(argv)
+        puts "No syntax errors for #{filepath}"
       end
-
-      result = Prism.parse(source)
-      raise "Expected #{source.inspect} to have errors" if result.success?
-
-      File.write(filepath, result.errors_format)
     end
 
     # bin/prism lex [source]
@@ -210,18 +191,10 @@ module Prism
 
     # bin/prism lex_compat [source]
     def lex_compat(argv)
+      require "ripper"
       source, filepath = read_source(argv)
 
-      ripper_value =
-        begin
-          Prism.lex_ripper(source)
-        rescue ArgumentError, SyntaxError
-          # If Ripper raises a syntax error, we want to continue as if it didn't
-          # return any tokens at all. prism won't raise a syntax error, so it's
-          # nicer to still be able to see the tokens that prism generated.
-          []
-        end
-
+      ripper_value = Ripper.lex(source)
       prism_compat = Prism.lex_compat(source, filepath: filepath)
       prism = Prism.lex(source, filepath: filepath)
 

config.yml

@@ -653,10 +653,6 @@ tokens:
     comment: "a separator between words in a list"
   - name: __END__
     comment: "marker for the point in the file at which the parser should stop"
-  - name: MISSING
-    comment: "a token that was expected but not found"
-  - name: NOT_PROVIDED
-    comment: "a token that was not present but it is okay"
 flags:
   - name: ArgumentsNodeFlags
     values:

docs/configuration.md

@@ -61,7 +61,7 @@ The available values for `type` are:
 * `constant` - A field that is an integer that represents an index in the constant pool. This is a `pm_constant_id_t` in C.
 * `constant[]` - A field that is an array of constants. This is a `pm_constant_id_list_t` in C.
 * `location` - A field that is a location. This is a `pm_location_t` in C.
-* `location?` - A field that is a location that is optionally present. This is a `pm_location_t` in C, but if the value is not present then the `start` and `end` fields will be `NULL`.
+* `location?` - A field that is a location that is optionally present. This is a `pm_location_t` in C, but if the value is not present then the `length` field will be `0`.
 * `uint8` - A field that is an 8-bit unsigned integer. This is a `uint8_t` in C.
 * `uint32` - A field that is a 32-bit unsigned integer. This is a `uint32_t` in C.
 

ext/prism/extension.c

@@ -201,9 +201,22 @@ build_options_i(VALUE key, VALUE value, VALUE argument) {
             const char *version = check_string(value);
 
             if (RSTRING_LEN(value) == 7 && strncmp(version, "current", 7) == 0) {
-                const char *current_version = RSTRING_PTR(rb_const_get(rb_cObject, rb_intern("RUBY_VERSION")));
-                if (!pm_options_version_set(options, current_version, 3)) {
-                    rb_exc_raise(rb_exc_new_cstr(rb_cPrismCurrentVersionError, current_version));
+                if (!pm_options_version_set(options, ruby_version, 3)) {
+                    rb_exc_raise(rb_exc_new_cstr(rb_cPrismCurrentVersionError, ruby_version));
+                }
+            } else if (RSTRING_LEN(value) == 7 && strncmp(version, "nearest", 7) == 0) {
+                const char *nearest_version;
+
+                if (ruby_version[0] < '3' || (ruby_version[0] == '3' && ruby_version[2] < '3')) {
+                    nearest_version = "3.3";
+                } else if (ruby_version[0] > '4' || (ruby_version[0] == '4' && ruby_version[2] > '1')) {
+                    nearest_version = "4.1";
+                } else {
+                    nearest_version = ruby_version;
+                }
+
+                if (!pm_options_version_set(options, nearest_version, 3)) {
+                    rb_raise(rb_eArgError, "invalid nearest version: %s", nearest_version);
                 }
             } else if (!pm_options_version_set(options, version, RSTRING_LEN(value))) {
                 rb_raise(rb_eArgError, "invalid version: %" PRIsVALUE, value);
@@ -455,23 +468,23 @@ rb_class_new_instance_freeze(int argc, const VALUE *argv, VALUE klass, bool free
  * Create a new Location instance from the given parser and bounds.
  */
 static inline VALUE
-parser_location(const pm_parser_t *parser, VALUE source, bool freeze, const uint8_t *start, size_t length) {
-    VALUE argv[] = { source, LONG2FIX(start - parser->start), LONG2FIX(length) };
+parser_location(VALUE source, bool freeze, uint32_t start, uint32_t length) {
+    VALUE argv[] = { source, LONG2FIX(start), LONG2FIX(length) };
     return rb_class_new_instance_freeze(3, argv, rb_cPrismLocation, freeze);
 }
 
 /**
  * Create a new Location instance from the given parser and location.
  */
-#define PARSER_LOCATION_LOC(parser, source, freeze, loc) \
-    parser_location(parser, source, freeze, loc.start, (size_t) (loc.end - loc.start))
+#define PARSER_LOCATION(source, freeze, location) \
+    parser_location(source, freeze, location.start, location.length)
 
 /**
  * Build a new Comment instance from the given parser and comment.
  */
 static inline VALUE
-parser_comment(const pm_parser_t *parser, VALUE source, bool freeze, const pm_comment_t *comment) {
-    VALUE argv[] = { PARSER_LOCATION_LOC(parser, source, freeze, comment->location) };
+parser_comment(VALUE source, bool freeze, const pm_comment_t *comment) {
+    VALUE argv[] = { PARSER_LOCATION(source, freeze, comment->location) };
     VALUE type = (comment->type == PM_COMMENT_EMBDOC) ? rb_cPrismEmbDocComment : rb_cPrismInlineComment;
     return rb_class_new_instance_freeze(1, argv, type, freeze);
 }
@@ -488,7 +501,7 @@ parser_comments(const pm_parser_t *parser, VALUE source, bool freeze) {
         comment != NULL;
         comment = (const pm_comment_t *) comment->node.next
     ) {
-        VALUE value = parser_comment(parser, source, freeze, comment);
+        VALUE value = parser_comment(source, freeze, comment);
         rb_ary_push(comments, value);
     }
 
@@ -500,9 +513,9 @@ parser_comments(const pm_parser_t *parser, VALUE source, bool freeze) {
  * Build a new MagicComment instance from the given parser and magic comment.
  */
 static inline VALUE
-parser_magic_comment(const pm_parser_t *parser, VALUE source, bool freeze, const pm_magic_comment_t *magic_comment) {
-    VALUE key_loc = parser_location(parser, source, freeze, magic_comment->key_start, magic_comment->key_length);
-    VALUE value_loc = parser_location(parser, source, freeze, magic_comment->value_start, magic_comment->value_length);
+parser_magic_comment(VALUE source, bool freeze, const pm_magic_comment_t *magic_comment) {
+    VALUE key_loc = parser_location(source, freeze, magic_comment->key.start, magic_comment->key.length);
+    VALUE value_loc = parser_location(source, freeze, magic_comment->value.start, magic_comment->value.length);
     VALUE argv[] = { key_loc, value_loc };
     return rb_class_new_instance_freeze(2, argv, rb_cPrismMagicComment, freeze);
 }
@@ -519,7 +532,7 @@ parser_magic_comments(const pm_parser_t *parser, VALUE source, bool freeze) {
         magic_comment != NULL;
         magic_comment = (const pm_magic_comment_t *) magic_comment->node.next
     ) {
-        VALUE value = parser_magic_comment(parser, source, freeze, magic_comment);
+        VALUE value = parser_magic_comment(source, freeze, magic_comment);
         rb_ary_push(magic_comments, value);
     }
 
@@ -533,10 +546,10 @@ parser_magic_comments(const pm_parser_t *parser, VALUE source, bool freeze) {
  */
 static VALUE
 parser_data_loc(const pm_parser_t *parser, VALUE source, bool freeze) {
-    if (parser->data_loc.end == NULL) {
+    if (parser->data_loc.length == 0) {
         return Qnil;
     } else {
-        return PARSER_LOCATION_LOC(parser, source, freeze, parser->data_loc);
+        return parser_location(source, freeze, parser->data_loc.start, parser->data_loc.length);
     }
 }
 
@@ -554,7 +567,7 @@ parser_errors(const pm_parser_t *parser, rb_encoding *encoding, VALUE source, bo
     ) {
         VALUE type = ID2SYM(rb_intern(pm_diagnostic_id_human(error->diag_id)));
         VALUE message = rb_obj_freeze(rb_enc_str_new_cstr(error->message, encoding));
-        VALUE location = PARSER_LOCATION_LOC(parser, source, freeze, error->location);
+        VALUE location = PARSER_LOCATION(source, freeze, error->location);
 
         VALUE level = Qnil;
         switch (error->level) {
@@ -594,7 +607,7 @@ parser_warnings(const pm_parser_t *parser, rb_encoding *encoding, VALUE source,
     ) {
         VALUE type = ID2SYM(rb_intern(pm_diagnostic_id_human(warning->diag_id)));
         VALUE message = rb_obj_freeze(rb_enc_str_new_cstr(warning->message, encoding));
-        VALUE location = PARSER_LOCATION_LOC(parser, source, freeze, warning->location);
+        VALUE location = PARSER_LOCATION(source, freeze, warning->location);
 
         VALUE level = Qnil;
         switch (warning->level) {
@@ -894,8 +907,10 @@ parse_input(pm_string_t *input, const pm_options_t *options) {
  *       version of Ruby syntax (which you can trigger with `nil` or
  *       `"latest"`). You may also restrict the syntax to a specific version of
  *       Ruby, e.g., with `"3.3.0"`. To parse with the same syntax version that
- *       the current Ruby is running use `version: "current"`. Raises
- *       ArgumentError if the version is not currently supported by Prism.
+ *       the current Ruby is running use `version: "current"`. To parse with the
+ *       nearest version to the current Ruby that is running, use
+ *       `version: "nearest"`. Raises ArgumentError if the version is not
+ *       currently supported by Prism.
  */
 static VALUE
 parse(int argc, VALUE *argv, VALUE self) {

ext/prism/extension.h

@@ -1,10 +1,11 @@
 #ifndef PRISM_EXT_NODE_H
 #define PRISM_EXT_NODE_H
 
-#define EXPECTED_PRISM_VERSION "1.8.0"
+#define EXPECTED_PRISM_VERSION "1.9.0"
 
 #include <ruby.h>
 #include <ruby/encoding.h>
+#include <ruby/version.h>
 #include "prism.h"
 
 VALUE pm_source_new(const pm_parser_t *parser, rb_encoding *encoding, bool freeze);