Syntax highlighting using Prism.parse_lex (#1704)

Completely drops ripper dependency.

### Drops `ruby-node` css class used to colorize dstring and backref
backref(`$&`) is now colored as identifier, same as global variable.
dstring(`"#{expr}"`, `` `#{expr}` ``) is now colored as string. Embedded
expression parts are colored separately.

### Stops token squashing:
`"string"` was a single string token. Changed to `"` + `string` + `"`
`"string#{:symbol}"` was a single dstring token. Changed to string
tokens `"` `string` `#{` `}` `"`. `:` and `symbol is colorized as symbol
color.
So there should be lots of diff.

Before
<img width="759" height="278" alt="before"
src="https://github.com/user-attachments/assets/6ff20691-082f-42d4-b4f1-25131f8c508b"
/>
After
<img width="767" height="287" alt="after"
src="https://github.com/user-attachments/assets/b2a80eb3-aab9-4b4e-8585-918931b2e4eb"
/>

### Fix some unkown bugs
```ruby
# Documenting this code crashed
def foo
  <<~A; :\
  A
  b
end
```

Author: tompng

lib/rdoc/generator/markup.rb

@@ -125,9 +125,6 @@ def markup_code
 
     src = RDoc::TokenStream.to_html @token_stream
 
-    # add initial whitespace so that the indent gets calculated correctly
-    src.prepend(' ' * @token_stream.first[:char_no]) if source_language == 'ruby' && @token_stream.first
-
     # dedent the source
     common_indent = src.length
     src.scan(/^ *(?=\S)/) do |whitespace|

lib/rdoc/generator/template/aliki/css/rdoc.css

@@ -1021,7 +1021,6 @@ main h6 a:hover {
 .ruby-ivar       { color: var(--code-orange); }
 .ruby-operator   { color: var(--code-green); }
 .ruby-identifier { color: var(--code-blue); }
-.ruby-node       { color: var(--code-purple); }
 
 .ruby-comment {
   color: var(--color-neutral-500);
@@ -1037,7 +1036,6 @@ main h6 a:hover {
 [data-theme="dark"] .ruby-ivar       { color: var(--code-orange); }
 [data-theme="dark"] .ruby-operator   { color: var(--code-green); }
 [data-theme="dark"] .ruby-identifier { color: var(--code-blue); }
-[data-theme="dark"] .ruby-node       { color: var(--code-purple); }
 
 [data-theme="dark"] .ruby-comment {
   color: var(--color-neutral-400);

lib/rdoc/generator/template/darkfish/css/rdoc.css

@@ -449,7 +449,6 @@ main h6 {
 .ruby-ivar       { color: #B57614; }  /* Brown */
 .ruby-operator   { color: #427B58; }  /* Dark Teal */
 .ruby-identifier { color: #076678; }  /* Deep Teal */
-.ruby-node       { color: #8F3F71; }  /* Plum */
 .ruby-comment    { color: #928374; font-style: italic; }  /* Gray */
 .ruby-regexp     { color: #8F3F71; }  /* Plum */
 .ruby-value      { color: #AF3A03; }  /* Dark Orange */

lib/rdoc/markup/to_html.rb

@@ -2,6 +2,7 @@
 require 'cgi/escape'
 require 'cgi/util' unless defined?(CGI::EscapeExt)
 require 'prism'
+require 'rdoc/parser/ruby_colorizer'
 
 ##
 # Outputs RDoc markup as HTML.
@@ -321,34 +322,33 @@ def accept_paragraph(paragraph)
     @res << "</p>\n"
   end
 
+  # Generate syntax highlighted html for ruby-like text.
+
+  def parsable_text_to_html(text)
+    tokens = RDoc::Parser::RubyColorizer.colorize(text)
+    result = RDoc::TokenStream.to_html tokens
+    result = result + "\n" unless result.end_with?("\n")
+    result
+  end
+
   ##
   # Adds +verbatim+ to the output
 
   def accept_verbatim(verbatim)
     text = verbatim.text.rstrip
     format = verbatim.format
 
-    klass = nil
-
     # Apply Ruby syntax highlighting if
     # - explicitly marked as Ruby (via ruby? which accepts :ruby or :rb)
     # - no format specified but the text is parseable as Ruby
     # Otherwise, add language class when applicable and skip Ruby highlighting
-    content = if verbatim.ruby? || (format.nil? && parseable?(text))
-                begin
-                  tokens = RDoc::Parser::RipperStateLex.parse text
-                  klass  = ' class="ruby"'
-
-                  result = RDoc::TokenStream.to_html tokens
-                  result = result + "\n" unless "\n" == result[-1]
-                  result
-                rescue
-                  CGI.escapeHTML text
-                end
-              else
-                klass = " class=\"#{format}\"" if format
-                CGI.escapeHTML text
-              end
+    if verbatim.ruby? || (format.nil? && parseable?(text))
+      content = parsable_text_to_html(text)
+      klass = ' class="ruby"'
+    else
+      content = CGI.escapeHTML text
+      klass = " class=\"#{format}\"" if format
+    end
 
     if @pipe
       @res << "\n<pre><code>#{CGI.escapeHTML text}\n</code></pre>\n"