Skip to content

Commit 154ed98

Browse files
committed
Fix parser translator during string escaping with invalid utf-8
`byte.ord` can raise when it is not valid in its encoding. Instead, prefer `scan_byte` over `get_byte` since that already returns the byte as an integer, sidestepping conversion issues. Fixes #3582
1 parent 3bd6837 commit 154ed98

3 files changed

Lines changed: 50 additions & 34 deletions

File tree

lib/prism/translation/parser/lexer.rb

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -762,12 +762,12 @@ def escape_read(result, scanner, control, meta)
762762
elsif (value = scanner.scan(/M-\\?(?=[[:print:]])/))
763763
# \M-x where x is an ASCII printable character
764764
escape_read(result, scanner, control, true)
765-
elsif (byte = scanner.get_byte)
765+
elsif (byte = scanner.scan_byte)
766766
# Something else after an escape.
767-
if control && byte == "?"
767+
if control && byte == 0x3f # ASCII '?'
768768
result.append_as_bytes(escape_build(0x7f, false, meta))
769769
else
770-
result.append_as_bytes(escape_build(byte.ord, control, meta))
770+
result.append_as_bytes(escape_build(byte, control, meta))
771771
end
772772
end
773773
end

snapshots/strings.txt

Lines changed: 43 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
1-
@ ProgramNode (location: (1,0)-(153,15))
1+
@ ProgramNode (location: (1,0)-(157,15))
22
├── flags: ∅
33
├── locals: []
44
└── statements:
5-
@ StatementsNode (location: (1,0)-(153,15))
5+
@ StatementsNode (location: (1,0)-(157,15))
66
├── flags: ∅
7-
└── body: (length: 65)
7+
└── body: (length: 67)
88
├── @ StringNode (location: (1,0)-(1,6))
99
│ ├── flags: newline
1010
│ ├── opening_loc: (1,0)-(1,2) = "%%"
@@ -757,56 +757,68 @@
757757
│ ├── content_loc: (147,3)-(147,6) = "abc"
758758
│ ├── closing_loc: (147,6)-(147,7) = "}"
759759
│ └── unescaped: "abc"
760-
├── @ StringNode (location: (149,0)-(149,5))
761-
│ ├── flags: newline
762-
│ ├── opening_loc: (149,0)-(149,2) = "%^"
763-
│ ├── content_loc: (149,2)-(149,4) = "\#$"
764-
│ ├── closing_loc: (149,4)-(149,5) = "^"
760+
├── @ StringNode (location: (149,0)-(149,7))
761+
│ ├── flags: newline
762+
│ ├── opening_loc: (149,0)-(149,3) = "%Q("
763+
│ ├── content_loc: (149,3)-(149,6) = "\\«"
764+
│ ├── closing_loc: (149,6)-(149,7) = ")"
765+
│ └── unescaped: "«"
766+
├── @ StringNode (location: (151,0)-(151,7))
767+
│ ├── flags: newline
768+
│ ├── opening_loc: (151,0)-(151,3) = "%q("
769+
│ ├── content_loc: (151,3)-(151,6) = "\\«"
770+
│ ├── closing_loc: (151,6)-(151,7) = ")"
771+
│ └── unescaped: "\\«"
772+
├── @ StringNode (location: (153,0)-(153,5))
773+
│ ├── flags: newline
774+
│ ├── opening_loc: (153,0)-(153,2) = "%^"
775+
│ ├── content_loc: (153,2)-(153,4) = "\#$"
776+
│ ├── closing_loc: (153,4)-(153,5) = "^"
765777
│ └── unescaped: "\#$"
766-
├── @ StringNode (location: (151,0)-(151,4))
778+
├── @ StringNode (location: (155,0)-(155,4))
767779
│ ├── flags: newline
768-
│ ├── opening_loc: (151,0)-(151,2) = "%@"
769-
│ ├── content_loc: (151,2)-(151,3) = "#"
770-
│ ├── closing_loc: (151,3)-(151,4) = "@"
780+
│ ├── opening_loc: (155,0)-(155,2) = "%@"
781+
│ ├── content_loc: (155,2)-(155,3) = "#"
782+
│ ├── closing_loc: (155,3)-(155,4) = "@"
771783
│ └── unescaped: "#"
772-
└── @ InterpolatedStringNode (location: (153,0)-(153,15))
784+
└── @ InterpolatedStringNode (location: (157,0)-(157,15))
773785
├── flags: newline
774-
├── opening_loc: (153,0)-(153,1) = "\""
786+
├── opening_loc: (157,0)-(157,1) = "\""
775787
├── parts: (length: 2)
776-
│ ├── @ EmbeddedStatementsNode (location: (153,1)-(153,12))
788+
│ ├── @ EmbeddedStatementsNode (location: (157,1)-(157,12))
777789
│ │ ├── flags: ∅
778-
│ │ ├── opening_loc: (153,1)-(153,3) = "\#{"
790+
│ │ ├── opening_loc: (157,1)-(157,3) = "\#{"
779791
│ │ ├── statements:
780-
│ │ │ @ StatementsNode (location: (153,3)-(153,11))
792+
│ │ │ @ StatementsNode (location: (157,3)-(157,11))
781793
│ │ │ ├── flags: ∅
782794
│ │ │ └── body: (length: 1)
783-
│ │ │ └── @ InterpolatedStringNode (location: (153,3)-(153,11))
795+
│ │ │ └── @ InterpolatedStringNode (location: (157,3)-(157,11))
784796
│ │ │ ├── flags: ∅
785-
│ │ │ ├── opening_loc: (153,3)-(153,4) = "\""
797+
│ │ │ ├── opening_loc: (157,3)-(157,4) = "\""
786798
│ │ │ ├── parts: (length: 2)
787-
│ │ │ │ ├── @ EmbeddedStatementsNode (location: (153,4)-(153,8))
799+
│ │ │ │ ├── @ EmbeddedStatementsNode (location: (157,4)-(157,8))
788800
│ │ │ │ │ ├── flags: ∅
789-
│ │ │ │ │ ├── opening_loc: (153,4)-(153,6) = "\#{"
801+
│ │ │ │ │ ├── opening_loc: (157,4)-(157,6) = "\#{"
790802
│ │ │ │ │ ├── statements:
791-
│ │ │ │ │ │ @ StatementsNode (location: (153,6)-(153,7))
803+
│ │ │ │ │ │ @ StatementsNode (location: (157,6)-(157,7))
792804
│ │ │ │ │ │ ├── flags: ∅
793805
│ │ │ │ │ │ └── body: (length: 1)
794-
│ │ │ │ │ │ └── @ ConstantReadNode (location: (153,6)-(153,7))
806+
│ │ │ │ │ │ └── @ ConstantReadNode (location: (157,6)-(157,7))
795807
│ │ │ │ │ │ ├── flags: ∅
796808
│ │ │ │ │ │ └── name: :B
797-
│ │ │ │ │ └── closing_loc: (153,7)-(153,8) = "}"
798-
│ │ │ │ └── @ StringNode (location: (153,8)-(153,10))
809+
│ │ │ │ │ └── closing_loc: (157,7)-(157,8) = "}"
810+
│ │ │ │ └── @ StringNode (location: (157,8)-(157,10))
799811
│ │ │ │ ├── flags: static_literal, frozen
800812
│ │ │ │ ├── opening_loc: ∅
801-
│ │ │ │ ├── content_loc: (153,8)-(153,10) = " C"
813+
│ │ │ │ ├── content_loc: (157,8)-(157,10) = " C"
802814
│ │ │ │ ├── closing_loc: ∅
803815
│ │ │ │ └── unescaped: " C"
804-
│ │ │ └── closing_loc: (153,10)-(153,11) = "\""
805-
│ │ └── closing_loc: (153,11)-(153,12) = "}"
806-
│ └── @ StringNode (location: (153,12)-(153,14))
816+
│ │ │ └── closing_loc: (157,10)-(157,11) = "\""
817+
│ │ └── closing_loc: (157,11)-(157,12) = "}"
818+
│ └── @ StringNode (location: (157,12)-(157,14))
807819
│ ├── flags: static_literal, frozen
808820
│ ├── opening_loc: ∅
809-
│ ├── content_loc: (153,12)-(153,14) = " D"
821+
│ ├── content_loc: (157,12)-(157,14) = " D"
810822
│ ├── closing_loc: ∅
811823
│ └── unescaped: " D"
812-
└── closing_loc: (153,14)-(153,15) = "\""
824+
└── closing_loc: (157,14)-(157,15) = "\""

test/prism/fixtures/strings.txt

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,10 @@ baz
146146

147147
%Q{abc}
148148

149+
%Q(\«)
150+
151+
%q(\«)
152+
149153
%^#$^#
150154

151155
%@#@#

0 commit comments

Comments
 (0)