fix another ICE caused by improper unicode handling

Author: pitaj

compiler/rustc_parse/src/lexer/mod.rs

@@ -249,7 +249,7 @@ impl<'psess, 'src> StringReader<'psess, 'src> {
                     // lint for each `#` before the string.
                     if !(
                         start > self.start_pos &&
-                        self.str_from_to(start - BytePos(1), start) == "#"
+                        self.src.as_bytes()[self.src_index(start) - 1] == b'#'
                     ) {
                         let span = self.mk_sp(start, self.pos);
                         let space_span = n_start_hashes.map(|n_hashes| {

tests/ui/rust-2024/reserved-guarded-strings-lexing.rs

@@ -0,0 +1,22 @@
+//@ edition:2021
+
+macro_rules! demo2 {
+    ( $a:tt $b:tt ) => { println!("two tokens") };
+}
+
+macro_rules! demo3 {
+    ( $a:tt $b:tt $c:tt ) => { println!("three tokens") };
+}
+
+macro_rules! demo4 {
+    ( $a:tt $b:tt $c:tt $d:tt ) => { println!("four tokens") };
+}
+
+fn main() {
+    // Non-ascii identifiers
+    demo2!(Ñ"foo"); //~ ERROR prefix `Ñ` is unknown
+    demo4!(Ñ#""#);  //~ ERROR prefix `Ñ` is unknown
+    demo3!(🙃#"");
+    //~^ ERROR prefix `🙃` is unknown
+    //~| WARNING identifier contains an uncommon character
+}

tests/ui/rust-2024/reserved-guarded-strings-lexing.stderr

@@ -0,0 +1,47 @@
+error: prefix `Ñ` is unknown
+  --> $DIR/reserved-guarded-strings-lexing.rs:17:12
+   |
+LL |     demo2!(Ñ"foo");
+   |            ^ unknown prefix
+   |
+   = note: prefixed identifiers and literals are reserved since Rust 2021
+help: consider inserting whitespace here
+   |
+LL |     demo2!(Ñ "foo");
+   |             +
+
+error: prefix `Ñ` is unknown
+  --> $DIR/reserved-guarded-strings-lexing.rs:18:12
+   |
+LL |     demo4!(Ñ#""#);
+   |            ^ unknown prefix
+   |
+   = note: prefixed identifiers and literals are reserved since Rust 2021
+help: consider inserting whitespace here
+   |
+LL |     demo4!(Ñ #""#);
+   |             +
+
+error: prefix `🙃` is unknown
+  --> $DIR/reserved-guarded-strings-lexing.rs:19:12
+   |
+LL |     demo3!(🙃#"");
+   |            ^^ unknown prefix
+   |
+   = note: prefixed identifiers and literals are reserved since Rust 2021
+help: consider inserting whitespace here
+   |
+LL |     demo3!(🙃 #"");
+   |              +
+
+warning: identifier contains an uncommon character: '🙃'
+  --> $DIR/reserved-guarded-strings-lexing.rs:19:12
+   |
+LL |     demo3!(🙃#"");
+   |            ^^
+   |
+   = note: this character is included in the Unicode general security profile
+   = note: `#[warn(uncommon_codepoints)]` on by default
+
+error: aborting due to 3 previous errors; 1 warning emitted
+

tests/ui/rust-2024/reserved-guarded-strings.rs

@@ -44,8 +44,9 @@ fn main() {
     demo4!("foo"###);
 
     demo2!(blah"xx"); //~ ERROR prefix `blah` is unknown
-    demo2!(blah#"xx"#); //~ ERROR prefix `blah` is unknown
-    //~^ ERROR invalid string literal
+    demo2!(blah#"xx"#);
+    //~^ ERROR prefix `blah` is unknown
+    //~| ERROR invalid string literal
 
     demo1!(#""); //~ ERROR invalid string literal
     demo1!(#""#); //~ ERROR invalid string literal

tests/ui/rust-2024/reserved-guarded-strings.stderr

@@ -35,7 +35,7 @@ LL |     demo2!(blah# "xx"#);
    |                 +
 
 error: invalid string literal
-  --> $DIR/reserved-guarded-strings.rs:50:12
+  --> $DIR/reserved-guarded-strings.rs:51:12
    |
 LL |     demo1!(#"");
    |            ^^^
@@ -47,7 +47,7 @@ LL |     demo1!(# "");
    |             +
 
 error: invalid string literal
-  --> $DIR/reserved-guarded-strings.rs:51:12
+  --> $DIR/reserved-guarded-strings.rs:52:12
    |
 LL |     demo1!(#""#);
    |            ^^^^
@@ -59,7 +59,7 @@ LL |     demo1!(# ""#);
    |             +
 
 error: invalid string literal
-  --> $DIR/reserved-guarded-strings.rs:52:12
+  --> $DIR/reserved-guarded-strings.rs:53:12
    |
 LL |     demo1!(####"");
    |            ^^^^^^
@@ -71,7 +71,7 @@ LL |     demo1!(#### "");
    |                +
 
 error: invalid string literal
-  --> $DIR/reserved-guarded-strings.rs:53:12
+  --> $DIR/reserved-guarded-strings.rs:54:12
    |
 LL |     demo1!(#"foo");
    |            ^^^^^^
@@ -83,7 +83,7 @@ LL |     demo1!(# "foo");
    |             +
 
 error: invalid string literal
-  --> $DIR/reserved-guarded-strings.rs:54:12
+  --> $DIR/reserved-guarded-strings.rs:55:12
    |
 LL |     demo1!(###"foo");
    |            ^^^^^^^^
@@ -95,7 +95,7 @@ LL |     demo1!(### "foo");
    |               +
 
 error: invalid string literal
-  --> $DIR/reserved-guarded-strings.rs:55:12
+  --> $DIR/reserved-guarded-strings.rs:56:12
    |
 LL |     demo1!(#"foo"#);
    |            ^^^^^^^
@@ -107,7 +107,7 @@ LL |     demo1!(# "foo"#);
    |             +
 
 error: invalid string literal
-  --> $DIR/reserved-guarded-strings.rs:56:12
+  --> $DIR/reserved-guarded-strings.rs:57:12
    |
 LL |     demo1!(###"foo"#);
    |            ^^^^^^^^^
@@ -119,7 +119,7 @@ LL |     demo1!(### "foo"#);
    |               +
 
 error: invalid string literal
-  --> $DIR/reserved-guarded-strings.rs:57:12
+  --> $DIR/reserved-guarded-strings.rs:58:12
    |
 LL |     demo1!(###"foo"##);
    |            ^^^^^^^^^^
@@ -131,7 +131,7 @@ LL |     demo1!(### "foo"##);
    |               +
 
 error: invalid string literal
-  --> $DIR/reserved-guarded-strings.rs:58:12
+  --> $DIR/reserved-guarded-strings.rs:59:12
    |
 LL |     demo1!(###"foo"###);
    |            ^^^^^^^^^^^