Skip to content

Commit 0269aec

Browse files
borsbors
committed
Auto merge of #147859 - cyrgani:nonfatal-tokenstream-parse, r=<try>
reduce the amount of panics in `{TokenStream, Literal}::from_str` calls
2 parents 3f6250a + f1c80f5 commit 0269aec

12 files changed

Lines changed: 74 additions & 119 deletions

File tree

‎compiler/rustc_errors/src/diagnostic.rs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1340,6 +1340,13 @@ impl<'a, G: EmissionGuarantee> Diag<'a, G> {
13401340
drop(self);
13411341
}
13421342

1343+
/// Cancels this diagnostic and returns its first message, if it exists.
1344+
pub fn cancel_into_message(self) -> Option<String> {
1345+
let s = self.diag.as_ref()?.messages.get(0)?.0.as_str().map(ToString::to_string);
1346+
self.cancel();
1347+
s
1348+
}
1349+
13431350
/// See `DiagCtxt::stash_diagnostic` for details.
13441351
pub fn stash(mut self, span: Span, key: StashKey) -> Option<ErrorGuaranteed> {
13451352
let diag = self.take_diag();

‎compiler/rustc_expand/src/proc_macro_server.rs‎

Lines changed: 22 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ use rustc_data_structures::fx::FxHashMap;
1010
use rustc_errors::{Diag, ErrorGuaranteed, MultiSpan};
1111
use rustc_parse::lexer::{StripTokens, nfc_normalize};
1212
use rustc_parse::parser::Parser;
13-
use rustc_parse::{exp, new_parser_from_source_str, source_str_to_stream, unwrap_or_emit_fatal};
13+
use rustc_parse::{exp, new_parser_from_source_str, source_str_to_stream};
1414
use rustc_proc_macro::bridge::{
1515
DelimSpan, Diagnostic, ExpnGlobals, Group, Ident, LitKind, Literal, Punct, TokenTree, server,
1616
};
@@ -431,6 +431,13 @@ impl ToInternal<rustc_errors::Level> for Level {
431431
}
432432
}
433433

434+
fn cancel_diags_into_string(diags: Vec<Diag<'_>>) -> String {
435+
let mut messages = diags.into_iter().flat_map(Diag::cancel_into_message);
436+
let msg = messages.next().expect("no diagnostic has a message");
437+
messages.for_each(|_| ()); // consume iterator to cancel the remaining diagnostics
438+
msg
439+
}
440+
434441
pub(crate) struct Rustc<'a, 'b> {
435442
ecx: &'a mut ExtCtxt<'b>,
436443
def_site: Span,
@@ -494,35 +501,32 @@ impl server::Server for Rustc<'_, '_> {
494501
self.psess().file_depinfo.borrow_mut().insert(Symbol::intern(path));
495502
}
496503

497-
fn literal_from_str(&mut self, s: &str) -> Result<Literal<Self::Span, Self::Symbol>, ()> {
504+
fn literal_from_str(&mut self, s: &str) -> Result<Literal<Self::Span, Self::Symbol>, String> {
498505
let name = FileName::proc_macro_source_code(s);
499506

500-
let mut parser = unwrap_or_emit_fatal(new_parser_from_source_str(
501-
self.psess(),
502-
name,
503-
s.to_owned(),
504-
StripTokens::Nothing,
505-
));
507+
let mut parser =
508+
new_parser_from_source_str(self.psess(), name, s.to_owned(), StripTokens::Nothing)
509+
.map_err(cancel_diags_into_string)?;
506510

507511
let first_span = parser.token.span.data();
508512
let minus_present = parser.eat(exp!(Minus));
509513

510514
let lit_span = parser.token.span.data();
511515
let token::Literal(mut lit) = parser.token.kind else {
512-
return Err(());
516+
return Err("not a literal".to_string());
513517
};
514518

515519
// Check no comment or whitespace surrounding the (possibly negative)
516520
// literal, or more tokens after it.
517521
if (lit_span.hi.0 - first_span.lo.0) as usize != s.len() {
518-
return Err(());
522+
return Err("comment or whitespace around literal".to_string());
519523
}
520524

521525
if minus_present {
522526
// If minus is present, check no comment or whitespace in between it
523527
// and the literal token.
524528
if first_span.hi.0 != lit_span.lo.0 {
525-
return Err(());
529+
return Err("comment or whitespace after minus".to_string());
526530
}
527531

528532
// Check literal is a kind we allow to be negated in a proc macro token.
@@ -536,7 +540,9 @@ impl server::Server for Rustc<'_, '_> {
536540
| token::LitKind::ByteStrRaw(_)
537541
| token::LitKind::CStr
538542
| token::LitKind::CStrRaw(_)
539-
| token::LitKind::Err(_) => return Err(()),
543+
| token::LitKind::Err(_) => {
544+
return Err("non-numeric literal may not be negated".to_string());
545+
}
540546
token::LitKind::Integer | token::LitKind::Float => {}
541547
}
542548

@@ -576,13 +582,14 @@ impl server::Server for Rustc<'_, '_> {
576582
stream.is_empty()
577583
}
578584

579-
fn ts_from_str(&mut self, src: &str) -> Self::TokenStream {
580-
unwrap_or_emit_fatal(source_str_to_stream(
585+
fn ts_from_str(&mut self, src: &str) -> Result<Self::TokenStream, String> {
586+
source_str_to_stream(
581587
self.psess(),
582588
FileName::proc_macro_source_code(src),
583589
src.to_string(),
584590
Some(self.call_site),
585-
))
591+
)
592+
.map_err(cancel_diags_into_string)
586593
}
587594

588595
fn ts_to_string(&mut self, stream: &Self::TokenStream) -> String {

‎library/proc_macro/src/bridge/mod.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,14 +37,14 @@ macro_rules! with_api {
3737
fn injected_env_var(var: &str) -> Option<String>;
3838
fn track_env_var(var: &str, value: Option<&str>);
3939
fn track_path(path: &str);
40-
fn literal_from_str(s: &str) -> Result<Literal<$Span, $Symbol>, ()>;
40+
fn literal_from_str(s: &str) -> Result<Literal<$Span, $Symbol>, String>;
4141
fn emit_diagnostic(diagnostic: Diagnostic<$Span>);
4242

4343
fn ts_drop(stream: $TokenStream);
4444
fn ts_clone(stream: &$TokenStream) -> $TokenStream;
4545
fn ts_is_empty(stream: &$TokenStream) -> bool;
4646
fn ts_expand_expr(stream: &$TokenStream) -> Result<$TokenStream, ()>;
47-
fn ts_from_str(src: &str) -> $TokenStream;
47+
fn ts_from_str(src: &str) -> Result<$TokenStream, String>;
4848
fn ts_to_string(stream: &$TokenStream) -> String;
4949
fn ts_from_token_tree(
5050
tree: TokenTree<$TokenStream, $Span, $Symbol>,

‎library/proc_macro/src/lib.rs‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -110,15 +110,18 @@ impl !Send for TokenStream {}
110110
impl !Sync for TokenStream {}
111111

112112
/// Error returned from `TokenStream::from_str`.
113+
///
114+
/// The contained error message is explicitly not guaranteed to be stable in any way,
115+
/// and may change between Rust versions or across compilations.
113116
#[stable(feature = "proc_macro_lib", since = "1.15.0")]
114117
#[non_exhaustive]
115118
#[derive(Debug)]
116-
pub struct LexError;
119+
pub struct LexError(String);
117120

118121
#[stable(feature = "proc_macro_lexerror_impls", since = "1.44.0")]
119122
impl fmt::Display for LexError {
120123
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
121-
f.write_str("cannot parse string into token stream")
124+
f.write_str(&self.0)
122125
}
123126
}
124127

@@ -197,7 +200,7 @@ impl FromStr for TokenStream {
197200
type Err = LexError;
198201

199202
fn from_str(src: &str) -> Result<TokenStream, LexError> {
200-
Ok(TokenStream(Some(BridgeMethods::ts_from_str(src))))
203+
Ok(TokenStream(Some(BridgeMethods::ts_from_str(src).map_err(LexError)?)))
201204
}
202205
}
203206

@@ -1594,7 +1597,7 @@ impl FromStr for Literal {
15941597
fn from_str(src: &str) -> Result<Self, LexError> {
15951598
match BridgeMethods::literal_from_str(src) {
15961599
Ok(literal) => Ok(Literal(literal)),
1597-
Err(()) => Err(LexError),
1600+
Err(msg) => Err(LexError(msg)),
15981601
}
15991602
}
16001603
}

‎src/tools/rust-analyzer/crates/proc-macro-srv/src/server_impl/rust_analyzer_span.rs‎

Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -62,8 +62,9 @@ impl server::Server for RaSpanServer<'_> {
6262
self.tracked_paths.insert(path.into());
6363
}
6464

65-
fn literal_from_str(&mut self, s: &str) -> Result<Literal<Self::Span>, ()> {
65+
fn literal_from_str(&mut self, s: &str) -> Result<Literal<Self::Span>, String> {
6666
literal_from_str(s, self.call_site)
67+
.map_err(|()| "cannot parse string into literal".to_string())
6768
}
6869

6970
fn emit_diagnostic(&mut self, _: Diagnostic<Self::Span>) {
@@ -81,14 +82,9 @@ impl server::Server for RaSpanServer<'_> {
8182
fn ts_is_empty(&mut self, stream: &Self::TokenStream) -> bool {
8283
stream.is_empty()
8384
}
84-
fn ts_from_str(&mut self, src: &str) -> Self::TokenStream {
85-
Self::TokenStream::from_str(src, self.call_site).unwrap_or_else(|e| {
86-
Self::TokenStream::from_str(
87-
&format!("compile_error!(\"failed to parse str to token stream: {e}\")"),
88-
self.call_site,
89-
)
90-
.unwrap()
91-
})
85+
fn ts_from_str(&mut self, src: &str) -> Result<Self::TokenStream, String> {
86+
Self::TokenStream::from_str(src, self.call_site)
87+
.map_err(|e| format!("failed to parse str to token stream: {e}"))
9288
}
9389
fn ts_to_string(&mut self, stream: &Self::TokenStream) -> String {
9490
stream.to_string()

‎src/tools/rust-analyzer/crates/proc-macro-srv/src/server_impl/token_id.rs‎

Lines changed: 5 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -67,8 +67,9 @@ impl server::Server for SpanIdServer<'_> {
6767
self.tracked_paths.insert(path.into());
6868
}
6969

70-
fn literal_from_str(&mut self, s: &str) -> Result<Literal<Self::Span>, ()> {
70+
fn literal_from_str(&mut self, s: &str) -> Result<Literal<Self::Span>, String> {
7171
literal_from_str(s, self.call_site)
72+
.map_err(|()| "cannot parse string into literal".to_string())
7273
}
7374

7475
fn emit_diagnostic(&mut self, _: Diagnostic<Self::Span>) {}
@@ -84,14 +85,9 @@ impl server::Server for SpanIdServer<'_> {
8485
fn ts_is_empty(&mut self, stream: &Self::TokenStream) -> bool {
8586
stream.is_empty()
8687
}
87-
fn ts_from_str(&mut self, src: &str) -> Self::TokenStream {
88-
Self::TokenStream::from_str(src, self.call_site).unwrap_or_else(|e| {
89-
Self::TokenStream::from_str(
90-
&format!("compile_error!(\"failed to parse str to token stream: {e}\")"),
91-
self.call_site,
92-
)
93-
.unwrap()
94-
})
88+
fn ts_from_str(&mut self, src: &str) -> Result<Self::TokenStream, String> {
89+
Self::TokenStream::from_str(src, self.call_site)
90+
.map_err(|e| format!("failed to parse str to token stream: {e}"))
9591
}
9692
fn ts_to_string(&mut self, stream: &Self::TokenStream) -> String {
9793
stream.to_string()

‎tests/ui/proc-macro/auxiliary/invalid-punct-ident.rs‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,5 +20,9 @@ pub fn invalid_raw_ident(_: TokenStream) -> TokenStream {
2020

2121
#[proc_macro]
2222
pub fn lexer_failure(_: TokenStream) -> TokenStream {
23-
"a b ) c".parse().expect("parsing failed without panic")
23+
assert_eq!(
24+
"a b ) c".parse::<TokenStream>().unwrap_err().to_string(),
25+
"unexpected closing delimiter: `)`"
26+
);
27+
TokenStream::new()
2428
}

‎tests/ui/proc-macro/auxiliary/nonfatal-parsing-body.rs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,10 @@ pub fn run() {
110110
lit("3//\n4", NormalErr);
111111
lit("18.u8E", NormalErr);
112112
lit("/*a*/ //", NormalErr);
113+
stream("1 ) 2", NormalErr);
114+
stream("( x [ ) ]", NormalErr);
115+
lit("1 ) 2", NormalErr);
116+
lit("( x [ ) ]", NormalErr);
113117
// FIXME: all of the cases below should return an Err and emit no diagnostics, but don't yet.
114118

115119
// emits diagnostics and returns LexError
@@ -122,8 +126,6 @@ pub fn run() {
122126

123127
for parse in [stream as fn(&str, Mode), lit] {
124128
// emits diagnostic(s), then panics
125-
parse("1 ) 2", OtherWithPanic);
126-
parse("( x [ ) ]", OtherWithPanic);
127129
parse("r#", OtherWithPanic);
128130

129131
// emits diagnostic(s), then returns Ok(Literal { kind: ErrWithGuar, .. })

‎tests/ui/proc-macro/invalid-punct-ident-4.rs‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,9 @@
11
//@ proc-macro: invalid-punct-ident.rs
2-
//@ needs-unwind proc macro panics to report errors
2+
//@ check-pass
33

44
#[macro_use]
55
extern crate invalid_punct_ident;
66

77
lexer_failure!();
8-
//~^ ERROR proc macro panicked
9-
//~| ERROR unexpected closing delimiter: `)`
108

11-
fn main() {
12-
let _recovery_witness: () = 0; //~ ERROR mismatched types
13-
}
9+
fn main() {}

‎tests/ui/proc-macro/invalid-punct-ident-4.stderr‎

Lines changed: 0 additions & 25 deletions
This file was deleted.

0 commit comments

Comments
 (0)