Skip to content

Commit 5f585bb

Browse files
ghaithclaude
ghaith
and
claude
committed
fix(parser): recover from unexpected tokens in expression position (#1306)
`parse_atomic_leaf_expression`'s fallback emitted an `E007` diagnostic but left the bad token in the stream, so binary-op parsers higher in the cascade re-consumed it (e.g. `&` taken as binary AND with an `EmptyStatement` LHS). The malformed AST flowed into codegen, producing a confusing "no type hint available for EmptyStatement" while compilation reported success. `prog(p := &g)` from #1306 was the salient case; any binary-only operator misused as a unary prefix had the same shape. Recovery now: - emits a clearer "expected expression" diagnostic (was "Literal"); - when the bad token is itself an operator, advances past it and retries the leaf so the operand the user wrote is captured (`&g` -> `g`); - otherwise leaves the token in place so outer parsers can use it for synchronization (e.g. `END_CASE`, `END_VAR`). The missing-parameter-assignment carve-out (`foo(p := )`) is narrowed to fire only when the next token is actually `)` or `,`; previously any unrecognised token after `:=` was silently treated as an empty parameter, masking real errors. Resolver tests that exercised pointer arithmetic via the dead-code `&x` path are migrated to `REF(x)`. Fixes #1306 Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
1 parent 689757d commit 5f585bb

18 files changed

Lines changed: 194 additions & 46 deletions

File tree

‎src/parser/expressions_parser.rs‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -327,17 +327,28 @@ fn parse_atomic_leaf_expression(lexer: &mut ParseSession<'_>) -> Option<AstNode>
327327
_ => {
328328
if lexer.closing_keywords.contains(&vec![KeywordParensClose])
329329
&& matches!(lexer.last_token, KeywordOutputAssignment | KeywordAssignment)
330+
&& matches!(lexer.token, KeywordParensClose | KeywordComma)
330331
{
331332
// due to closing keyword ')' and last_token '=>' / ':='
332333
// we are probably in a call statement missing a parameter assignment 'foo(param := );
333334
// optional parameter assignments are allowed, validation should handle any unwanted cases
334335
Some(AstFactory::create_empty_statement(lexer.location(), lexer.next_id()))
335336
} else {
336337
lexer.accept_diagnostic(Diagnostic::unexpected_token_found(
337-
"Literal",
338+
"expression",
338339
lexer.slice(),
339340
lexer.location(),
340341
));
342+
// If the bad token is a binary-only operator misused as a
343+
// prefix (e.g. `&y`, `MOD y`), consume it and retry the leaf
344+
// so the recovered AST captures the operand the user wrote.
345+
// For non-operator tokens (e.g. END_CASE, END_VAR), leave the
346+
// token in the stream so outer parsers can use it for
347+
// synchronization.
348+
if to_operator(&lexer.token).is_some() {
349+
lexer.advance();
350+
return parse_atomic_leaf_expression(lexer);
351+
}
341352
None
342353
}
343354
}

‎src/parser/tests/expressions_parser_tests.rs‎

Lines changed: 142 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
// Copyright (c) 2020 Ghaith Hachem and Mathias Rieder
22
use crate::parser::tests::ref_to;
3-
use crate::test_utils::tests::parse;
3+
use crate::test_utils::tests::{parse, parse_buffered};
44
use insta::{assert_debug_snapshot, assert_snapshot};
55
use plc_ast::ast::{Assignment, AstFactory, AstNode, AstStatement, Operator};
66
use plc_ast::literals::AstLiteral;
@@ -1014,6 +1014,146 @@ fn amp_as_and_test() {
10141014
assert_debug_snapshot!(statement);
10151015
}
10161016

1017+
// Recovery for unexpected tokens in expression position.
1018+
// `&` (and other binary-only operators) used as a prefix used to leave the
1019+
// bad token in the stream; binary-AND parsing then re-consumed it and built
1020+
// `EmptyStatement & rhs`, which crashed codegen with E071. The leaf-fallback
1021+
// now advances past the bad token after diagnosing it, so the cascade above
1022+
// cannot misinterpret it.
1023+
1024+
#[test]
1025+
fn amp_as_address_of_in_call_arg_recovers() {
1026+
let src = "
1027+
VAR_GLOBAL g : DINT; END_VAR
1028+
FUNCTION foo : DINT VAR_INPUT p : DINT; END_VAR foo := p; END_FUNCTION
1029+
PROGRAM main foo(p := &g); END_PROGRAM
1030+
";
1031+
let (unit, diagnostics) = parse_buffered(src);
1032+
assert_snapshot!(diagnostics, @r"
1033+
error[E007]: Unexpected token: expected expression but found &
1034+
┌─ <internal>:4:31
1035+
│
1036+
4 │ PROGRAM main foo(p := &g); END_PROGRAM
1037+
│ ^ Unexpected token: expected expression but found &
1038+
");
1039+
let main = unit.implementations.iter().find(|i| i.name == "main").unwrap();
1040+
assert_debug_snapshot!(main.statements, @r#"
1041+
[
1042+
CallStatement {
1043+
operator: ReferenceExpr {
1044+
kind: Member(
1045+
Identifier {
1046+
name: "foo",
1047+
},
1048+
),
1049+
base: None,
1050+
},
1051+
parameters: Some(
1052+
Assignment {
1053+
left: ReferenceExpr {
1054+
kind: Member(
1055+
Identifier {
1056+
name: "p",
1057+
},
1058+
),
1059+
base: None,
1060+
},
1061+
right: ReferenceExpr {
1062+
kind: Member(
1063+
Identifier {
1064+
name: "g",
1065+
},
1066+
),
1067+
base: None,
1068+
},
1069+
},
1070+
),
1071+
},
1072+
]
1073+
"#);
1074+
}
1075+
1076+
#[test]
1077+
fn amp_as_unary_prefix_in_assignment_recovers() {
1078+
let src = "
1079+
PROGRAM prg
1080+
VAR x : DINT; y : DINT; END_VAR
1081+
x := &y;
1082+
END_PROGRAM
1083+
";
1084+
let (unit, diagnostics) = parse_buffered(src);
1085+
assert_snapshot!(diagnostics, @r"
1086+
error[E007]: Unexpected token: expected expression but found &
1087+
┌─ <internal>:4:18
1088+
│
1089+
4 │ x := &y;
1090+
│ ^ Unexpected token: expected expression but found &
1091+
");
1092+
assert_debug_snapshot!(unit.implementations[0].statements, @r#"
1093+
[
1094+
Assignment {
1095+
left: ReferenceExpr {
1096+
kind: Member(
1097+
Identifier {
1098+
name: "x",
1099+
},
1100+
),
1101+
base: None,
1102+
},
1103+
right: ReferenceExpr {
1104+
kind: Member(
1105+
Identifier {
1106+
name: "y",
1107+
},
1108+
),
1109+
base: None,
1110+
},
1111+
},
1112+
]
1113+
"#);
1114+
}
1115+
1116+
#[test]
1117+
fn binary_only_operator_as_prefix_recovers() {
1118+
// Generality witness — recovery is not specific to `&`.
1119+
let src = "
1120+
PROGRAM prg
1121+
VAR x : DINT; y : DINT; END_VAR
1122+
x := MOD y;
1123+
END_PROGRAM
1124+
";
1125+
let (unit, diagnostics) = parse_buffered(src);
1126+
assert_snapshot!(diagnostics, @r"
1127+
error[E007]: Unexpected token: expected expression but found MOD
1128+
┌─ <internal>:4:18
1129+
│
1130+
4 │ x := MOD y;
1131+
│ ^^^ Unexpected token: expected expression but found MOD
1132+
");
1133+
assert_debug_snapshot!(unit.implementations[0].statements, @r#"
1134+
[
1135+
Assignment {
1136+
left: ReferenceExpr {
1137+
kind: Member(
1138+
Identifier {
1139+
name: "x",
1140+
},
1141+
),
1142+
base: None,
1143+
},
1144+
right: ReferenceExpr {
1145+
kind: Member(
1146+
Identifier {
1147+
name: "y",
1148+
},
1149+
),
1150+
base: None,
1151+
},
1152+
},
1153+
]
1154+
"#);
1155+
}
1156+
10171157
#[test]
10181158
fn and_then_test() {
10191159
let src = "
@@ -1635,7 +1775,7 @@ fn reference_location_test() {
16351775

16361776
#[test]
16371777
fn qualified_reference_location_test() {
1638-
let source = "PROGRAM prg a.b.c;aa.bb.cc[2];aaa.bbb.ccc^;&aaa.bbb.ccc; END_PROGRAM";
1778+
let source = "PROGRAM prg a.b.c;aa.bb.cc[2];aaa.bbb.ccc^; END_PROGRAM";
16391779
let parse_result = parse(source).0;
16401780

16411781
let unit = &parse_result.implementations[0];
@@ -1648,9 +1788,6 @@ fn qualified_reference_location_test() {
16481788

16491789
let location = &unit.statements[2].get_location();
16501790
assert_eq!(source[location.to_range().unwrap()].to_string(), "aaa.bbb.ccc^");
1651-
1652-
let location = &unit.statements[3].get_location();
1653-
assert_eq!(source[location.to_range().unwrap()].to_string(), "&aaa.bbb.ccc");
16541791
}
16551792

16561793
#[test]

‎src/parser/tests/parse_errors/parse_error_containers_tests.rs‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -189,7 +189,7 @@ fn super_is_a_reserved_keyword() {
189189
// Related: https://github.com/PLC-lang/rusty/issues/1408
190190

191191
let diagnostics = parse_and_report_parse_errors_buffered(src);
192-
assert_snapshot!(diagnostics, @r"
192+
assert_snapshot!(diagnostics, @"
193193
error[E006]: Expected a name for the interface definition but got nothing
194194
┌─ <internal>:2:5
195195
│
@@ -229,11 +229,11 @@ fn super_is_a_reserved_keyword() {
229229
│ ╰─────────────────^ Unexpected token: expected KeywordSemicolon but found 'VAR
230230
super'
231231
232-
error[E007]: Unexpected token: expected Literal but found END_VAR
232+
error[E007]: Unexpected token: expected expression but found END_VAR
233233
┌─ <internal>:6:9
234234
│
235235
6 │ END_VAR
236-
│ ^^^^^^^ Unexpected token: expected Literal but found END_VAR
236+
│ ^^^^^^^ Unexpected token: expected expression but found END_VAR
237237
238238
error[E007]: Unexpected token: expected KeywordSemicolon but found 'END_VAR
239239
METHOD super END_METHOD'
@@ -273,7 +273,7 @@ fn this_is_a_reserved_keyword() {
273273
// TODO(mhasel): the parser produces a lot of noise for keyword errors,
274274
// we need to find a way to handle keywords as identifiers
275275
let diagnostics = parse_and_validate_buffered(src);
276-
assert_snapshot!(diagnostics, @r"
276+
assert_snapshot!(diagnostics, @"
277277
error[E006]: Expected a name for the interface definition but got nothing
278278
┌─ <internal>:2:5
279279
│
@@ -313,11 +313,11 @@ fn this_is_a_reserved_keyword() {
313313
│ ╰────────────────^ Unexpected token: expected KeywordSemicolon but found 'VAR
314314
this'
315315
316-
error[E007]: Unexpected token: expected Literal but found END_VAR
316+
error[E007]: Unexpected token: expected expression but found END_VAR
317317
┌─ <internal>:6:9
318318
│
319319
6 │ END_VAR
320-
│ ^^^^^^^ Unexpected token: expected Literal but found END_VAR
320+
│ ^^^^^^^ Unexpected token: expected expression but found END_VAR
321321
322322
error[E007]: Unexpected token: expected KeywordSemicolon but found 'END_VAR
323323
METHOD this END_METHOD'

‎src/parser/tests/parse_errors/snapshots/rusty__parser__tests__parse_errors__parse_error_classes_tests__method_with_invalid_return_type.snap‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,11 @@ error[E007]: Unexpected token: expected Datatype but found ABSTRACT
1414
1 │ CLASS TestClass METHOD foo : ABSTRACT END_METHOD END_CLASS
1515
│ ^^^^^^^^ Unexpected token: expected Datatype but found ABSTRACT
1616

17-
error[E007]: Unexpected token: expected Literal but found ABSTRACT
17+
error[E007]: Unexpected token: expected expression but found ABSTRACT
1818
┌─ <internal>:1:30
1919
│
2020
1 │ CLASS TestClass METHOD foo : ABSTRACT END_METHOD END_CLASS
21-
│ ^^^^^^^^ Unexpected token: expected Literal but found ABSTRACT
21+
│ ^^^^^^^^ Unexpected token: expected expression but found ABSTRACT
2222

2323
error[E007]: Unexpected token: expected KeywordSemicolon but found 'ABSTRACT'
2424
┌─ <internal>:1:30

‎src/parser/tests/parse_errors/snapshots/rusty__parser__tests__parse_errors__parse_error_containers_tests__missing_pou_name_2.snap‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,11 @@
22
source: src/parser/tests/parse_errors/parse_error_containers_tests.rs
33
expression: diagnostics
44
---
5-
error[E007]: Unexpected token: expected Literal but found :=
5+
error[E007]: Unexpected token: expected expression but found :=
66
┌─ <internal>:3:15
77
│
88
3 │ a := 2;
9-
│ ^^ Unexpected token: expected Literal but found :=
9+
│ ^^ Unexpected token: expected expression but found :=
1010

1111
error[E007]: Unexpected token: expected KeywordSemicolon but found ':= 2'
1212
┌─ <internal>:3:15

‎src/parser/tests/parse_errors/snapshots/rusty__parser__tests__parse_errors__parse_error_literals_tests__illegal_literal_time_missing_segments_test.snap‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@
22
source: src/parser/tests/parse_errors/parse_error_literals_tests.rs
33
expression: diagnostics
44
---
5-
error[E007]: Unexpected token: expected Literal but found ;
5+
error[E007]: Unexpected token: expected expression but found ;
66
┌─ <internal>:3:15
77
│
88
3 │ T#;
9-
│ ^ Unexpected token: expected Literal but found ;
9+
│ ^ Unexpected token: expected expression but found ;

‎src/parser/tests/parse_errors/snapshots/rusty__parser__tests__parse_errors__parse_error_messages_test__case_with_unexpected_token.snap‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,11 @@ error[E007]: Unexpected token: expected KeywordSemicolon but found '1'
1414
4 │ 1: x;
1515
│ ^ Unexpected token: expected KeywordSemicolon but found '1'
1616

17-
error[E007]: Unexpected token: expected Literal but found END_CASE
17+
error[E007]: Unexpected token: expected expression but found END_CASE
1818
┌─ <internal>:5:9
1919
│
2020
5 │ END_CASE
21-
│ ^^^^^^^^ Unexpected token: expected Literal but found END_CASE
21+
│ ^^^^^^^^ Unexpected token: expected expression but found END_CASE
2222

2323
error[E007]: Unexpected token: expected KeywordSemicolon but found 'END_CASE'
2424
┌─ <internal>:5:9

‎src/parser/tests/parse_errors/snapshots/rusty__parser__tests__parse_errors__parse_error_messages_test__for_with_unexpected_token_1.snap‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,11 @@ error[E007]: Unexpected token: expected KeywordSemicolon but found 'x TO y DO
1818
│ ╰─────────────^ Unexpected token: expected KeywordSemicolon but found 'x TO y DO
1919
x'
2020

21-
error[E007]: Unexpected token: expected Literal but found END_FOR
21+
error[E007]: Unexpected token: expected expression but found END_FOR
2222
┌─ <internal>:6:9
2323
│
2424
6 │ END_FOR
25-
│ ^^^^^^^ Unexpected token: expected Literal but found END_FOR
25+
│ ^^^^^^^ Unexpected token: expected expression but found END_FOR
2626

2727
error[E007]: Unexpected token: expected KeywordSemicolon but found 'END_FOR'
2828
┌─ <internal>:6:9

‎src/parser/tests/parse_errors/snapshots/rusty__parser__tests__parse_errors__parse_error_messages_test__for_with_unexpected_token_2.snap‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,11 @@ error[E007]: Unexpected token: expected KeywordSemicolon but found 'y DO
1818
│ ╰─────────────^ Unexpected token: expected KeywordSemicolon but found 'y DO
1919
x'
2020

21-
error[E007]: Unexpected token: expected Literal but found END_FOR
21+
error[E007]: Unexpected token: expected expression but found END_FOR
2222
┌─ <internal>:6:9
2323
│
2424
6 │ END_FOR
25-
│ ^^^^^^^ Unexpected token: expected Literal but found END_FOR
25+
│ ^^^^^^^ Unexpected token: expected expression but found END_FOR
2626

2727
error[E007]: Unexpected token: expected KeywordSemicolon but found 'END_FOR'
2828
┌─ <internal>:6:9

‎src/parser/tests/parse_errors/snapshots/rusty__parser__tests__parse_errors__parse_error_messages_test__if_then_with_unexpected_token.snap‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,11 @@ error[E007]: Unexpected token: expected KeywordSemicolon but found 'x'
1414
4 │ x;
1515
│ ^ Unexpected token: expected KeywordSemicolon but found 'x'
1616

17-
error[E007]: Unexpected token: expected Literal but found ELSE
17+
error[E007]: Unexpected token: expected expression but found ELSE
1818
┌─ <internal>:5:9
1919
│
2020
5 │ ELSE
21-
│ ^^^^ Unexpected token: expected Literal but found ELSE
21+
│ ^^^^ Unexpected token: expected expression but found ELSE
2222

2323
error[E007]: Unexpected token: expected KeywordSemicolon but found 'ELSE
2424
y'
@@ -29,11 +29,11 @@ error[E007]: Unexpected token: expected KeywordSemicolon but found 'ELSE
2929
│ ╰─────────────^ Unexpected token: expected KeywordSemicolon but found 'ELSE
3030
y'
3131

32-
error[E007]: Unexpected token: expected Literal but found END_IF
32+
error[E007]: Unexpected token: expected expression but found END_IF
3333
┌─ <internal>:7:9
3434
│
3535
7 │ END_IF
36-
│ ^^^^^^ Unexpected token: expected Literal but found END_IF
36+
│ ^^^^^^ Unexpected token: expected expression but found END_IF
3737

3838
error[E007]: Unexpected token: expected KeywordSemicolon but found 'END_IF'
3939
┌─ <internal>:7:9

0 commit comments

Comments
 (0)