Skip to content

Commit e1357e2

Browse files
CppCXYCppCXY
committed
support underscore number and namedvararg for luajit3
1 parent 6c49c8d commit e1357e2

8 files changed

Lines changed: 67 additions & 34 deletions

File tree

‎crates/emmylua_parser/src/grammar/lua/expr.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
use crate::{
2-
SpecialFunction,
2+
LuaFeatures, SpecialFunction,
33
grammar::{ParseFailReason, ParseResult, lua::is_statement_start_token},
44
kind::{BinaryOperator, LuaOpKind, LuaSyntaxKind, LuaTokenKind, UNARY_PRIORITY, UnaryOperator},
55
parser::{LuaParser, MarkerEventContainer},
@@ -259,7 +259,7 @@ fn parse_param_name(p: &mut LuaParser, is_vararg: &mut bool) -> ParseResult {
259259
*is_vararg = true;
260260
p.bump();
261261
if token == LuaTokenKind::TkDots
262-
&& p.parse_config.support_named_var_args()
262+
&& p.parse_config.support(LuaFeatures::NamedVararg)
263263
&& p.current_token() == LuaTokenKind::TkName
264264
{
265265
p.bump();

‎crates/emmylua_parser/src/grammar/lua/stat.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
use crate::{
2-
LuaLanguageLevel,
2+
LuaFeatures, LuaLanguageLevel,
33
grammar::{ParseFailReason, ParseResult, lua::is_statement_start_token},
44
kind::{LuaSyntaxKind, LuaTokenKind},
55
parser::{CompleteMarker, LuaParser, MarkerEventContainer},
@@ -737,7 +737,7 @@ fn parse_attrib(p: &mut LuaParser) -> ParseResult {
737737
));
738738
}
739739
}
740-
if !p.parse_config.support_local_attrib() {
740+
if !p.parse_config.support(LuaFeatures::LocalAttrib) {
741741
p.errors.push(LuaParseError::syntax_error_from(
742742
&t!(
743743
"local attribute is not supported for current version: %{level}",

‎crates/emmylua_parser/src/kind/lua_features.rs‎

Lines changed: 20 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,24 @@
11
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
22
#[repr(u64)]
33
pub enum LuaFeatures {
4+
// std lua
45
Goto = 1, // "goto"
5-
ComplexNumber, // "0x1.2p3i"
6-
LLInteger, // "0LL"
7-
BinaryInteger, // "0b1010"
86
BitwiseOperation, // "5 & 2"
97
IntegerFloorDivision, // "5 // 2"
8+
LocalAttrib, // "local a<const> = 1"
109
GlobalDeclaration, // "global a = 1"
10+
NamedVararg, // "function f(a, b, c, ...args) end"
11+
1112
// non-standard symbols
1213
DoubleSlash, // "//"
1314
SlashStar, // "/**/"
1415

15-
// luajit2-extension symbols
16+
// luajit
17+
ComplexNumber, // "0x1.2p3i"
18+
LLInteger, // "0LL"
19+
BinaryInteger, // "0b1010"
20+
21+
// luajit2-extension
1622
PlusAssign, // "+="
1723
MinusAssign, // "-="
1824
StarAssign, // "*="
@@ -39,7 +45,9 @@ pub enum LuaFeatures {
3945

4046
// luajit3
4147
StringInterpolation, // "`"
42-
NilCoalescingAssign, // "??="
48+
// luajit not consider this syntax
49+
// NilCoalescingAssign, // "??="
50+
UnderscoreNumber, // "1_23"
4351
}
4452

4553
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
@@ -72,13 +80,15 @@ impl LuaFeaturesSet {
7280
}
7381

7482
pub fn features_lua54() -> Self {
75-
let set = LuaFeaturesSet::features_lua53();
83+
let mut set = LuaFeaturesSet::features_lua53();
84+
set.add(LuaFeatures::LocalAttrib);
7685
set
7786
}
7887

7988
pub fn features_lua55() -> Self {
8089
let mut set = LuaFeaturesSet::features_lua54();
8190
set.add(LuaFeatures::GlobalDeclaration);
91+
set.add(LuaFeatures::NamedVararg);
8292
set
8393
}
8494

@@ -122,12 +132,13 @@ impl LuaFeaturesSet {
122132

123133
pub fn features_luajit3() -> Self {
124134
let mut set = LuaFeaturesSet::features_luajit_extension();
125-
// lua5.3+
126-
set.add(LuaFeatures::IntegerFloorDivision);
127135

128136
// luajit3
129-
set.add(LuaFeatures::NilCoalescingAssign);
137+
set.add(LuaFeatures::IntegerFloorDivision);
138+
// set.add(LuaFeatures::NilCoalescingAssign);
130139
set.add(LuaFeatures::StringInterpolation);
140+
set.add(LuaFeatures::UnderscoreNumber);
141+
set.add(LuaFeatures::NamedVararg);
131142
set
132143
}
133144

‎crates/emmylua_parser/src/kind/lua_token_kind.rs‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -79,8 +79,7 @@ pub enum LuaTokenKind {
7979
TkShiftRightAssign, // >>=
8080
TkShrArithmeticAssign, // ~>>=
8181
TkConcatAssign, // ..=
82-
TkNilCoalescingAssign, // ??=
83-
82+
// TkNilCoalescingAssign, // ??=
8483
TkNilCoalescing, // ??
8584
TkSafeNavigation, // ?.
8685
TkTernary, // ?
@@ -263,7 +262,7 @@ impl LuaTokenKind {
263262
LuaTokenKind::TkTernary => "?",
264263
LuaTokenKind::TkShrArithmeticAssign => "~>>=",
265264
LuaTokenKind::TkConcatAssign => "..=",
266-
LuaTokenKind::TkNilCoalescingAssign => "??=",
265+
// LuaTokenKind::TkNilCoalescingAssign => "??=",
267266
_ => return None,
268267
})
269268
}
@@ -313,8 +312,7 @@ impl LuaTokenKind {
313312
| LuaTokenKind::TkShiftLeftAssign
314313
| LuaTokenKind::TkShiftRightAssign
315314
| LuaTokenKind::TkShrArithmeticAssign
316-
| LuaTokenKind::TkConcatAssign
317-
| LuaTokenKind::TkNilCoalescingAssign
315+
| LuaTokenKind::TkConcatAssign // | LuaTokenKind::TkNilCoalescingAssign
318316
)
319317
}
320318
}

‎crates/emmylua_parser/src/lexer/lua_lexer.rs‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -475,12 +475,13 @@ impl<'a> LuaLexer<'a> {
475475
'?' if self.support(LuaFeatures::NilCoalescingOperator) => {
476476
self.reader.bump();
477477

478-
if self.support(LuaFeatures::NilCoalescingAssign)
479-
&& self.reader.current_char() == '='
480-
{
481-
self.reader.bump();
482-
return LuaTokenKind::TkNilCoalescingAssign;
483-
}
478+
// luajit abandoned this syntax
479+
// if self.support(LuaFeatures::NilCoalescingAssign)
480+
// && self.reader.current_char() == '='
481+
// {
482+
// self.reader.bump();
483+
// return LuaTokenKind::TkNilCoalescingAssign;
484+
// }
484485
LuaTokenKind::TkNilCoalescing
485486
}
486487
'.' if self.support(LuaFeatures::SafeNavigationOperator) => {
@@ -645,6 +646,10 @@ impl<'a> LuaLexer<'a> {
645646

646647
while !self.reader.is_eof() {
647648
let ch = self.reader.current_char();
649+
if self.lexer_config.support(LuaFeatures::UnderscoreNumber) && ch == '_' {
650+
self.reader.bump();
651+
continue;
652+
}
648653
let continue_ = match state {
649654
NumberState::Int => match ch {
650655
'0'..='9' => true,

‎crates/emmylua_parser/src/parser/parser_config.rs‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ use std::collections::HashMap;
22

33
use rowan::NodeCache;
44

5-
use crate::{LuaFeaturesSet, kind::LuaLanguageLevel, lexer::LexerConfig};
5+
use crate::{LuaFeatures, LuaFeaturesSet, kind::LuaLanguageLevel, lexer::LexerConfig};
66

77
pub struct ParserConfig<'cache> {
88
pub level: LuaLanguageLevel,
@@ -33,18 +33,14 @@ impl<'cache> ParserConfig<'cache> {
3333
self.lexer_config
3434
}
3535

36-
pub fn support_local_attrib(&self) -> bool {
37-
self.level >= LuaLanguageLevel::Lua54
36+
pub fn support(&self, symbol: LuaFeatures) -> bool {
37+
self.lexer_config.support(symbol)
3838
}
3939

4040
pub fn support_emmylua_doc(&self) -> bool {
4141
self.enable_emmylua_doc
4242
}
4343

44-
pub fn support_named_var_args(&self) -> bool {
45-
self.level >= LuaLanguageLevel::Lua55
46-
}
47-
4844
pub fn node_cache(&mut self) -> Option<&mut NodeCache> {
4945
self.node_cache.as_deref_mut()
5046
}

‎crates/emmylua_parser/src/syntax/node/token/number_analyzer.rs‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,12 @@ use crate::{
66
};
77

88
pub fn float_token_value(token: &LuaSyntaxToken) -> Result<f64, LuaParseError> {
9-
let text = token.text();
9+
let raw_text = token.text();
10+
let text = if raw_text.contains('_') {
11+
raw_text.replace('_', "")
12+
} else {
13+
raw_text.to_string()
14+
};
1015
let hex = text.starts_with("0x") || text.starts_with("0X");
1116

1217
// This section handles the parsing of hexadecimal floating-point numbers.
@@ -55,7 +60,7 @@ pub fn float_token_value(token: &LuaSyntaxToken) -> Result<f64, LuaParseError> {
5560
if let Some(pos) = text.find('e').or_else(|| text.find('E')) {
5661
(&text[..pos], &text[(pos + 1)..])
5762
} else {
58-
(text, "")
63+
(text.as_str(), "")
5964
};
6065

6166
let mut value = float_part.parse::<f64>().map_err(|e| {
@@ -129,7 +134,12 @@ impl Display for NumberResult {
129134
}
130135

131136
pub fn int_token_value(token: &LuaSyntaxToken) -> Result<NumberResult, LuaParseError> {
132-
let text = token.text();
137+
let raw_text = token.text();
138+
let text = if raw_text.contains('_') {
139+
raw_text.replace('_', "")
140+
} else {
141+
raw_text.to_string()
142+
};
133143
let repr = if text.starts_with("0x") || text.starts_with("0X") {
134144
IntegerRepr::Hex
135145
} else if text.starts_with("0b") || text.starts_with("0B") {

‎crates/emmylua_parser/src/syntax/node/token/test.rs‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,10 @@ mod tests {
161161
test_float_token_value!(test_number_token_value_hex_float5, "0x1.8p-1", 0.75);
162162
test_float_token_value!(test_number_token_value_hex_float6, "0xABCDE2", 11259362.0);
163163

164+
test_float_token_value!(test_float_token_value_underscore, "1_000.5", 1000.5);
165+
test_float_token_value!(test_float_token_value_underscore_2, "1_000.5e2", 100050.0);
166+
test_float_token_value!(test_float_token_value_underscore_hex, "0x1_0.0p0", 16.0);
167+
164168
macro_rules! test_int_token_value {
165169
($name:ident, $code:expr, $expected:expr) => {
166170
#[test]
@@ -181,4 +185,13 @@ mod tests {
181185
test_int_token_value!(test_number_token_value_int7, "0x0", 0);
182186
test_int_token_value!(test_number_token_value_int8, "0x11LL", 17);
183187
test_int_token_value!(test_number_token_value_int9, "0b10101", 21);
188+
189+
test_int_token_value!(test_number_token_value_int_underscore, "1_000", 1000);
190+
test_int_token_value!(
191+
test_number_token_value_int_underscore_2,
192+
"1_000_000",
193+
1000000
194+
);
195+
test_int_token_value!(test_number_token_value_int_underscore_hex, "0xFF_FF", 65535);
196+
test_int_token_value!(test_number_token_value_int_underscore_bin, "0b1_0", 2);
184197
}

0 commit comments

Comments
 (0)