Skip to content

Commit 4fef40b

Browse files
sideshowbarkersideshowbarker
committed
Address review comments
- Read :def parameter and :var/:final variable names with read_word(), not read_alpha() — so digits and underscores in names are accepted. - Capture read_type() return values for :def parameters into varnode.type_str — so the parsed type is preserved on the AST, rather than getting discarded. - Require whitespace immediately before # in separate_nextcmd — to match Vim9 inline-comment rules. - Strip a trailing comment from :import lines — so the comment text is no longer captured in the import spec. (Implemented with a streaming reader rather than substitute()/slicing — so the Python and JavaScript transpilers can translate the code without new helpers.) - Emit type_str in compile_var and compile_final output when present — so types in declarations like “var count: number = 42” round-trip through the S-expression form as (var = count: number 42). - Rename the VAR/FINAL doc-comment fields from .type to .type_str — so they match the actual node field names.
1 parent 75b8305 commit 4fef40b

7 files changed

Lines changed: 117 additions & 28 deletions

File tree

‎autoload/vimlparser.vim‎

Lines changed: 39 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -432,8 +432,8 @@ endfunction
432432
" HEREDOC .rlist .op .body
433433
" DEF .ea .body .left .rlist .default_args .attr .enddef
434434
" ENDDEF .ea
435-
" VAR .ea .op .left .list .rest .right .type
436-
" FINAL .ea .op .left .list .rest .right .type
435+
" VAR .ea .op .left .list .rest .right .type_str
436+
" FINAL .ea .op .left .list .rest .right .type_str
437437
" EXPORT .ea .body
438438
" IMPORT .ea .str .left
439439
" VIM9SCRIPT .ea
@@ -1225,7 +1225,7 @@ function! s:VimLParser.separate_nextcmd() abort
12251225
\ || self.reader.getpos() !=# self.ea.argpos)
12261226
\ && (self.ea.cmd.name !=# 'redir'
12271227
\ || self.reader.getpos().i !=# self.ea.argpos.i + 1 || pc !=# '@'))
1228-
\ || (self.vim9script && c ==# '#' && self.ea.cmd.flags !~# '\<NOTRLCOM\>')
1228+
\ || (self.vim9script && c ==# '#' && s:iswhite(pc) && self.ea.cmd.flags !~# '\<NOTRLCOM\>')
12291229
let has_cpo_bar = s:FALSE " &cpoptions =~ 'b'
12301230
if (!has_cpo_bar || self.ea.cmd.flags !~# '\<USECTRLV\>') && pc ==# '\'
12311231
call self.reader.get()
@@ -1785,19 +1785,20 @@ function! s:VimLParser.parse_cmd_def() abort
17851785
while s:TRUE
17861786
let varnode = s:Node(s:NODE_IDENTIFIER)
17871787
call self.reader.skip_white()
1788+
let varnode.type_str = ''
17881789
" Check for ...name (variadic)
17891790
if self.reader.peekn(3) ==# '...'
17901791
call self.reader.getn(3)
17911792
call self.reader.skip_white()
1792-
let vname = self.reader.read_alpha()
1793+
let vname = self.reader.read_word()
17931794
let varnode.pos = self.reader.getpos()
17941795
let varnode.value = '...' . vname
17951796
" Optionally read type annotation
17961797
call self.reader.skip_white()
17971798
if self.reader.peekn(1) ==# ':'
17981799
call self.reader.getn(1)
17991800
call self.reader.skip_white()
1800-
call self.read_type()
1801+
let varnode.type_str = self.read_type()
18011802
endif
18021803
call add(node.rlist, varnode)
18031804
call self.reader.skip_white()
@@ -1809,7 +1810,7 @@ function! s:VimLParser.parse_cmd_def() abort
18091810
endif
18101811
endif
18111812
let npos = self.reader.getpos()
1812-
let pname = self.reader.read_alpha()
1813+
let pname = self.reader.read_word()
18131814
if pname ==# ''
18141815
if self.reader.peekn(1) ==# ')'
18151816
call self.reader.getn(1)
@@ -1829,7 +1830,7 @@ function! s:VimLParser.parse_cmd_def() abort
18291830
if self.reader.peekn(1) ==# ':'
18301831
call self.reader.getn(1)
18311832
call self.reader.skip_white()
1832-
call self.read_type()
1833+
let varnode.type_str = self.read_type()
18331834
endif
18341835
" Check for default value
18351836
call self.reader.skip_white()
@@ -2101,8 +2102,31 @@ function! s:VimLParser.parse_cmd_import() abort
21012102
let node.left = s:NIL
21022103
let node.str = ''
21032104
call self.reader.skip_white()
2104-
" Read the rest of the import line as a string (import autoload 'path' or import 'path' as Name)
2105-
let node.str = self.reader.getn(-1)
2105+
" Read the rest of the import line, stopping at a trailing comment.
2106+
let line = ''
2107+
let pending_ws = ''
2108+
let last_was_white = s:FALSE
2109+
while s:TRUE
2110+
let c = self.reader.peekn(1)
2111+
if c ==# ''
2112+
break
2113+
endif
2114+
if c ==# '#' && last_was_white
2115+
" Consume the rest of the line, so parse_trail doesn't see it.
2116+
call self.reader.getn(-1)
2117+
break
2118+
endif
2119+
call self.reader.getn(1)
2120+
if s:iswhite(c)
2121+
let pending_ws .= c
2122+
let last_was_white = s:TRUE
2123+
else
2124+
let line .= pending_ws . c
2125+
let pending_ws = ''
2126+
let last_was_white = s:FALSE
2127+
endif
2128+
endwhile
2129+
let node.str = line
21062130
call self.add_node(node)
21072131
endfunction
21082132

@@ -6275,6 +6299,9 @@ function! s:Compiler.compile_var(node) abort
62756299
endif
62766300
let left = '(' . left . ')'
62776301
endif
6302+
if a:node.type_str !=# ''
6303+
let left .= ': ' . a:node.type_str
6304+
endif
62786305
let right = self.compile(a:node.right)
62796306
call self.out('(var %s %s %s)', a:node.op, left, right)
62806307
endfunction
@@ -6290,6 +6317,9 @@ function! s:Compiler.compile_final(node) abort
62906317
endif
62916318
let left = '(' . left . ')'
62926319
endif
6320+
if a:node.type_str !=# ''
6321+
let left .= ': ' . a:node.type_str
6322+
endif
62936323
let right = self.compile(a:node.right)
62946324
call self.out('(final %s %s %s)', a:node.op, left, right)
62956325
endfunction

‎js/vimlparser.js‎

Lines changed: 40 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -610,8 +610,8 @@ function ExArg() {
610610
// HEREDOC .rlist .op .body
611611
// DEF .ea .body .left .rlist .default_args .attr .enddef
612612
// ENDDEF .ea
613-
// VAR .ea .op .left .list .rest .right .type
614-
// FINAL .ea .op .left .list .rest .right .type
613+
// VAR .ea .op .left .list .rest .right .type_str
614+
// FINAL .ea .op .left .list .rest .right .type_str
615615
// EXPORT .ea .body
616616
// IMPORT .ea .str .left
617617
// VIM9SCRIPT .ea
@@ -1513,7 +1513,7 @@ VimLParser.prototype.separate_nextcmd = function() {
15131513
}
15141514
this.reader.getn(1);
15151515
}
1516-
else if (c == "|" || c == "\n" || c == "\"" && !viml_eqregh(this.ea.cmd.flags, "\\<NOTRLCOM\\>") && (this.ea.cmd.name != "@" && this.ea.cmd.name != "*" || this.reader.getpos() != this.ea.argpos) && (this.ea.cmd.name != "redir" || this.reader.getpos().i != this.ea.argpos.i + 1 || pc != "@") || this.vim9script && c == "#" && !viml_eqregh(this.ea.cmd.flags, "\\<NOTRLCOM\\>")) {
1516+
else if (c == "|" || c == "\n" || c == "\"" && !viml_eqregh(this.ea.cmd.flags, "\\<NOTRLCOM\\>") && (this.ea.cmd.name != "@" && this.ea.cmd.name != "*" || this.reader.getpos() != this.ea.argpos) && (this.ea.cmd.name != "redir" || this.reader.getpos().i != this.ea.argpos.i + 1 || pc != "@") || this.vim9script && c == "#" && iswhite(pc) && !viml_eqregh(this.ea.cmd.flags, "\\<NOTRLCOM\\>")) {
15171517
var has_cpo_bar = FALSE;
15181518
// &cpoptions =~ 'b'
15191519
if ((!has_cpo_bar || !viml_eqregh(this.ea.cmd.flags, "\\<USECTRLV\\>")) && pc == "\\") {
@@ -2079,19 +2079,20 @@ VimLParser.prototype.parse_cmd_def = function() {
20792079
while (TRUE) {
20802080
var varnode = Node(NODE_IDENTIFIER);
20812081
this.reader.skip_white();
2082+
varnode.type_str = "";
20822083
// Check for ...name (variadic)
20832084
if (this.reader.peekn(3) == "...") {
20842085
this.reader.getn(3);
20852086
this.reader.skip_white();
2086-
var vname = this.reader.read_alpha();
2087+
var vname = this.reader.read_word();
20872088
varnode.pos = this.reader.getpos();
20882089
varnode.value = "..." + vname;
20892090
// Optionally read type annotation
20902091
this.reader.skip_white();
20912092
if (this.reader.peekn(1) == ":") {
20922093
this.reader.getn(1);
20932094
this.reader.skip_white();
2094-
this.read_type();
2095+
varnode.type_str = this.read_type();
20952096
}
20962097
viml_add(node.rlist, varnode);
20972098
this.reader.skip_white();
@@ -2104,7 +2105,7 @@ VimLParser.prototype.parse_cmd_def = function() {
21042105
}
21052106
}
21062107
var npos = this.reader.getpos();
2107-
var pname = this.reader.read_alpha();
2108+
var pname = this.reader.read_word();
21082109
if (pname == "") {
21092110
if (this.reader.peekn(1) == ")") {
21102111
this.reader.getn(1);
@@ -2124,7 +2125,7 @@ VimLParser.prototype.parse_cmd_def = function() {
21242125
if (this.reader.peekn(1) == ":") {
21252126
this.reader.getn(1);
21262127
this.reader.skip_white();
2127-
this.read_type();
2128+
varnode.type_str = this.read_type();
21282129
}
21292130
// Check for default value
21302131
this.reader.skip_white();
@@ -2394,8 +2395,32 @@ VimLParser.prototype.parse_cmd_import = function() {
23942395
node.left = NIL;
23952396
node.str = "";
23962397
this.reader.skip_white();
2397-
// Read the rest of the import line as a string (import autoload 'path' or import 'path' as Name)
2398-
node.str = this.reader.getn(-1);
2398+
// Read the rest of the import line, stopping at a trailing comment.
2399+
var line = "";
2400+
var pending_ws = "";
2401+
var last_was_white = FALSE;
2402+
while (TRUE) {
2403+
var c = this.reader.peekn(1);
2404+
if (c == "") {
2405+
break;
2406+
}
2407+
if (c == "#" && last_was_white) {
2408+
// Consume the rest of the line, so parse_trail doesn't see it.
2409+
this.reader.getn(-1);
2410+
break;
2411+
}
2412+
this.reader.getn(1);
2413+
if (iswhite(c)) {
2414+
pending_ws += c;
2415+
var last_was_white = TRUE;
2416+
}
2417+
else {
2418+
line += pending_ws + c;
2419+
var pending_ws = "";
2420+
var last_was_white = FALSE;
2421+
}
2422+
}
2423+
node.str = line;
23992424
this.add_node(node);
24002425
}
24012426

@@ -5725,6 +5750,9 @@ Compiler.prototype.compile_var = function(node) {
57255750
}
57265751
var left = "(" + left + ")";
57275752
}
5753+
if (node.type_str != "") {
5754+
left += ": " + node.type_str;
5755+
}
57285756
var right = this.compile(node.right);
57295757
this.out("(var %s %s %s)", node.op, left, right);
57305758
}
@@ -5741,6 +5769,9 @@ Compiler.prototype.compile_final = function(node) {
57415769
}
57425770
var left = "(" + left + ")";
57435771
}
5772+
if (node.type_str != "") {
5773+
left += ": " + node.type_str;
5774+
}
57445775
var right = this.compile(node.right);
57455776
this.out("(final %s %s %s)", node.op, left, right);
57465777
}

‎py/vimlparser.py‎

Lines changed: 33 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -599,8 +599,8 @@ def ExArg():
599599
# HEREDOC .rlist .op .body
600600
# DEF .ea .body .left .rlist .default_args .attr .enddef
601601
# ENDDEF .ea
602-
# VAR .ea .op .left .list .rest .right .type
603-
# FINAL .ea .op .left .list .rest .right .type
602+
# VAR .ea .op .left .list .rest .right .type_str
603+
# FINAL .ea .op .left .list .rest .right .type_str
604604
# EXPORT .ea .body
605605
# IMPORT .ea .str .left
606606
# VIM9SCRIPT .ea
@@ -1256,7 +1256,7 @@ def separate_nextcmd(self):
12561256
if c != "`":
12571257
raise VimLParserException(Err(viml_printf("unexpected character: %s", c), self.reader.getpos()))
12581258
self.reader.getn(1)
1259-
elif c == "|" or c == "\n" or c == "\"" and not viml_eqregh(self.ea.cmd.flags, "\\<NOTRLCOM\\>") and (self.ea.cmd.name != "@" and self.ea.cmd.name != "*" or self.reader.getpos() != self.ea.argpos) and (self.ea.cmd.name != "redir" or self.reader.getpos().i != self.ea.argpos.i + 1 or pc != "@") or self.vim9script and c == "#" and not viml_eqregh(self.ea.cmd.flags, "\\<NOTRLCOM\\>"):
1259+
elif c == "|" or c == "\n" or c == "\"" and not viml_eqregh(self.ea.cmd.flags, "\\<NOTRLCOM\\>") and (self.ea.cmd.name != "@" and self.ea.cmd.name != "*" or self.reader.getpos() != self.ea.argpos) and (self.ea.cmd.name != "redir" or self.reader.getpos().i != self.ea.argpos.i + 1 or pc != "@") or self.vim9script and c == "#" and iswhite(pc) and not viml_eqregh(self.ea.cmd.flags, "\\<NOTRLCOM\\>"):
12601260
has_cpo_bar = FALSE
12611261
# &cpoptions =~ 'b'
12621262
if (not has_cpo_bar or not viml_eqregh(self.ea.cmd.flags, "\\<USECTRLV\\>")) and pc == "\\":
@@ -1713,19 +1713,20 @@ def parse_cmd_def(self):
17131713
while TRUE:
17141714
varnode = Node(NODE_IDENTIFIER)
17151715
self.reader.skip_white()
1716+
varnode.type_str = ""
17161717
# Check for ...name (variadic)
17171718
if self.reader.peekn(3) == "...":
17181719
self.reader.getn(3)
17191720
self.reader.skip_white()
1720-
vname = self.reader.read_alpha()
1721+
vname = self.reader.read_word()
17211722
varnode.pos = self.reader.getpos()
17221723
varnode.value = "..." + vname
17231724
# Optionally read type annotation
17241725
self.reader.skip_white()
17251726
if self.reader.peekn(1) == ":":
17261727
self.reader.getn(1)
17271728
self.reader.skip_white()
1728-
self.read_type()
1729+
varnode.type_str = self.read_type()
17291730
viml_add(node.rlist, varnode)
17301731
self.reader.skip_white()
17311732
if self.reader.peekn(1) == ")":
@@ -1734,7 +1735,7 @@ def parse_cmd_def(self):
17341735
else:
17351736
raise VimLParserException(Err(viml_printf("E475: Invalid argument: %s", self.reader.peekn(1)), self.reader.getpos()))
17361737
npos = self.reader.getpos()
1737-
pname = self.reader.read_alpha()
1738+
pname = self.reader.read_word()
17381739
if pname == "":
17391740
if self.reader.peekn(1) == ")":
17401741
self.reader.getn(1)
@@ -1751,7 +1752,7 @@ def parse_cmd_def(self):
17511752
if self.reader.peekn(1) == ":":
17521753
self.reader.getn(1)
17531754
self.reader.skip_white()
1754-
self.read_type()
1755+
varnode.type_str = self.read_type()
17551756
# Check for default value
17561757
self.reader.skip_white()
17571758
if self.reader.peekn(1) == "=":
@@ -1975,8 +1976,27 @@ def parse_cmd_import(self):
19751976
node.left = NIL
19761977
node.str = ""
19771978
self.reader.skip_white()
1978-
# Read the rest of the import line as a string (import autoload 'path' or import 'path' as Name)
1979-
node.str = self.reader.getn(-1)
1979+
# Read the rest of the import line, stopping at a trailing comment.
1980+
line = ""
1981+
pending_ws = ""
1982+
last_was_white = FALSE
1983+
while TRUE:
1984+
c = self.reader.peekn(1)
1985+
if c == "":
1986+
break
1987+
if c == "#" and last_was_white:
1988+
# Consume the rest of the line, so parse_trail doesn't see it.
1989+
self.reader.getn(-1)
1990+
break
1991+
self.reader.getn(1)
1992+
if iswhite(c):
1993+
pending_ws += c
1994+
last_was_white = TRUE
1995+
else:
1996+
line += pending_ws + c
1997+
pending_ws = ""
1998+
last_was_white = FALSE
1999+
node.str = line
19802000
self.add_node(node)
19812001

19822002
def parse_cmd_unlet(self):
@@ -4567,6 +4587,8 @@ def compile_var(self, node):
45674587
if node.rest is not NIL:
45684588
left += " . " + self.compile(node.rest)
45694589
left = "(" + left + ")"
4590+
if node.type_str != "":
4591+
left += ": " + node.type_str
45704592
right = self.compile(node.right)
45714593
self.out("(var %s %s %s)", node.op, left, right)
45724594

@@ -4579,6 +4601,8 @@ def compile_final(self, node):
45794601
if node.rest is not NIL:
45804602
left += " . " + self.compile(node.rest)
45814603
left = "(" + left + ")"
4604+
if node.type_str != "":
4605+
left += ": " + node.type_str
45824606
right = self.compile(node.right)
45834607
self.out("(final %s %s %s)", node.op, left, right)
45844608

‎test/test_vim9import.ok‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
(vim9script)
22
(import "autoload 'mylib.vim'")
33
(import "'./util.vim' as util")
4+
(import "'./x.vim' as util2")
45
(export
56
(def (MyFunc): string
67
(return "hello")))

‎test/test_vim9import.vim‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
vim9script
22
import autoload 'mylib.vim'
33
import './util.vim' as util
4+
import './x.vim' as util2 # trailing comment
45
export def MyFunc(): string
56
return "hello"
67
enddef

‎test/test_vim9var.ok‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
(vim9script)
22
(var = name "hello")
3-
(var = count 42)
3+
(var = count: number 42)
44
(final = x (list 1 2 3))
5+
(final = pi: float 3.14)

‎test/test_vim9var.vim‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,4 @@ vim9script
22
var name = "hello"
33
var count: number = 42
44
final x = [1, 2, 3]
5+
final pi: float = 3.14

0 commit comments

Comments
 (0)