Skip to content

Commit 09d6f15

Browse files
Copilotandrewbranch
Copilot
and
andrewbranch
authored
Port FindNextToken and FindPrecedingToken from Go to TypeScript (#2963)
Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: Andrew Branch <andrew@wheream.io> Co-authored-by: andrewbranch <3277153+andrewbranch@users.noreply.github.com>
1 parent 00b22b0 commit 09d6f15

6 files changed

Lines changed: 16629 additions & 10 deletions

File tree

‎_packages/api/test/async/astnav.test.ts‎

Lines changed: 27 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ import { API } from "@typescript/api/async"; // @sync-skip
44
// @sync-only-end
55
import { createVirtualFileSystem } from "@typescript/api/fs";
66
import {
7+
findNextToken,
8+
findPrecedingToken,
79
formatSyntaxKind,
810
getTokenAtPosition,
911
getTouchingPropertyName,
@@ -117,12 +119,22 @@ describe("astnav", () => {
117119
{
118120
name: "getTokenAtPosition",
119121
baselineFile: "GetTokenAtPosition.mapCode.ts.baseline.json",
120-
fn: getTokenAtPosition,
122+
fn: (sf: SourceFile, pos: number) => getTokenAtPosition(sf, pos) as Node | undefined,
121123
},
122124
{
123125
name: "getTouchingPropertyName",
124126
baselineFile: "GetTouchingPropertyName.mapCode.ts.baseline.json",
125-
fn: getTouchingPropertyName,
127+
fn: (sf: SourceFile, pos: number) => getTouchingPropertyName(sf, pos) as Node | undefined,
128+
},
129+
{
130+
name: "findPrecedingToken",
131+
baselineFile: "FindPrecedingToken.mapCode.ts.baseline.json",
132+
fn: (sf: SourceFile, pos: number) => findPrecedingToken(sf, pos),
133+
},
134+
{
135+
name: "findNextToken",
136+
baselineFile: "FindNextToken.mapCode.ts.baseline.json",
137+
fn: (sf: SourceFile, pos: number) => findNextToken(getTokenAtPosition(sf, pos), sf, sf),
126138
},
127139
];
128140

@@ -135,11 +147,23 @@ describe("astnav", () => {
135147
const failures: string[] = [];
136148

137149
for (let pos = 0; pos < fileText.length; pos++) {
138-
const result = toTokenInfo(tc.fn(sourceFile, pos));
150+
const node = tc.fn(sourceFile, pos);
139151
const goExpected = expected.get(pos);
140152

141153
if (!goExpected) continue;
142154

155+
if (node === undefined) {
156+
failures.push(
157+
` pos ${pos}: expected ${goExpected.kind} [${goExpected.pos}, ${goExpected.end}), got undefined`,
158+
);
159+
if (failures.length >= 50) {
160+
failures.push(" ... (truncated, too many failures)");
161+
break;
162+
}
163+
continue;
164+
}
165+
166+
const result = toTokenInfo(node);
143167
if (result.kind !== goExpected.kind || result.pos !== goExpected.pos || result.end !== goExpected.end) {
144168
failures.push(
145169
` pos ${pos}: expected ${goExpected.kind} [${goExpected.pos}, ${goExpected.end}), ` +

‎_packages/api/test/sync/astnav.test.ts‎

Lines changed: 27 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@
99
import { createVirtualFileSystem } from "@typescript/api/fs";
1010
import { API } from "@typescript/api/sync";
1111
import {
12+
findNextToken,
13+
findPrecedingToken,
1214
formatSyntaxKind,
1315
getTokenAtPosition,
1416
getTouchingPropertyName,
@@ -122,12 +124,22 @@ describe("astnav", () => {
122124
{
123125
name: "getTokenAtPosition",
124126
baselineFile: "GetTokenAtPosition.mapCode.ts.baseline.json",
125-
fn: getTokenAtPosition,
127+
fn: (sf: SourceFile, pos: number) => getTokenAtPosition(sf, pos) as Node | undefined,
126128
},
127129
{
128130
name: "getTouchingPropertyName",
129131
baselineFile: "GetTouchingPropertyName.mapCode.ts.baseline.json",
130-
fn: getTouchingPropertyName,
132+
fn: (sf: SourceFile, pos: number) => getTouchingPropertyName(sf, pos) as Node | undefined,
133+
},
134+
{
135+
name: "findPrecedingToken",
136+
baselineFile: "FindPrecedingToken.mapCode.ts.baseline.json",
137+
fn: (sf: SourceFile, pos: number) => findPrecedingToken(sf, pos),
138+
},
139+
{
140+
name: "findNextToken",
141+
baselineFile: "FindNextToken.mapCode.ts.baseline.json",
142+
fn: (sf: SourceFile, pos: number) => findNextToken(getTokenAtPosition(sf, pos), sf, sf),
131143
},
132144
];
133145

@@ -140,11 +152,23 @@ describe("astnav", () => {
140152
const failures: string[] = [];
141153

142154
for (let pos = 0; pos < fileText.length; pos++) {
143-
const result = toTokenInfo(tc.fn(sourceFile, pos));
155+
const node = tc.fn(sourceFile, pos);
144156
const goExpected = expected.get(pos);
145157

146158
if (!goExpected) continue;
147159

160+
if (node === undefined) {
161+
failures.push(
162+
` pos ${pos}: expected ${goExpected.kind} [${goExpected.pos}, ${goExpected.end}), got undefined`,
163+
);
164+
if (failures.length >= 50) {
165+
failures.push(" ... (truncated, too many failures)");
166+
break;
167+
}
168+
continue;
169+
}
170+
171+
const result = toTokenInfo(node);
148172
if (result.kind !== goExpected.kind || result.pos !== goExpected.pos || result.end !== goExpected.end) {
149173
failures.push(
150174
` pos ${pos}: expected ${goExpected.kind} [${goExpected.pos}, ${goExpected.end}), ` +

‎_packages/ast/src/astnav.ts‎

Lines changed: 169 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,87 @@ export function getTouchingToken(sourceFile: SourceFile, position: number): Node
3030
return getTokenAtPositionImpl(sourceFile, position, /*allowPositionInLeadingTrivia*/ false, /*includePrecedingTokenAtEndPosition*/ undefined);
3131
}
3232

33+
/**
34+
* Finds the token that starts immediately after `previousToken` ends, searching
35+
* within `parent`. Returns `undefined` if no such token exists.
36+
*/
37+
export function findNextToken(previousToken: Node, parent: Node, sourceFile: SourceFile): Node | undefined {
38+
return find(parent);
39+
40+
function find(n: Node): Node | undefined {
41+
if (isTokenKind(n.kind) && n.pos === previousToken.end) {
42+
// This is the token that starts at the end of previousToken – return it.
43+
return n;
44+
}
45+
46+
// Find the child node that contains `previousToken` or starts immediately after it.
47+
let foundNode: Node | undefined;
48+
49+
const visitChild = (node: Node) => {
50+
if (node.flags & NodeFlags.Reparsed) {
51+
return undefined;
52+
}
53+
if (node.pos <= previousToken.end && node.end > previousToken.end) {
54+
foundNode = node;
55+
}
56+
return undefined;
57+
};
58+
59+
// Visit JSDoc children first (mirrors Go's VisitEachChildAndJSDoc).
60+
if (n.jsDoc) {
61+
for (const jsdoc of n.jsDoc) {
62+
visitChild(jsdoc);
63+
}
64+
}
65+
66+
n.forEachChild(
67+
visitChild,
68+
nodes => {
69+
if (nodes.length > 0 && foundNode === undefined) {
70+
for (const node of nodes) {
71+
if (node.flags & NodeFlags.Reparsed) continue;
72+
if (node.pos > previousToken.end) break;
73+
if (node.end > previousToken.end) {
74+
foundNode = node;
75+
break;
76+
}
77+
}
78+
}
79+
return undefined;
80+
},
81+
);
82+
83+
// Recurse into the found child.
84+
if (foundNode !== undefined) {
85+
return find(foundNode);
86+
}
87+
88+
// No AST child covers the position; use the scanner to find the syntactic token.
89+
// The scanner is initialized at `previousToken.end`, so tokenFullStart === previousToken.end.
90+
const startPos = previousToken.end;
91+
if (startPos >= n.pos && startPos < n.end) {
92+
const scanner = getScannerForSourceFile(sourceFile, startPos);
93+
const token = scanner.getToken();
94+
const tokenFullStart = scanner.getTokenFullStart();
95+
const tokenEnd = scanner.getTokenEnd();
96+
const flags = scanner.getTokenFlags();
97+
return getOrCreateToken(sourceFile, token, tokenFullStart, tokenEnd, n, flags);
98+
}
99+
100+
return undefined;
101+
}
102+
}
103+
104+
/**
105+
* Finds the leftmost token satisfying `position < token.end`.
106+
* If the position is in the trivia of that leftmost token, or the token is invalid,
107+
* returns the rightmost valid token with `token.end <= position`.
108+
* Excludes `JsxText` tokens containing only whitespace.
109+
*/
110+
export function findPrecedingToken(sourceFile: SourceFile, position: number): Node | undefined {
111+
return findPrecedingTokenImpl(sourceFile, position, sourceFile);
112+
}
113+
33114
function getTokenAtPositionImpl(
34115
sourceFile: SourceFile,
35116
position: number,
@@ -244,11 +325,29 @@ function findPrecedingTokenImpl(sourceFile: SourceFile, position: number, startN
244325
let foundChild: Node | undefined;
245326
let prevChild: Node | undefined;
246327

328+
// Visit JSDoc nodes first (mirrors Go's VisitEachChildAndJSDoc).
329+
if (n.jsDoc) {
330+
for (const jsdoc of n.jsDoc) {
331+
if (jsdoc.flags & NodeFlags.Reparsed) continue;
332+
if (foundChild !== undefined) break;
333+
if (position < jsdoc.end && (prevChild === undefined || prevChild.end <= position)) {
334+
foundChild = jsdoc;
335+
}
336+
else {
337+
prevChild = jsdoc;
338+
}
339+
}
340+
}
341+
342+
let skipSingleCommentChildrenImpl = false;
247343
n.forEachChild(
248344
node => {
249345
if (node.flags & NodeFlags.Reparsed) {
250346
return undefined;
251347
}
348+
if (skipSingleCommentChildrenImpl && isJSDocCommentChildKind(node.kind)) {
349+
return undefined;
350+
}
252351
if (foundChild !== undefined) {
253352
return undefined;
254353
}
@@ -261,10 +360,11 @@ function findPrecedingTokenImpl(sourceFile: SourceFile, position: number, startN
261360
return undefined;
262361
},
263362
nodes => {
363+
skipSingleCommentChildrenImpl = isJSDocSingleCommentNodeList(nodes);
264364
if (foundChild !== undefined) {
265365
return undefined;
266366
}
267-
if (nodes.length > 0) {
367+
if (nodes.length > 0 && !skipSingleCommentChildrenImpl) {
268368
const index = binarySearchForPrecedingToken(nodes, position);
269369
if (index >= 0 && !(nodes[index].flags & NodeFlags.Reparsed)) {
270370
foundChild = nodes[index];
@@ -284,9 +384,29 @@ function findPrecedingTokenImpl(sourceFile: SourceFile, position: number, startN
284384
);
285385

286386
if (foundChild !== undefined) {
287-
const start = getTokenPosOfNode(foundChild, sourceFile);
387+
const start = getTokenPosOfNode(foundChild, sourceFile, /*includeJSDoc*/ true);
288388
if (start >= position) {
289-
// cursor in leading trivia; find rightmost valid token in prevChild
389+
if (position >= foundChild.pos) {
390+
// We are in the leading trivia of foundChild. Check for JSDoc nodes of n
391+
// preceding foundChild, mirroring Go's findPrecedingToken logic.
392+
let jsDoc: Node | undefined;
393+
if (n.jsDoc) {
394+
for (let i = n.jsDoc.length - 1; i >= 0; i--) {
395+
if (n.jsDoc[i].pos >= foundChild.pos) {
396+
jsDoc = n.jsDoc[i];
397+
break;
398+
}
399+
}
400+
}
401+
if (jsDoc !== undefined) {
402+
if (position < jsDoc.end) {
403+
return find(jsDoc);
404+
}
405+
return findRightmostValidToken(sourceFile, jsDoc.end, n, position);
406+
}
407+
return findRightmostValidToken(sourceFile, foundChild.pos, n, -1);
408+
}
409+
// Answer is in tokens between two visited children.
290410
return findRightmostValidToken(sourceFile, foundChild.pos, n, position);
291411
}
292412
return find(foundChild);
@@ -314,11 +434,27 @@ function findRightmostValidToken(sourceFile: SourceFile, endPos: number, contain
314434
let rightmostValidNode: Node | undefined;
315435
let hasChildren = false;
316436

437+
// Visit JSDoc nodes first (mirrors Go's VisitEachChildAndJSDoc).
438+
if (n.jsDoc) {
439+
hasChildren = true;
440+
for (const jsdoc of n.jsDoc) {
441+
if (jsdoc.flags & NodeFlags.Reparsed) continue;
442+
if (jsdoc.end > endPos || getTokenPosOfNode(jsdoc, sourceFile) >= position) continue;
443+
if (isValidPrecedingNode(jsdoc, sourceFile)) {
444+
rightmostValidNode = jsdoc;
445+
}
446+
}
447+
}
448+
449+
let skipSingleCommentChildren = false;
317450
n.forEachChild(
318451
node => {
319452
if (node.flags & NodeFlags.Reparsed) {
320453
return undefined;
321454
}
455+
if (skipSingleCommentChildren && isJSDocCommentChildKind(node.kind)) {
456+
return undefined;
457+
}
322458
hasChildren = true;
323459
if (node.end > endPos || getTokenPosOfNode(node, sourceFile) >= position) {
324460
return undefined;
@@ -329,7 +465,10 @@ function findRightmostValidToken(sourceFile: SourceFile, endPos: number, contain
329465
return undefined;
330466
},
331467
nodes => {
332-
if (nodes.length > 0) {
468+
// Skip single-comment JSDoc NodeLists (e.g. JSDocText children of a JSDoc node):
469+
// In Go, these are stored as string properties and are never visited as children.
470+
skipSingleCommentChildren = isJSDocSingleCommentNodeList(nodes);
471+
if (nodes.length > 0 && !skipSingleCommentChildren) {
333472
hasChildren = true;
334473
for (let i = nodes.length - 1; i >= 0; i--) {
335474
const node = nodes[i];
@@ -345,6 +484,32 @@ function findRightmostValidToken(sourceFile: SourceFile, endPos: number, contain
345484
},
346485
);
347486

487+
// Scan for syntactic tokens (e.g. `{`, `,`) between AST nodes, matching Go's
488+
// findRightmostValidToken scanner step.
489+
if (!shouldSkipChild(n)) {
490+
const startPos = rightmostValidNode !== undefined ? rightmostValidNode.end : n.pos;
491+
const targetEnd = Math.min(endPos, position);
492+
if (startPos < targetEnd) {
493+
const scanner = getScannerForSourceFile(sourceFile, startPos);
494+
let pos = startPos;
495+
let lastScannedToken: Node | undefined;
496+
while (pos < targetEnd) {
497+
const tokenStart = scanner.getTokenStart();
498+
if (tokenStart >= position) break;
499+
const tokenFullStart = scanner.getTokenFullStart();
500+
const tokenEnd = scanner.getTokenEnd();
501+
const token = scanner.getToken();
502+
const flags = scanner.getTokenFlags();
503+
lastScannedToken = getOrCreateToken(sourceFile, token, tokenFullStart, tokenEnd, n, flags);
504+
pos = tokenEnd;
505+
scanner.scan();
506+
}
507+
if (lastScannedToken !== undefined) {
508+
return lastScannedToken;
509+
}
510+
}
511+
}
512+
348513
if (!hasChildren) {
349514
if (n !== containingNode) {
350515
return n;

‎internal/astnav/tokens_test.go‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -476,6 +476,35 @@ func TestFindPrecedingToken(t *testing.T) {
476476
},
477477
)
478478
})
479+
480+
t.Run("go baseline json", func(t *testing.T) {
481+
t.Parallel()
482+
baselineGoTokensJSON(t, "FindPrecedingToken", func(file *ast.SourceFile, pos int) *tokenInfo {
483+
return toTokenInfo(astnav.FindPrecedingToken(file, pos))
484+
})
485+
})
486+
}
487+
488+
func TestFindNextToken(t *testing.T) {
489+
t.Parallel()
490+
repo.SkipIfNoTypeScriptSubmodule(t)
491+
492+
t.Run("go baseline json", func(t *testing.T) {
493+
t.Parallel()
494+
baselineGoTokensJSON(t, "FindNextToken", func(file *ast.SourceFile, pos int) (result *tokenInfo) {
495+
// FindNextToken panics (like Go's assert) when the scanner finds trivia between
496+
// previousToken.End() and the next syntactic token. Catch those to avoid crashing
497+
// the baseline generator; those positions will be absent from the baseline.
498+
defer func() {
499+
if r := recover(); r != nil {
500+
result = nil
501+
}
502+
}()
503+
token := astnav.GetTokenAtPosition(file, pos)
504+
next := astnav.FindNextToken(token, file.AsNode(), file)
505+
return toTokenInfo(next)
506+
})
507+
})
479508
}
480509

481510
func TestUnitFindPrecedingToken(t *testing.T) {

0 commit comments

Comments
 (0)