From e985ae9344ba487825af857b12dda7ce5437af73 Mon Sep 17 00:00:00 2001 From: HrushiBorhade Date: Tue, 24 Mar 2026 16:54:53 +0530 Subject: [PATCH 1/2] feat: switch tree-sitter from native N-API to web-tree-sitter (WASM) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Replaced native tree-sitter + 6 grammar packages with web-tree-sitter + tree-sitter-wasms - WASM grammars loaded at runtime via Language.load() (TypeScript, TSX, JS, Python, Rust, Go, CSS) - Parser initialized async via Parser.init() — chunkAST is now async - Removed native N-API dependencies from pnpm.onlyBuiltDependencies - Compatible versions: web-tree-sitter@0.22.6 + tree-sitter-wasms@0.1.13 - All 32 chunker tests pass, 132/132 non-embedder tests pass - WASM runs anywhere: Node.js, Linux containers, Trigger.dev workers Closes #69 --- package.json | 9 +- packages/core/package.json | 9 +- packages/core/src/chunker/ast.ts | 85 +++++++++------ pnpm-lock.yaml | 174 ++++--------------------------- 4 files changed, 76 insertions(+), 201 deletions(-) diff --git a/package.json b/package.json index e87d90d..0c7f6ca 100644 --- a/package.json +++ b/package.json @@ -28,14 +28,7 @@ "pnpm": { "onlyBuiltDependencies": [ "better-sqlite3", - "esbuild", - "tree-sitter", - "tree-sitter-css", - "tree-sitter-go", - "tree-sitter-javascript", - "tree-sitter-python", - "tree-sitter-rust", - "tree-sitter-typescript" + "esbuild" ] } } diff --git a/packages/core/package.json b/packages/core/package.json index 80d2c50..99c4e46 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -34,13 +34,8 @@ "fast-glob": "^3.3.3", "pino": "^10.3.1", "pino-pretty": "^13.1.3", - "tree-sitter": "^0.21.1", - "tree-sitter-css": "^0.21.1", - "tree-sitter-go": "^0.21.2", - "tree-sitter-javascript": "^0.21.4", - "tree-sitter-python": "^0.21.0", - "tree-sitter-rust": "^0.21.0", - "tree-sitter-typescript": "^0.23.2", + "tree-sitter-wasms": "^0.1.13", + "web-tree-sitter": "0.22.6", "zod": "^4.3.6" }, "devDependencies": { diff --git a/packages/core/src/chunker/ast.ts b/packages/core/src/chunker/ast.ts index 6a1b8c8..aada9a2 100644 --- a/packages/core/src/chunker/ast.ts +++ b/packages/core/src/chunker/ast.ts @@ -1,41 +1,56 @@ -import Parser from 'tree-sitter'; -import TypeScript from 'tree-sitter-typescript'; -import JavaScript from 'tree-sitter-javascript'; -import Python from 'tree-sitter-python'; -import Rust from 'tree-sitter-rust'; -import Go from 'tree-sitter-go'; -import CSS from 'tree-sitter-css'; +import { resolve } from 'node:path'; +import { createRequire } from 'node:module'; import type { Chunk } from './types.ts'; import { fallbackChunk } from './fallback.ts'; import { createLogger } from '../utils/logger.ts'; const log = createLogger('chunker'); -// --- Parser cache (one parser per language, reused) --- - -const parsers = new Map(); +// --- WASM initialization --- +// web-tree-sitter types use `declare module` pattern which doesn't play well +// with nodenext resolution. We use a typed wrapper to keep things clean. // eslint-disable-next-line @typescript-eslint/no-explicit-any -const GRAMMARS: Record = { - typescript: TypeScript.typescript, - tsx: TypeScript.tsx, - javascript: JavaScript, - python: Python, - rust: Rust, - go: Go, - css: CSS, +let ParserClass: any; +let initialized = false; + +async function ensureInit(): Promise { + if (initialized) return; + const mod = await import('web-tree-sitter'); + ParserClass = mod.default; + await ParserClass.init(); + initialized = true; +} + +// --- Grammar loading (WASM files from tree-sitter-wasms package) --- + +const require = createRequire(import.meta.url); +const wasmsDir = resolve(require.resolve('tree-sitter-wasms/package.json'), '..', 'out'); + +const GRAMMAR_FILES: Record = { + typescript: 'tree-sitter-typescript.wasm', + tsx: 'tree-sitter-tsx.wasm', + javascript: 'tree-sitter-javascript.wasm', + python: 'tree-sitter-python.wasm', + rust: 'tree-sitter-rust.wasm', + go: 'tree-sitter-go.wasm', + css: 'tree-sitter-css.wasm', }; -function getParser(language: string): Parser | undefined { - if (parsers.has(language)) return parsers.get(language)!; +// eslint-disable-next-line @typescript-eslint/no-explicit-any +const languages = new Map(); - const grammar = GRAMMARS[language]; - if (!grammar) return undefined; +// eslint-disable-next-line @typescript-eslint/no-explicit-any +async function getLanguage(language: string): Promise { + if (languages.has(language)) return languages.get(language)!; + + const file = GRAMMAR_FILES[language]; + if (!file) return undefined; - const parser = new Parser(); - parser.setLanguage(grammar); - parsers.set(language, parser); - return parser; + const wasmPath = resolve(wasmsDir, file); + const lang = await ParserClass.Language.load(wasmPath); + languages.set(language, lang); + return lang; } // --- Top-level node types to extract per language --- @@ -65,11 +80,13 @@ const AST_NODE_TYPES: Record> = { css: new Set(['rule_set', 'media_statement', 'keyframes_statement']), }; -// --- AST chunking --- +// --- AST chunking (async — requires WASM init + language loading) --- + +async function chunkAST(source: string, filePath: string, language: string): Promise { + await ensureInit(); -function chunkAST(source: string, filePath: string, language: string): Chunk[] { - const parser = getParser(language); - if (!parser) { + const lang = await getLanguage(language); + if (!lang) { log.warn(`No tree-sitter grammar for "${language}", using fallback: ${filePath}`); return [fallbackChunk(source, filePath, language)]; } @@ -82,7 +99,10 @@ function chunkAST(source: string, filePath: string, language: string): Chunk[] { return [fallbackChunk(source, filePath, language)]; } - let tree: Parser.Tree; + const parser = new ParserClass(); + parser.setLanguage(lang); + + let tree; try { tree = parser.parse(source); } catch (err: unknown) { @@ -110,6 +130,9 @@ function chunkAST(source: string, filePath: string, language: string): Chunk[] { }); } + parser.delete(); + tree.delete(); + if (chunks.length === 0 && source.trim().length > 0) { console.warn( `[chunker] AST parsed but found 0 matching nodes for "${language}" in: ${filePath}`, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0a2c76e..dd080c6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -129,27 +129,12 @@ importers: pino-pretty: specifier: ^13.1.3 version: 13.1.3 - tree-sitter: - specifier: ^0.21.1 - version: 0.21.1 - tree-sitter-css: - specifier: ^0.21.1 - version: 0.21.1(tree-sitter@0.21.1) - tree-sitter-go: - specifier: ^0.21.2 - version: 0.21.2(tree-sitter@0.21.1) - tree-sitter-javascript: - specifier: ^0.21.4 - version: 0.21.4(tree-sitter@0.21.1) - tree-sitter-python: - specifier: ^0.21.0 - version: 0.21.0(tree-sitter@0.21.1) - tree-sitter-rust: - specifier: ^0.21.0 - version: 0.21.0(tree-sitter@0.21.1) - tree-sitter-typescript: - specifier: ^0.23.2 - version: 0.23.2(tree-sitter@0.21.1) + tree-sitter-wasms: + specifier: ^0.1.13 + version: 0.1.13 + web-tree-sitter: + specifier: 0.22.6 + version: 0.22.6 zod: specifier: ^4.3.6 version: 4.3.6 @@ -4364,13 +4349,6 @@ packages: resolution: {integrity: sha512-6u9UwL0HlAl21+agMN3YAMXcKByMqwGx+pq+P76vii5f7hTPtKDp08/H9py6DY+cfDw7kQNTGEj/rly3IgbNQA==} engines: {node: '>=10'} - node-addon-api@7.1.1: - resolution: {integrity: sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==} - - node-addon-api@8.6.0: - resolution: {integrity: sha512-gBVjCaqDlRUk0EwoPNKzIr9KkS9041G/q31IBShPs1Xz6UTA+EXdZADbzqAJQrpDRq71CIMnOP5VMut3SL0z5Q==} - engines: {node: ^18 || ^20 || >= 21} - node-domexception@1.0.0: resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} engines: {node: '>=10.5.0'} @@ -4384,10 +4362,6 @@ packages: resolution: {integrity: sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - node-gyp-build@4.8.4: - resolution: {integrity: sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==} - hasBin: true - node-releases@2.0.36: resolution: {integrity: sha512-TdC8FSgHz8Mwtw9g5L4gR/Sh9XhSP/0DEkQxfEFXOpiul5IiHgHan2VhYYb6agDSfp4KuvltmGApc8HMgUrIkA==} @@ -5151,69 +5125,8 @@ packages: resolution: {integrity: sha512-hdF5ZgjTqgAntKkklYw0R03MG2x/bSzTtkxmIRw/sTNV8YXsCJ1tfLAX23lhxhHJlEf3CRCOCGGWw3vI3GaSPw==} engines: {node: '>=18'} - tree-sitter-css@0.21.1: - resolution: {integrity: sha512-xihcA5XR/46TiJKWZXIm+K+q4crkHOSbD7j0r2tHHmkGguBa89hFnEmSJTK52wq9dJzStxdTUj/5eCP0xnCQpQ==} - peerDependencies: - tree-sitter: ^0.21.0 - tree_sitter: '*' - peerDependenciesMeta: - tree_sitter: - optional: true - - tree-sitter-go@0.21.2: - resolution: {integrity: sha512-aMFwjsB948nWhURiIxExK8QX29JYKs96P/IfXVvluVMRJZpL04SREHsdOZHYqJr1whkb7zr3/gWHqqvlkczmvw==} - peerDependencies: - tree-sitter: ^0.21.0 - tree_sitter: '*' - peerDependenciesMeta: - tree_sitter: - optional: true - - tree-sitter-javascript@0.21.4: - resolution: {integrity: sha512-Lrk8yahebwrwc1sWJE9xPcz1OnnqiEV7Dh5fbN6EN3wNAdu9r06HpTqLqDwUUbnG4EB46Sfk+FJFAOldfoKLOw==} - peerDependencies: - tree-sitter: ^0.21.1 - tree_sitter: '*' - peerDependenciesMeta: - tree_sitter: - optional: true - - tree-sitter-javascript@0.23.1: - resolution: {integrity: sha512-/bnhbrTD9frUYHQTiYnPcxyHORIw157ERBa6dqzaKxvR/x3PC4Yzd+D1pZIMS6zNg2v3a8BZ0oK7jHqsQo9fWA==} - peerDependencies: - tree-sitter: ^0.21.1 - peerDependenciesMeta: - tree-sitter: - optional: true - - tree-sitter-python@0.21.0: - resolution: {integrity: sha512-IUKx7JcTVbByUx1iHGFS/QsIjx7pqwTMHL9bl/NGyhyyydbfNrpruo2C7W6V4KZrbkkCOlX8QVrCoGOFW5qecg==} - peerDependencies: - tree-sitter: ^0.21.0 - tree_sitter: '*' - peerDependenciesMeta: - tree_sitter: - optional: true - - tree-sitter-rust@0.21.0: - resolution: {integrity: sha512-unVr73YLn3VC4Qa/GF0Nk+Wom6UtI526p5kz9Rn2iZSqwIFedyCZ3e0fKCEmUJLIPGrTb/cIEdu3ZUNGzfZx7A==} - peerDependencies: - tree-sitter: ^0.21.1 - tree_sitter: '*' - peerDependenciesMeta: - tree_sitter: - optional: true - - tree-sitter-typescript@0.23.2: - resolution: {integrity: sha512-e04JUUKxTT53/x3Uq1zIL45DoYKVfHH4CZqwgZhPg5qYROl5nQjV+85ruFzFGZxu+QeFVbRTPDRnqL9UbU4VeA==} - peerDependencies: - tree-sitter: ^0.21.0 - peerDependenciesMeta: - tree-sitter: - optional: true - - tree-sitter@0.21.1: - resolution: {integrity: sha512-7dxoA6kYvtgWw80265MyqJlkRl4yawIjO7S5MigytjELkX43fV2WsAXzsNfO7sBpPPCF5Gp0+XzHk0DwLCq3xQ==} + tree-sitter-wasms@0.1.13: + resolution: {integrity: sha512-wT+cR6DwaIz80/vho3AvSF0N4txuNx/5bcRKoXouOfClpxh/qqrF4URNLQXbbt8MaAxeksZcZd1j8gcGjc+QxQ==} ts-api-utils@2.5.0: resolution: {integrity: sha512-OJ/ibxhPlqrMM0UiNHJ/0CKQkoKF243/AEmplt3qpRgkW8VG7IfOS41h7V8TjITqdByHzrjcS/2si+y4lIh8NA==} @@ -5444,6 +5357,9 @@ packages: resolution: {integrity: sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==} engines: {node: '>= 8'} + web-tree-sitter@0.22.6: + resolution: {integrity: sha512-hS87TH71Zd6mGAmYCvlgxeGDjqd9GTeqXNqTT+u0Gs51uIozNIaaq/kUAbV/Zf56jb2ZOyG8BxZs2GG9wbLi6Q==} + webidl-conversions@7.0.0: resolution: {integrity: sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==} engines: {node: '>=12'} @@ -8492,7 +8408,7 @@ snapshots: '@next/eslint-plugin-next': 16.2.0 eslint: 9.39.4(jiti@2.6.1) eslint-import-resolver-node: 0.3.9 - eslint-import-resolver-typescript: 3.10.1(eslint-plugin-import@2.32.0)(eslint@9.39.4(jiti@2.6.1)) + eslint-import-resolver-typescript: 3.10.1(eslint-plugin-import@2.32.0(eslint@9.39.4(jiti@2.6.1)))(eslint@9.39.4(jiti@2.6.1)) eslint-plugin-import: 2.32.0(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.4(jiti@2.6.1)) eslint-plugin-jsx-a11y: 6.10.2(eslint@9.39.4(jiti@2.6.1)) eslint-plugin-react: 7.37.5(eslint@9.39.4(jiti@2.6.1)) @@ -8519,7 +8435,7 @@ snapshots: transitivePeerDependencies: - supports-color - eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0)(eslint@9.39.4(jiti@2.6.1)): + eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0(eslint@9.39.4(jiti@2.6.1)))(eslint@9.39.4(jiti@2.6.1)): dependencies: '@nolyfill/is-core-module': 1.0.39 debug: 4.4.3 @@ -8534,13 +8450,13 @@ snapshots: transitivePeerDependencies: - supports-color - eslint-module-utils@2.12.1(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.4(jiti@2.6.1)): + eslint-module-utils@2.12.1(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0(eslint@9.39.4(jiti@2.6.1)))(eslint@9.39.4(jiti@2.6.1)))(eslint@9.39.4(jiti@2.6.1)): dependencies: debug: 3.2.7 optionalDependencies: eslint: 9.39.4(jiti@2.6.1) eslint-import-resolver-node: 0.3.9 - eslint-import-resolver-typescript: 3.10.1(eslint-plugin-import@2.32.0)(eslint@9.39.4(jiti@2.6.1)) + eslint-import-resolver-typescript: 3.10.1(eslint-plugin-import@2.32.0(eslint@9.39.4(jiti@2.6.1)))(eslint@9.39.4(jiti@2.6.1)) transitivePeerDependencies: - supports-color @@ -8555,7 +8471,7 @@ snapshots: doctrine: 2.1.0 eslint: 9.39.4(jiti@2.6.1) eslint-import-resolver-node: 0.3.9 - eslint-module-utils: 2.12.1(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.4(jiti@2.6.1)) + eslint-module-utils: 2.12.1(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0(eslint@9.39.4(jiti@2.6.1)))(eslint@9.39.4(jiti@2.6.1)))(eslint@9.39.4(jiti@2.6.1)) hasown: 2.0.2 is-core-module: 2.16.1 is-glob: 4.0.3 @@ -9611,10 +9527,6 @@ snapshots: dependencies: semver: 7.7.4 - node-addon-api@7.1.1: {} - - node-addon-api@8.6.0: {} - node-domexception@1.0.0: {} node-exports-info@1.6.0: @@ -9630,8 +9542,6 @@ snapshots: fetch-blob: 3.2.0 formdata-polyfill: 4.0.10 - node-gyp-build@4.8.4: {} - node-releases@2.0.36: {} npm-run-path@4.0.1: @@ -10603,55 +10513,7 @@ snapshots: dependencies: punycode: 2.3.1 - tree-sitter-css@0.21.1(tree-sitter@0.21.1): - dependencies: - node-addon-api: 8.6.0 - node-gyp-build: 4.8.4 - tree-sitter: 0.21.1 - - tree-sitter-go@0.21.2(tree-sitter@0.21.1): - dependencies: - node-addon-api: 8.6.0 - node-gyp-build: 4.8.4 - tree-sitter: 0.21.1 - - tree-sitter-javascript@0.21.4(tree-sitter@0.21.1): - dependencies: - node-addon-api: 8.6.0 - node-gyp-build: 4.8.4 - tree-sitter: 0.21.1 - - tree-sitter-javascript@0.23.1(tree-sitter@0.21.1): - dependencies: - node-addon-api: 8.6.0 - node-gyp-build: 4.8.4 - optionalDependencies: - tree-sitter: 0.21.1 - - tree-sitter-python@0.21.0(tree-sitter@0.21.1): - dependencies: - node-addon-api: 7.1.1 - node-gyp-build: 4.8.4 - tree-sitter: 0.21.1 - - tree-sitter-rust@0.21.0(tree-sitter@0.21.1): - dependencies: - node-addon-api: 7.1.1 - node-gyp-build: 4.8.4 - tree-sitter: 0.21.1 - - tree-sitter-typescript@0.23.2(tree-sitter@0.21.1): - dependencies: - node-addon-api: 8.6.0 - node-gyp-build: 4.8.4 - tree-sitter-javascript: 0.23.1(tree-sitter@0.21.1) - optionalDependencies: - tree-sitter: 0.21.1 - - tree-sitter@0.21.1: - dependencies: - node-addon-api: 8.6.0 - node-gyp-build: 4.8.4 + tree-sitter-wasms@0.1.13: {} ts-api-utils@2.5.0(typescript@5.9.3): dependencies: @@ -10947,6 +10809,8 @@ snapshots: web-streams-polyfill@3.3.3: {} + web-tree-sitter@0.22.6: {} + webidl-conversions@7.0.0: {} whatwg-url@14.2.0: From dfeb2946e27c92d450d51bcd681f4ae6bf84e4f7 Mon Sep 17 00:00:00 2001 From: HrushiBorhade Date: Tue, 24 Mar 2026 19:37:53 +0530 Subject: [PATCH 2/2] fix: parser memory leak on parse failure + use Pino logger consistently - Wrapped parser/tree in try/finally to ensure cleanup on all code paths - Replaced console.warn with log.warn for consistent structured logging --- packages/core/src/chunker/ast.ts | 68 ++++++++++++++++---------------- 1 file changed, 33 insertions(+), 35 deletions(-) diff --git a/packages/core/src/chunker/ast.ts b/packages/core/src/chunker/ast.ts index aada9a2..d0ad74a 100644 --- a/packages/core/src/chunker/ast.ts +++ b/packages/core/src/chunker/ast.ts @@ -93,54 +93,52 @@ async function chunkAST(source: string, filePath: string, language: string): Pro const nodeTypes = AST_NODE_TYPES[language]; if (!nodeTypes) { - console.warn( - `[chunker] No AST node types configured for "${language}", using fallback: ${filePath}`, - ); + log.warn(`No AST node types configured for "${language}", using fallback: ${filePath}`); return [fallbackChunk(source, filePath, language)]; } const parser = new ParserClass(); parser.setLanguage(lang); - let tree; try { - tree = parser.parse(source); + const tree = parser.parse(source); + + try { + const chunks: Chunk[] = []; + + for (const node of tree.rootNode.children) { + if (!nodeTypes.has(node.type)) continue; + + const content = node.text.trim(); + if (content.length === 0) continue; + + chunks.push({ + content, + filePath, + lineStart: node.startPosition.row + 1, + lineEnd: node.endPosition.row + 1, + language, + type: 'ast', + }); + } + + if (chunks.length === 0 && source.trim().length > 0) { + log.warn(`AST parsed but found 0 matching nodes for "${language}" in: ${filePath}`); + return [fallbackChunk(source, filePath, language)]; + } + + return chunks; + } finally { + tree.delete(); + } } catch (err: unknown) { log.error( `tree-sitter parse failed for ${filePath}: ${err instanceof Error ? err.message : err}`, ); return [fallbackChunk(source, filePath, language)]; + } finally { + parser.delete(); } - - const chunks: Chunk[] = []; - - for (const node of tree.rootNode.children) { - if (!nodeTypes.has(node.type)) continue; - - const content = node.text.trim(); - if (content.length === 0) continue; - - chunks.push({ - content, - filePath, - lineStart: node.startPosition.row + 1, - lineEnd: node.endPosition.row + 1, - language, - type: 'ast', - }); - } - - parser.delete(); - tree.delete(); - - if (chunks.length === 0 && source.trim().length > 0) { - console.warn( - `[chunker] AST parsed but found 0 matching nodes for "${language}" in: ${filePath}`, - ); - return [fallbackChunk(source, filePath, language)]; - } - - return chunks; } export { chunkAST };