Merge branch 'main' into fix-and-scanner

Author: aspeddro

.editorconfig

@@ -0,0 +1,46 @@
+root = true
+
+[*]
+charset = utf-8
+
+[*.{json,toml,yml,gyp}]
+indent_style = space
+indent_size = 2
+
+[*.js]
+indent_style = space
+indent_size = 2
+
+[*.scm]
+indent_style = space
+indent_size = 2
+
+[*.{c,cc,h}]
+indent_style = space
+indent_size = 4
+
+[*.rs]
+indent_style = space
+indent_size = 4
+
+[*.{py,pyi}]
+indent_style = space
+indent_size = 4
+
+[*.swift]
+indent_style = space
+indent_size = 4
+
+[*.go]
+indent_style = tab
+indent_size = 8
+
+[Makefile]
+indent_style = tab
+indent_size = 8
+
+[parser.c]
+indent_size = 2
+
+[{alloc,array,parser}.h]
+indent_size = 2

.gitattributes

@@ -0,0 +1,41 @@
+* text=auto eol=lf
+
+# Generated source files
+src/*.json linguist-generated
+src/parser.c linguist-generated
+src/tree_sitter/* linguist-generated
+
+# C bindings
+bindings/c/** linguist-generated
+CMakeLists.txt linguist-generated
+Makefile linguist-generated
+
+# Rust bindings
+bindings/rust/* linguist-generated
+Cargo.toml linguist-generated
+Cargo.lock linguist-generated
+
+# Node.js bindings
+bindings/node/* linguist-generated
+binding.gyp linguist-generated
+package.json linguist-generated
+package-lock.json linguist-generated
+
+# Python bindings
+bindings/python/** linguist-generated
+setup.py linguist-generated
+pyproject.toml linguist-generated
+
+# Go bindings
+bindings/go/* linguist-generated
+go.mod linguist-generated
+go.sum linguist-generated
+
+# Swift bindings
+bindings/swift/** linguist-generated
+Package.swift linguist-generated
+Package.resolved linguist-generated
+
+# Zig bindings
+build.zig linguist-generated
+build.zig.zon linguist-generated

.github/workflows/build-test.yml

@@ -0,0 +1,55 @@
+name: CI
+on:
+  push:
+    branches: [main]
+    paths:
+      - grammar.js
+      - src/**
+      - test/**
+      - bindings/**
+      - binding.gyp
+      - package.json
+      - package-lock.json
+      - .nvmrc
+      - .github/workflows/**
+  pull_request:
+    paths:
+      - grammar.js
+      - src/**
+      - test/**
+      - bindings/**
+      - binding.gyp
+      - package.json
+      - package-lock.json
+      - .nvmrc
+      - .github/workflows/**
+
+concurrency:
+  group: ${{github.workflow}}-${{github.ref}}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version-file: '.nvmrc'
+
+      - name: Set up tree-sitter
+        uses: tree-sitter/setup-action/cli@v2
+        with:
+          tree-sitter-ref: v0.26.3
+
+      - name: Run tests
+        uses: tree-sitter/parser-test-action@v2
+        with:
+          test-rust: true
+          test-node: false
+          test-python: true
+          test-go: true
+          test-swift: false

.github/workflows/ci.yml

@@ -0,0 +1,59 @@
+name: GitHub Pages
+
+on:
+  push:
+    branches: [main]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+concurrency:
+  group: pages
+  cancel-in-progress: false
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version-file: .nvmrc
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Build WASM
+        run: npx tree-sitter build --wasm
+
+      - name: Export playground
+        run: npx tree-sitter playground --export ./public
+
+      - name: Set LANGUAGE_BASE_URL for GitHub Pages
+        run: |
+          sed -i 's|LANGUAGE_BASE_URL = ""|LANGUAGE_BASE_URL = "/tree-sitter-rescript"|' ./public/index.html
+
+      - name: Setup Pages
+        uses: actions/configure-pages@v5
+
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v3
+        with:
+          path: ./public
+
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    needs: build
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4

.github/workflows/gh-pages.yml

@@ -1,8 +1,54 @@
-bindings/
+# Rust artifacts
+target/
+
+# Node artifacts
+build/
+prebuilds/
 node_modules/
-src/*
-!src/scanner.c
+
+# Swift artifacts
+.build/
+
+# Go artifacts
+_obj/
+
+# Python artifacts
+.venv/
+dist/
+*.egg-info
+*.whl
+
+# C artifacts
+*.a
+*.so
+*.so.*
+*.dylib
+*.dll
+*.pc
+*.exp
+*.lib
+
+# Zig artifacts
+.zig-cache/
+zig-cache/
+zig-out/
+
+# Example dirs
+/examples/*/
+
+# Grammar volatiles
+*.wasm
+*.obj
+*.o
+
+# Archives
+*.tar.gz
+*.tgz
+*.zip
+
+# Others
 log.html
-binding.gyp
 test_wild/
-build/
+
+# Playground
+public/

.gitignore

@@ -0,0 +1 @@
+v24.11.1

.nvmrc

@@ -0,0 +1,80 @@
+# AGENTS.md
+
+This file provides guidance to AI/LLM when working with code in this repository.
+
+## Build and Test Commands
+
+```bash
+# Generate parser from grammar (required after grammar.js changes)
+npx tree-sitter generate
+
+# Run all corpus tests
+npx tree-sitter test
+
+# Run tests for a specific grammar rule (filter by name)
+npx tree-sitter test --include "switch"
+
+# Parse a file to see syntax tree (useful for debugging)
+npm run parse path/to/file.res
+
+# Compare with ReScript compiler's AST (useful for debugging grammar issues)
+npx bsc -dparsetree -only-parse -ignore-parse-errors path/to/file.res
+
+# Launch interactive playground (builds WASM first)
+npm start
+
+# Install npm dependencies
+npm install
+```
+
+## Architecture
+
+### Grammar Definition
+
+**grammar.js** - The main grammar definition using Tree-sitter's JavaScript DSL. Key sections:
+- `externals` - Tokens handled by the custom scanner (newlines, comments, template strings, decorators, parentheses)
+- `precedences` - Operator precedence rules for expressions and module paths
+- `conflicts` - Shift/reduce conflict resolutions for ambiguous constructs
+- `rules` - All grammar rules (~80+ covering statements, expressions, types, modules, JSX)
+
+### External Scanner
+
+**src/scanner.c** - Custom C scanner for context-sensitive tokens that the grammar alone cannot handle:
+- Significant vs insignificant newlines (statement-ending vs formatting)
+- Nested multiline comments (`/* /* */ */`)
+- Template string interpolation (`\`hello ${name}\``)
+- Parenthesis nesting tracking (affects newline significance)
+- `list{` and `dict{` constructor detection
+- Decorator parsing (`@decorator` vs `@decorator(args)`)
+
+The scanner maintains state (`ScannerState`) tracking parenthesis nesting depth, whether inside quotes/backticks, and EOF reporting.
+
+### Generated Files (do not edit manually)
+
+- `src/parser.c` - Generated LR parser from grammar.json
+- `src/grammar.json` - Intermediate grammar representation
+- `src/node-types.json` - AST node type definitions
+
+### Query Files
+
+**queries/** - TreeSitter query files in S-expression syntax:
+- `highlights.scm` - Syntax highlighting rules mapping AST nodes to semantic scopes
+- `injections.scm` - Language injection boundaries
+- `locals.scm` - Variable scope definitions
+- `textobjects.scm` - Editor text object definitions
+
+### Test Corpus
+
+**test/corpus/*.txt** - Test cases in Tree-sitter's corpus format. Each test has a name, ReScript code, and expected parse tree. Files cover: comments, decorators, expressions, functions, JSX, let bindings, literals, modules, type declarations.
+
+## Development Workflow
+
+1. Modify `grammar.js` to add/fix grammar rules
+2. Run `tree-sitter generate` to regenerate the parser
+3. Add test cases in `test/corpus/*.txt` with expected parse trees
+4. Run `tree-sitter test` to verify
+5. For scanner changes, edit `src/scanner.c` and rebuild
+
+## Language Bindings
+
+Bindings are provided for Node.js, Python, Rust, Go, Swift, and C in the `bindings/` directory. CI tests Rust, Python, and Go bindings automatically.

AGENTS.md

@@ -0,0 +1 @@
+AGENTS.md