fix(grammar): build grammar on CI (#698)

Author: juleswritescode

.github/workflows/pull_request.yml

@@ -132,10 +132,12 @@ jobs:
             ~/.cargo/bin/tree-sitter
             ~/.cargo/bin/sqlx
             ~/.cargo/bin/cargo-sqlx
-          key: ${{ runner.os }}-tree-sitter-${{ hashFiles('rust-toolchain.toml') }}
+            target/**/build/pgls_treesitter_grammar-*/out/generated
+          key: ${{ runner.os }}-tree-sitter-${{ hashFiles('.tree-sitter-cli-version', 'crates/pgls_treesitter_grammar/grammar.js', 'crates/pgls_treesitter_grammar/tree-sitter.json', 'rust-toolchain.toml', 'Cargo.lock') }}
 
       - name: Setup tree-sitter
-        run: command -v tree-sitter || cargo install tree-sitter-cli
+        run: command -v tree-sitter || cargo install tree-sitter-cli --version "$(cat .tree-sitter-cli-version)" --locked
+        shell: bash
 
       - name: Setup sqlx-cli
         run: command -v sqlx ||cargo install sqlx-cli
@@ -199,10 +201,12 @@ jobs:
         with:
           path: |
             ~/.cargo/bin/tree-sitter
-          key: ${{ runner.os }}-tree-sitter-${{ hashFiles('rust-toolchain.toml') }}
+            target/**/build/pgls_treesitter_grammar-*/out/generated
+          key: ${{ runner.os }}-tree-sitter-${{ hashFiles('.tree-sitter-cli-version', 'crates/pgls_treesitter_grammar/grammar.js', 'crates/pgls_treesitter_grammar/tree-sitter.json', 'rust-toolchain.toml', 'Cargo.lock') }}
 
-      - name: Setup Postgres
-        run: command -v tree-sitter || cargo install tree-sitter-cli
+      - name: Setup tree-sitter
+        run: command -v tree-sitter || cargo install tree-sitter-cli --version "$(cat .tree-sitter-cli-version)" --locked
+        shell: bash
 
       - name: Run tests
         run: cargo test --workspace
@@ -236,9 +240,11 @@ jobs:
         with:
           path: |
             ~/.cargo/bin/tree-sitter
-          key: ${{ runner.os }}-tree-sitter-${{ hashFiles('rust-toolchain.toml') }}
+            target/**/build/pgls_treesitter_grammar-*/out/generated
+          key: ${{ runner.os }}-tree-sitter-${{ hashFiles('.tree-sitter-cli-version', 'crates/pgls_treesitter_grammar/grammar.js', 'crates/pgls_treesitter_grammar/tree-sitter.json', 'rust-toolchain.toml', 'Cargo.lock') }}
       - name: setup tree-sitter
-        run: command -v tree-sitter || cargo install tree-sitter-cli
+        run: command -v tree-sitter || cargo install tree-sitter-cli --version "$(cat .tree-sitter-cli-version)" --locked
+        shell: bash
       - name: Build main binary
         run: cargo build -p pgls_cli --release
       - name: Setup Bun
@@ -324,9 +330,11 @@ jobs:
         with:
           path: |
             ~/.cargo/bin/tree-sitter
-          key: ${{ runner.os }}-tree-sitter-${{ hashFiles('rust-toolchain.toml') }}
+            target/**/build/pgls_treesitter_grammar-*/out/generated
+          key: ${{ runner.os }}-tree-sitter-${{ hashFiles('.tree-sitter-cli-version', 'crates/pgls_treesitter_grammar/grammar.js', 'crates/pgls_treesitter_grammar/tree-sitter.json', 'rust-toolchain.toml', 'Cargo.lock') }}
       - name: setup tree-sitter
-        run: command -v tree-sitter || cargo install tree-sitter-cli
+        run: command -v tree-sitter || cargo install tree-sitter-cli --version "$(cat .tree-sitter-cli-version)" --locked
+        shell: bash
       - name: Ensure RustFMT on nightly toolchain
         run: rustup component add rustfmt --toolchain nightly
       - name: Setup Bun

.github/workflows/release.yml

@@ -120,10 +120,12 @@ jobs:
         with:
           path: |
             ~/.cargo/bin/tree-sitter
-          key: ${{ runner.os }}-tree-sitter-${{ hashFiles('rust-toolchain.toml') }}
+            target/**/build/pgls_treesitter_grammar-*/out/generated
+          key: ${{ runner.os }}-tree-sitter-${{ hashFiles('.tree-sitter-cli-version', 'crates/pgls_treesitter_grammar/grammar.js', 'crates/pgls_treesitter_grammar/tree-sitter.json', 'rust-toolchain.toml', 'Cargo.lock') }}
 
       - name: Setup tree-sitter
-        run: command -v tree-sitter || cargo install tree-sitter-cli
+        run: command -v tree-sitter || cargo install tree-sitter-cli --version "$(cat .tree-sitter-cli-version)" --locked
+        shell: bash
 
       - name: Setup Postgres
         uses: ./.github/actions/setup-postgres

.tree-sitter-cli-version

@@ -0,0 +1 @@
+0.25.9

crates/pgls_treesitter_grammar/Cargo.toml

@@ -1,68 +1,135 @@
+use std::collections::hash_map::DefaultHasher;
+use std::hash::{Hash, Hasher};
+use std::path::{Path, PathBuf};
+use std::process::Command;
+
 fn main() {
-    let grammar_file = std::path::Path::new("grammar.js");
-    let src_dir = std::path::Path::new("src");
-    let parser_path = src_dir.join("parser.c");
+    let manifest_dir =
+        PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").expect("Missing CARGO_MANIFEST_DIR"));
+    let out_dir = PathBuf::from(std::env::var("OUT_DIR").expect("Missing OUT_DIR"));
+    let grammar_file = manifest_dir.join("grammar.js");
+    let config_file = manifest_dir.join("tree-sitter.json");
+    let src_dir = manifest_dir.join("src");
+    let scanner_path = src_dir.join("scanner.c");
+    let generated_dir = out_dir.join("generated");
+    let parser_path = generated_dir.join("parser.c");
+    let node_types_path = generated_dir.join("node-types.json");
+    let stamp_path = generated_dir.join(".stamp");
+
+    println!("cargo:rerun-if-changed={}", grammar_file.display());
+    println!("cargo:rerun-if-changed={}", config_file.display());
+    println!("cargo:rerun-if-changed={}", scanner_path.display());
+    println!(
+        "cargo:rerun-if-changed={}",
+        manifest_dir.join("build.rs").display()
+    );
 
-    // Detect Emscripten target for WASM builds
+    // Detect Emscripten target for WASM builds.
     let target = std::env::var("TARGET").unwrap_or_default();
     let is_emscripten = target.contains("emscripten");
 
-    // regenerate parser if grammar.js changes
-    println!("cargo:rerun-if-changed={}", grammar_file.to_str().unwrap());
-
-    // generate parser if it does not exist.
-    if !parser_path.exists() || is_file_newer(grammar_file, parser_path.as_path()) {
-        let output = std::process::Command::new("tree-sitter")
-            .arg("generate")
-            .output();
-
-        match output {
-            Ok(result) if result.status.success() => {
-                println!("cargo:warning=Successfully generated parser from grammar.js");
-            }
-            Ok(result) => {
-                panic!(
-                    "Failed to generate parser: {}",
-                    String::from_utf8_lossy(&result.stderr)
-                );
-            }
-            Err(_) => {
-                panic!("tree-sitter CLI not found. Please install it with: `just install`");
-            }
-        }
+    std::fs::create_dir_all(&generated_dir).expect("Failed to create generated output directory");
+
+    let stamp = compute_stamp([&grammar_file, &config_file]);
+    let should_regenerate =
+        !parser_path.exists() || !node_types_path.exists() || read_stamp(&stamp_path) != stamp;
+
+    if should_regenerate {
+        generate_grammar(&grammar_file, &config_file, &generated_dir);
+        std::fs::write(&stamp_path, &stamp).expect("Failed to write grammar stamp");
     }
 
     let mut c_config = cc::Build::new();
 
-    // Use Emscripten compiler for WASM builds
+    // Use Emscripten compiler for WASM builds.
     if is_emscripten {
         c_config.compiler("emcc").archiver("emar");
     }
 
-    c_config.std("c11").include(src_dir);
+    // Generated parser.c includes tree_sitter headers from generated_dir/tree_sitter.
+    // scanner.c still lives in src/, so both include roots are required.
+    c_config
+        .std("c11")
+        .include(&generated_dir)
+        .include(&src_dir);
 
     #[cfg(target_env = "msvc")]
     c_config.flag("-utf-8");
 
     c_config.file(&parser_path);
-    println!("cargo:rerun-if-changed={}", parser_path.to_str().unwrap());
-
-    let scanner_path = src_dir.join("scanner.c");
     if scanner_path.exists() {
         c_config.file(&scanner_path);
-        println!("cargo:rerun-if-changed={}", scanner_path.to_str().unwrap());
     }
 
     c_config.compile("tree_sitter_pgls");
 }
 
-fn is_file_newer(file1: &std::path::Path, file2: &std::path::Path) -> bool {
-    if !file1.exists() || !file2.exists() {
-        return true;
+fn compute_stamp(files: [&Path; 2]) -> String {
+    let mut hasher = DefaultHasher::new();
+
+    for file in files {
+        file.as_os_str().hash(&mut hasher);
+        let contents = std::fs::read(file).unwrap_or_else(|error| {
+            panic!("Failed to read {}: {error}", file.display());
+        });
+        contents.hash(&mut hasher);
     }
 
-    let modified1 = file1.metadata().unwrap().modified().unwrap();
-    let modified2 = file2.metadata().unwrap().modified().unwrap();
+    format!("{:016x}", hasher.finish())
+}
 
-    modified1 > modified2
+fn read_stamp(stamp_path: &Path) -> String {
+    std::fs::read_to_string(stamp_path)
+        .map(|value| value.trim().to_owned())
+        .unwrap_or_default()
+}
+
+fn generate_grammar(grammar_file: &Path, config_file: &Path, generated_dir: &Path) {
+    // tree-sitter generate updates tree-sitter.json in its working directory.
+    // Use an isolated temp workdir under OUT_DIR to avoid mutating repository files.
+    let generator_workdir = generated_dir.join("tree-sitter-workdir");
+    let work_grammar = generator_workdir.join("grammar.js");
+    let work_config = generator_workdir.join("tree-sitter.json");
+
+    let _ = std::fs::remove_dir_all(&generator_workdir);
+    std::fs::create_dir_all(&generator_workdir)
+        .expect("Failed to create temporary tree-sitter generator workdir");
+    std::fs::copy(grammar_file, &work_grammar).unwrap_or_else(|error| {
+        panic!(
+            "Failed to copy {} into generator workdir: {error}",
+            grammar_file.display()
+        );
+    });
+    std::fs::copy(config_file, &work_config).unwrap_or_else(|error| {
+        panic!(
+            "Failed to copy {} into generator workdir: {error}",
+            config_file.display()
+        );
+    });
+
+    let output = Command::new("tree-sitter")
+        .arg("generate")
+        .arg("grammar.js")
+        .arg("--output")
+        .arg(generated_dir)
+        .current_dir(&generator_workdir)
+        .output();
+
+    let _ = std::fs::remove_dir_all(&generator_workdir);
+
+    match output {
+        Ok(result) if result.status.success() => {}
+        Ok(result) => {
+            panic!(
+                "Failed to generate tree-sitter grammar.\nstdout:\n{}\nstderr:\n{}",
+                String::from_utf8_lossy(&result.stdout),
+                String::from_utf8_lossy(&result.stderr)
+            );
+        }
+        Err(error) => {
+            panic!(
+                "tree-sitter CLI not found ({error}). Please install it with: `just install-tools`"
+            );
+        }
+    }
 }