Ship pregenerated bindings in ruby-prism-sys

.gitattributes

@@ -10,6 +10,7 @@ test/prism/snapshots/**/*.txt linguist-generated
 
 rbi/generated/**/*.rbi linguist-generated
 sig/generated/**/*.rbs linguist-generated
+rust/ruby-prism-sys/src/bindings.rs linguist-generated
 
 # All .rb files should have LF line ending, even on Windows, regardless of the git config core.autocrlf value.
 # All .txt should have their line endings as committed in the repository (there are some intentional CR in there),

.github/workflows/rust-bindings.yml

@@ -47,10 +47,40 @@ jobs:
       - name: Run tests
         run: bundle exec rake cargo:test
 
+      - name: Run tests with buildtime_bindgen
+        run: cargo test --features buildtime_bindgen -- --nocapture
+        working-directory: rust/ruby-prism-sys
+
+      - name: Check pregenerated bindings on wasm32-wasip1
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        run: cargo check --target wasm32-wasip1 --no-default-features
+        working-directory: rust/ruby-prism-sys
+        env:
+          PRISM_LIB_DIR: "."
+
       - name: Run examples
         if: ${{ matrix.os != 'windows-latest' }}
         run: bundle exec rake cargo:examples
 
+  sys-bindings:
+    name: cargo:bindings
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ruby
+          bundler-cache: true
+      - name: Set up Rust
+        uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: "1.91.1"
+      - name: rake cargo:bindings
+        run: bundle exec rake cargo:bindings
+      - name: Check that the pregenerated bindings are up to date
+        run: git diff --color --no-ext-diff --exit-code -- rust/ruby-prism-sys/src/bindings.rs
+
   lint:
     name: cargo:lint
     runs-on: ubuntu-latest

docs/releasing.md

@@ -57,6 +57,8 @@ chruby ruby-4.1.0-dev && BUNDLE_GEMFILE=gemfiles/4.1/Gemfile bundle install
 bundle exec rake cargo:build
 ```
 
+* Regenerate `rust/ruby-prism-sys/src/bindings.rs` with `bundle exec rake cargo:bindings`. CI fails if the committed file is stale.
+
 * Commit all of the updated files:
 
 ```sh

rakelib/cargo.rake

@@ -39,6 +39,31 @@ namespace :cargo do
     end
   end
 
+  desc "Regenerate the pregenerated bindings for ruby-prism-sys"
+  task bindings: [:build] do
+    gemspec = Gem::Specification.load("prism.gemspec")
+
+    generated_path = File.expand_path("../rust/target/pregenerated-bindings.rs", __dir__)
+    mkdir_p(File.dirname(generated_path))
+
+    Dir.chdir("rust/ruby-prism-sys") do
+      sh({ "PRISM_GENERATE_BINDINGS" => generated_path }, "cargo check --features buildtime_bindgen")
+    end
+
+    generated = File.read(generated_path)
+    bindgen_version = generated[/automatically generated by rust-bindgen (\S+)/, 1]
+    raise "Could not find the bindgen version banner in the generated bindings" if bindgen_version.nil?
+
+    header = <<~HEADER
+      /* Generated by `bundle exec rake cargo:bindings`. Do not edit by hand.
+       * prism #{gemspec.version}, bindgen #{bindgen_version}, layout_tests(false)
+       * so the file stays portable across pointer widths.
+       */
+    HEADER
+
+    File.write("rust/ruby-prism-sys/src/bindings.rs", header + generated)
+  end
+
   desc "Run all cargo tests"
   task test: [:build] do
     CRATES.each do |crate|

rust/ruby-prism-sys/Cargo.toml

@@ -22,10 +22,11 @@ build = "build/main.rs"
 include = ["src/", "build/", "Cargo.toml", "Cargo.lock", "README.md", "vendor"]
 
 [build-dependencies]
-bindgen = "0.72"
+bindgen = { version = "0.72", optional = true }
 cc = { version = "1.0", optional = true }
 shlex = "1.3"
 
 [features]
 default = ["vendored"]
 vendored = ["dep:cc"]
+buildtime_bindgen = ["dep:bindgen"]

rust/ruby-prism-sys/README.md

@@ -32,14 +32,20 @@ In addition to the Ruby prism dependencies, you shouldn't need anything else bes
 
 ### Updating bindings
 
-`build.rs` (which gets called as part of running `cargo build`, `cargo test`, etc) is where we tell
-`bindgen` which types, functions, etc. that we want it to generate for us. It's smart enough to know
-to generate dependencies for items we specify in there (ex. `pm_parser_t` has fields of type
-`pm_token_t`, but we don't need to tell `bindgen` about `pm_token_t`--it'll figure it out and
-generate bindings for that type too).
-
-If you want to generate new bindings, update `build.rs` accordingly, then run `cargo doc` and check
-the docs; that should tell you if `bindgen` generated all the things you need or not.
+The bindings are pregenerated and committed at `src/bindings.rs`, so building this crate does not
+require `bindgen` or libclang. `build/main.rs` is where we tell `bindgen` which types, functions,
+etc. that we want it to generate for us. It's smart enough to know to generate dependencies for
+items we specify in there (ex. `pm_parser_t` has fields of type `pm_token_t`, but we don't need to
+tell `bindgen` about `pm_token_t`--it'll figure it out and generate bindings for that type too).
+
+If you want to generate new bindings, update the allowlist in `build/main.rs` accordingly, then run
+`bundle exec rake cargo:bindings` from the repository root (this requires libclang) to regenerate
+`src/bindings.rs`. CI regenerates the bindings and fails if the committed file is stale.
+
+The `buildtime_bindgen` feature is the escape hatch that regenerates the bindings with `bindgen` at
+build time instead of using the pregenerated file. It's needed when `RUBY_PRISM_CFLAGS` contains
+defines that change the API surface, since the pregenerated bindings only reflect the default
+configuration.
 
 ### Testing
 

rust/ruby-prism-sys/build/main.rs

@@ -19,15 +19,30 @@ fn main() {
     // Add `[root]/build/` to the search paths, so it can find `libprism.a`.
     println!("cargo:rustc-link-search=native={}", ruby_build_path.to_str().unwrap());
 
-    // This is where the magic happens.
-    let bindings = generate_bindings(&ruby_include_path, &clang_args);
-
-    // Write the bindings to file.
-    write_bindings(&bindings);
+    #[cfg(not(feature = "buildtime_bindgen"))]
+    let _ = clang_args;
+
+    #[cfg(feature = "buildtime_bindgen")]
+    {
+        // This is where the magic happens.
+        let bindings = generate_bindings(&ruby_include_path, &clang_args, true);
+
+        // Write the bindings to file.
+        write_bindings(&bindings);
+
+        // The pregenerated copy omits layout tests, which hardcode the host
+        // target's sizes and would break other pointer widths (e.g. wasm32).
+        if let Some(path) = std::env::var_os("PRISM_GENERATE_BINDINGS") {
+            generate_bindings(&ruby_include_path, &clang_args, false)
+                .write_to_file(&path)
+                .expect("Couldn't write pregenerated bindings!");
+        }
+    }
 }
 
 fn emit_rerun_hints(ruby_include_path: &Path) {
     println!("cargo:rerun-if-env-changed=RUBY_PRISM_CFLAGS");
+    println!("cargo:rerun-if-env-changed=PRISM_GENERATE_BINDINGS");
     println!("cargo:rerun-if-env-changed=PRISM_INCLUDE_DIR");
     println!("cargo:rerun-if-env-changed=PRISM_LIB_DIR");
     println!("cargo:rerun-if-changed={}", ruby_include_path.display());
@@ -73,9 +88,11 @@ fn cargo_manifest_path() -> PathBuf {
     PathBuf::from(std::env::var_os("CARGO_MANIFEST_DIR").unwrap())
 }
 
+#[cfg(feature = "buildtime_bindgen")]
 #[derive(Debug)]
 struct Callbacks;
 
+#[cfg(feature = "buildtime_bindgen")]
 impl bindgen::callbacks::ParseCallbacks for Callbacks {
     /// Accept a string that is a comment on a node. Replace any internal code
     /// examples that are indented by 4 spaces with a fenced code block.
@@ -121,17 +138,19 @@ impl bindgen::callbacks::ParseCallbacks for Callbacks {
 ///
 /// This method only generates code in memory here--it doesn't write it to file.
 ///
-fn generate_bindings(ruby_include_path: &Path, clang_args: &[String]) -> bindgen::Bindings {
+#[cfg(feature = "buildtime_bindgen")]
+fn generate_bindings(ruby_include_path: &Path, clang_args: &[String], layout_tests: bool) -> bindgen::Bindings {
     bindgen::Builder::default()
         .derive_default(true)
+        .formatter(bindgen::Formatter::Prettyplease)
         .generate_block(true)
         .generate_comments(true)
         .header(ruby_include_path.join("prism.h").to_str().unwrap())
         .clang_arg(format!("-I{}", ruby_include_path.to_str().unwrap()))
         .clang_arg("-fparse-all-comments")
         .clang_args(clang_args)
         .impl_debug(true)
-        .layout_tests(true)
+        .layout_tests(layout_tests)
         .merge_extern_blocks(true)
         .parse_callbacks(Box::new(bindgen::CargoCallbacks::new()))
         .parse_callbacks(Box::new(Callbacks))
@@ -226,6 +245,7 @@ fn generate_bindings(ruby_include_path: &Path, clang_args: &[String]) -> bindgen
 
 /// Write the bindings to the `$OUT_DIR/bindings.rs` file. We'll pull these into
 /// the actual library in `src/lib.rs`.
+#[cfg(feature = "buildtime_bindgen")]
 fn write_bindings(bindings: &bindgen::Bindings) {
     let out_path = PathBuf::from(std::env::var_os("OUT_DIR").unwrap());