Reworked WIT syncing to avoid retriggered builds on repeated cargo make build (#3670)

noise64 · web-flow · commit b5dfe83ae052 · 2026-06-29T09:19:21.000+02:00
* Reworked WIT syncing to avoid retriggered builds on repeated `cargo make build`

* cleanups

* harden dir-mirror flag parsing

* test build tools as the "first step" before build, and not as a unit test

* portable diff-wit

* cleanups
diff --git a/.agents/skills/modifying-wit-interfaces/SKILL.md b/.agents/skills/modifying-wit-interfaces/SKILL.md
@@ -82,15 +82,15 @@ Edit the relevant `.wit` file in the root `wit/` directory (e.g., `wit/host.wit`
 cargo make wit
 ```
 
-This removes all `wit/deps/` directories in sub-projects and re-copies the correct subset from the root.
+This mirrors the correct subset of the root `wit/deps/` into each sub-project, idempotently (it rewrites only files whose bytes changed, so unchanged files keep their mtime — avoiding needless rebuilds).
 
 ### Step 3: Verify synchronization
 
 ```shell
 cargo make check-wit
 ```
 
-This re-runs the sync and then `git diff --exit-code wit golem-common/wit` to ensure the committed WIT files match what the sync produces. If this fails in CI, you forgot to run `cargo make wit` (or you hand-edited a generated sub-project copy).
+This re-runs the sync and then `git status` over every per-crate `wit/deps` copy (golem-common, golem-cli, and all four SDKs) to ensure the committed WIT files match what the sync produces. CI runs this; if it fails, you forgot to run `cargo make wit` (or you hand-edited a generated sub-project copy).
 
 ### Step 4: Build and verify
 
@@ -110,10 +110,11 @@ truth — there is no `deps.toml` and nothing is fetched.
 
 ### Step 2: Wire it into the sync tasks
 
-Edit `Makefile.toml` so the new package is copied where it's needed. The
-`wit-sdks` task copies **all** root deps to every SDK automatically, but the
-`wit-golem-common` and `wit-golem-cli` tasks copy an explicit subset — add a
-`cp wit/deps/<package> <target>/wit/deps` line there if those crates need it.
+Edit `Makefile.toml` so the new package is mirrored where it's needed. The
+`wit-sdks` task mirrors **all** root deps to every SDK automatically, but the
+`wit-golem-common` and `wit-golem-cli` tasks mirror an explicit subset — add a
+`wit/deps/<package> <target>/wit/deps/<package>` source/target pair to the
+`dir-mirror` args there if those crates need it.
 
 ### Step 3: Sync and verify
 
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -49,6 +49,9 @@ jobs:
       - uses: ./.github/actions/setup-rust
         with:
           cache-save-if: 'true'
+      # Test the build tooling (dev-tools/) first so a broken tool fails fast
+      - name: Test build tools (dev-tools)
+        run: cargo make --profile ci dev-tools-tests
       - uses: taiki-e/install-action@v2
         with:
           tool: nextest
@@ -125,6 +128,8 @@ jobs:
         run: cargo make --profile ci check-docs-skills
       - name: Check configs are up to date
         run: cargo make --profile ci check-configs
+      - name: Check WIT deps are synced
+        run: cargo make --profile ci check-wit
       - name: Unit tests
         run: cargo make --profile ci unit-tests
       - uses: ./.github/actions/publish-test-report
diff --git a/Cargo.toml b/Cargo.toml
@@ -24,7 +24,8 @@ members = [
 
 exclude = [
     "test-components",
-    "sdks/golem-rust"
+    "sdks/golem-rust",
+    "dev-tools"
 ]
 
 [workspace.metadata]
diff --git a/Makefile.toml b/Makefile.toml
@@ -1,8 +1,8 @@
 # List of top-level tasks intended for use:
 #
 # - `cargo make dev-flow` or just `cargo make`: runs a full development flow, including fixing format and clippy, building and running tests and generating OpenAPI specs
-# - `cargo make wit`: fetches the WIT dependencies based on wit/deps.toml
-# - `cargo make check-wit`: deletes then fetches the WIT dependencies based on wit/deps.toml, then checks if it's up-to-date
+# - `cargo make wit`: syncs the per-crate wit/deps copies from the canonical top-level wit/deps
+# - `cargo make check-wit`: re-syncs the per-crate wit/deps and fails if the committed copies are out of date
 # - `cargo make build`: builds everything in debug mode
 # - `cargo make build-release`: builds everything in release mode. customizable with PLATFORM_OVERRIDE env variable for docker builds
 # - `cargo make check`: runs rustfmt and clippy checks without applying any fix
@@ -72,11 +72,17 @@ dependencies = [
 ]
 
 # WIT DEPENDENCIES
+#
+# Sync the per-crate wit/deps copies from the canonical top-level wit/deps with the
+# dir-mirror tool (dev-tools/dir-mirror) rather than cp: it preserves the mtime of
+# unchanged files, so cargo (which tracks these .wit files via rerun-if-changed from
+# bindgen!) doesn't rebuild the workspace on every run. The <source> <target> pairs
+# below replace the old cp/glob_cp lines one-for-one. golem-common/cli take a per-subdir
+# subset of the deps, so removing a dep means also removing its pair here.
 [tasks.wit]
-description = "Fetches the WIT dependencies based on wit/deps.toml"
+description = "Syncs the per-crate WIT dependency copies from the canonical wit/deps"
 run_task = [
     { name = [
-        "remove-wit-deps",
         "wit-golem-common",
         "wit-golem-cli",
         "wit-sdks",
@@ -85,67 +91,66 @@ run_task = [
 
 [tasks.wit-golem-common]
 private = true
-script_runner = "@duckscript"
-script = """
-rm -r golem-common/wit/deps
-mkdir golem-common/wit/deps
-cp wit/deps/io golem-common/wit/deps
-cp wit/deps/clocks golem-common/wit/deps
-cp wit/deps/golem-1.x golem-common/wit/deps
-cp wit/deps/golem-core-v2 golem-common/wit/deps
-cp wit/deps/golem-agent golem-common/wit/deps
-cp wit/deps/golem-secrets golem-common/wit/deps
-cp wit/deps/golem-tool golem-common/wit/deps
-"""
+command = "cargo"
+args = [
+    "run", "--quiet", "--manifest-path", "dev-tools/Cargo.toml", "-p", "dir-mirror", "--",
+    "--src", "wit/deps/io", "--dst", "golem-common/wit/deps/io",
+    "--src", "wit/deps/clocks", "--dst", "golem-common/wit/deps/clocks",
+    "--src", "wit/deps/golem-1.x", "--dst", "golem-common/wit/deps/golem-1.x",
+    "--src", "wit/deps/golem-core-v2", "--dst", "golem-common/wit/deps/golem-core-v2",
+    "--src", "wit/deps/golem-agent", "--dst", "golem-common/wit/deps/golem-agent",
+    "--src", "wit/deps/golem-secrets", "--dst", "golem-common/wit/deps/golem-secrets",
+    "--src", "wit/deps/golem-tool", "--dst", "golem-common/wit/deps/golem-tool",
+]
 
 [tasks.wit-sdks]
 private = true
-script_runner = "@duckscript"
-script = """
-rm -r sdks/rust/golem-rust/wit/deps
-rm -r sdks/ts/wit/deps
-rm -r sdks/scala/wit/deps
-rm -r sdks/moonbit/golem_sdk/wit/deps
-mkdir sdks/rust/golem-rust/wit/deps
-mkdir sdks/ts/wit/deps
-mkdir sdks/scala/wit/deps
-mkdir sdks/moonbit/golem_sdk/wit/deps
-glob_cp wit/deps/**/* sdks/rust/golem-rust/wit/deps
-glob_cp sdks/rust/golem-rust/wit/golem-ai/**/* sdks/rust/golem-rust/wit/deps
-glob_cp wit/deps/**/* sdks/ts/wit/deps
-glob_cp sdks/ts/wit/golem-ai/**/* sdks/ts/wit/deps
-glob_cp wit/deps/**/* sdks/scala/wit/deps
-glob_cp wit/deps/**/* sdks/moonbit/golem_sdk/wit/deps
-"""
+command = "cargo"
+args = [
+    "run", "--quiet", "--manifest-path", "dev-tools/Cargo.toml", "-p", "dir-mirror", "--",
+    "--src", "wit/deps", "--dst", "sdks/rust/golem-rust/wit/deps",
+    "--src", "wit/deps", "--dst", "sdks/ts/wit/deps",
+    "--src", "wit/deps", "--dst", "sdks/scala/wit/deps",
+    "--src", "wit/deps", "--dst", "sdks/moonbit/golem_sdk/wit/deps",
+]
 
 [tasks.wit-golem-cli]
 private = true
-script_runner = "@duckscript"
-script = """
-rm -r cli/golem-cli/wit/deps
-mkdir cli/golem-cli/wit/deps
-cp wit/deps/clocks cli/golem-cli/wit/deps
-cp wit/deps/io cli/golem-cli/wit/deps
-cp wit/deps/golem-1.x cli/golem-cli/wit/deps
-cp wit/deps/golem-core-v2 cli/golem-cli/wit/deps
-cp wit/deps/golem-agent cli/golem-cli/wit/deps
-cp wit/deps/logging cli/golem-cli/wit/deps
-"""
-
-[tasks.remove-wit-deps]
-private = true
-script_runner = "@duckscript"
-script = """
-rm -rf golem-common/wit/deps
-rm -rf cli/golem-cli/wit/deps
-"""
+command = "cargo"
+args = [
+    "run", "--quiet", "--manifest-path", "dev-tools/Cargo.toml", "-p", "dir-mirror", "--",
+    "--src", "wit/deps/clocks", "--dst", "cli/golem-cli/wit/deps/clocks",
+    "--src", "wit/deps/io", "--dst", "cli/golem-cli/wit/deps/io",
+    "--src", "wit/deps/golem-1.x", "--dst", "cli/golem-cli/wit/deps/golem-1.x",
+    "--src", "wit/deps/golem-core-v2", "--dst", "cli/golem-cli/wit/deps/golem-core-v2",
+    "--src", "wit/deps/golem-agent", "--dst", "cli/golem-cli/wit/deps/golem-agent",
+    "--src", "wit/deps/logging", "--dst", "cli/golem-cli/wit/deps/logging",
+]
 
 [tasks.diff-wit]
 private = true
-script = "git diff --exit-code wit golem-common/wit"
+script_runner = "@duckscript"
+script = '''
+status = exec git status --porcelain -- golem-common/wit/deps cli/golem-cli/wit/deps sdks/rust/golem-rust/wit/deps sdks/ts/wit/deps sdks/scala/wit/deps sdks/moonbit/golem_sdk/wit/deps
+ok = eq ${status.code} 0
+if not ${ok}
+    echo "git status failed (exit ${status.code}):"
+    echo ${status.stderr}
+    trigger_error "git status failed while checking wit/deps"
+end
+changes = trim ${status.stdout}
+empty = is_empty ${changes}
+if not ${empty}
+    echo "Per-crate wit/deps copies are out of sync with the canonical wit/deps:"
+    echo ${changes}
+    echo "Run 'cargo make wit' and commit the result."
+    trigger_error "wit/deps copies are out of sync"
+end
+'''
 
 [tasks.check-wit]
-run_task = [{ name = ["remove-wit-deps", "wit", "diff-wit"] }]
+description = "Re-syncs wit/deps and fails if the committed per-crate copies are out of date"
+run_task = [{ name = ["wit", "diff-wit"] }]
 
 # BUILD
 
@@ -414,7 +419,13 @@ rm -f target/.local-publish-marker
 
 [tasks.check]
 description = "Runs rustfmt and clippy checks without applying any fix"
-dependencies = ["wit", "check-clippy", "check-rustfmt"]
+dependencies = [
+    "wit",
+    "check-clippy",
+    "check-rustfmt",
+    "check-clippy-dev-tools",
+    "check-rustfmt-dev-tools",
+]
 
 [tasks.check-rustfmt]
 description = "Runs rustfmt checks without applying any fix"
@@ -428,12 +439,36 @@ install_crate = "clippy"
 command = "cargo"
 args = ["clippy", "--all-targets", "--", "--no-deps", "-Dwarnings"]
 
+# The dev-tools workspace is separate, so it needs its own fmt/clippy invocations.
+[tasks.check-rustfmt-dev-tools]
+description = "Runs rustfmt checks on the dev-tools workspace without applying any fix"
+command = "cargo"
+args = ["fmt", "--manifest-path", "dev-tools/Cargo.toml", "--all", "--", "--check"]
+
+[tasks.check-clippy-dev-tools]
+description = "Runs clippy checks on the dev-tools workspace without applying any fix"
+command = "cargo"
+args = [
+    "clippy",
+    "--manifest-path",
+    "dev-tools/Cargo.toml",
+    "--all-targets",
+    "--",
+    "--no-deps",
+    "-Dwarnings",
+]
+
 ## ** FIX **
 
 [tasks.fix]
 description = "Runs rustfmt and clippy checks and applies fixes"
 #dependencies = ["wit", "fix-clippy", "fix-rustfmt"]
-dependencies = ["fix-clippy", "fix-rustfmt"]
+dependencies = [
+    "fix-clippy",
+    "fix-rustfmt",
+    "fix-clippy-dev-tools",
+    "fix-rustfmt-dev-tools",
+]
 
 [tasks.fix-rustfmt]
 description = "Runs rustfmt checks and applies fixes"
@@ -456,6 +491,27 @@ args = [
     "-Dwarnings",
 ]
 
+[tasks.fix-rustfmt-dev-tools]
+description = "Runs rustfmt on the dev-tools workspace and applies fixes"
+command = "cargo"
+args = ["fmt", "--manifest-path", "dev-tools/Cargo.toml", "--all"]
+
+[tasks.fix-clippy-dev-tools]
+description = "Runs clippy on the dev-tools workspace and applies fixes"
+command = "cargo"
+args = [
+    "clippy",
+    "--manifest-path",
+    "dev-tools/Cargo.toml",
+    "--all-targets",
+    "--fix",
+    "--allow-dirty",
+    "--allow-staged",
+    "--",
+    "--no-deps",
+    "-Dwarnings",
+]
+
 ## ** TEST **
 
 [tasks.test]
@@ -473,6 +529,11 @@ script = '''
 cargo-test-r run --workspace --lib -- --nocapture --report-time $JUNIT_OPTS
 '''
 
+[tasks.dev-tools-tests]
+description = "Runs the dev-tools workspace unit tests"
+command = "cargo"
+args = ["test", "--manifest-path", "dev-tools/Cargo.toml", "--lib"]
+
 [tasks.worker-executor-tests]
 dependencies = ["wit"]
 description = "Runs worker executor tests only"
diff --git a/dev-tools/Cargo.lock b/dev-tools/Cargo.lock
diff --git a/dev-tools/Cargo.toml b/dev-tools/Cargo.toml
@@ -0,0 +1,7 @@
+# Separate workspace for dev/build tooling, kept out of the main workspace's
+# --workspace/clippy/udeps/release. Tools here are intentionally std-only (no
+# dependencies) so they compile/check/test almost instantly — see README.md.
+# Add more tool crates as members.
+[workspace]
+resolver = "2"
+members = ["dir-mirror"]
diff --git a/dev-tools/README.md b/dev-tools/README.md
@@ -0,0 +1,42 @@
+# dev-tools
+
+A **separate cargo workspace** for small build/dev tooling, kept out of the main
+workspace (so it isn't pulled into `cargo build --workspace`, clippy, udeps, release,
+or `set-version`).
+
+## Intentionally std-only
+
+Tools here use **only the Rust standard library — no dependencies, not even for tests.**
+This is deliberate: these tools run on the hot path (e.g. `dir-mirror` is invoked by the
+`wit` cargo-make task before every build), so they must compile, check, and test almost
+instantly. Pulling in a dependency tree (proc-macros, async runtimes, a test framework,
+etc.) would add seconds of cold-compile time to the build and to CI for no real benefit
+at this size.
+
+Concretely:
+
+- **No runtime/build dependencies** — the `[dependencies]` table stays empty.
+- **No dev-dependencies** — tests are plain `#[test]` (libtest) with small std helpers
+  (e.g. a tiny temp-dir struct), not a test framework. This differs from the main
+  workspace, which uses `test-r`/`cargo-test-r`; that machinery is not worth its
+  compile cost here.
+- Run the tests with plain `cargo test` (the cargo-make `dev-tools-tests` task), **not**
+  `cargo-test-r` — in CI cargo-test-r runs in nextest-archive-reuse mode for the main
+  workspace and rejects `--manifest-path` for a separate workspace.
+
+If a tool ever genuinely needs a dependency, reconsider whether it belongs here or in the
+main workspace.
+
+## Tools
+
+- **`dir-mirror`** — idempotent directory mirror: makes a destination directory a
+  byte-identical copy of a source directory, rewriting only changed files (preserving
+  mtimes) and pruning stale ones. Used by the `wit` task to sync the per-crate
+  `wit/deps` copies without triggering needless rebuilds.
+
+## CI / cargo-make
+
+- `cargo make dev-tools-tests` — runs the tests (plain `cargo test`). In CI this runs
+  first in the `build-and-store` job, to fail fast if the build tooling is broken.
+- `cargo make check` / `cargo make fix` — also lint this workspace (the
+  `*-dev-tools` rustfmt/clippy tasks).
diff --git a/dev-tools/dir-mirror/Cargo.toml b/dev-tools/dir-mirror/Cargo.toml
@@ -0,0 +1,14 @@
+[package]
+name = "dir-mirror"
+version = "0.0.0"
+edition = "2024"
+license-file = "../../LICENSE"
+publish = false
+description = "Idempotent directory mirror used to keep per-crate wit/deps copies in sync"
+
+# No dependencies — std only, on purpose (see dev-tools/README.md). Tests use plain
+# `cargo test` (libtest), so there are no dev-dependencies either.
+
+[[bin]]
+name = "dir-mirror"
+test = false
diff --git a/dev-tools/dir-mirror/src/lib.rs b/dev-tools/dir-mirror/src/lib.rs
diff --git a/dev-tools/dir-mirror/src/main.rs b/dev-tools/dir-mirror/src/main.rs

Original file line number	Diff line number	Diff line change
`@@ -24,7 +24,8 @@ members = [`
`24`	`24`
`25`	`25`	`exclude = [`
`26`	`26`	`"test-components",`
`27`		`- "sdks/golem-rust"`
	`27`	`+ "sdks/golem-rust",`
	`28`	`+ "dev-tools"`
`28`	`29`	`]`
`29`	`30`
`30`	`31`	`[workspace.metadata]`