chore: Comprehensive randomized fuzzy mutation testing

It's such a shame that we keep commiting regressions that are leading ot
the missed events from the file watcher. I want to minimize this to 0.

Author: dmtrKovalenko

.github/workflows/rust.yml

@@ -39,6 +39,76 @@ jobs:
       - name: Run tests
         run: cargo test --features zlob --workspace --exclude fff-nvim
 
+  stress-test:
+    name: Stress Test (Watcher + Git)
+    # Cross-platform: macOS uses FSEvents, Ubuntu uses inotify — different
+    # backends with different debouncer quirks and race windows, so
+    # running the same fuzz suite on both is the whole point. A job
+    # failure on either platform fails the CI gate.
+    #
+    # Matches the `test` job matrix above; Windows is not exercised
+    # anywhere else in CI, so wiring it up here would be asymmetric and
+    # would surface `ReadDirectoryChangesW`-specific noise without a
+    # corresponding signal. Add it when the rest of CI does.
+    runs-on: ${{ matrix.os }}
+    strategy:
+      # Keep going after one OS fails so we can see whether a bug
+      # reproduces everywhere or is platform-specific.
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+    # Long-running; don't let a stuck watcher thread burn a full CI
+    # timeout. Two scenarios should finish well under this limit.
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Install Zig
+        uses: goto-bus-stop/setup-zig@v2
+        with:
+          version: 0.16.0
+
+      - name: Install Rust
+        uses: actions-rust-lang/setup-rust-toolchain@v1.15.4
+        with:
+          cache: true
+          cache-on-failure: true
+          cache-key: "v1-rust-stress-${{ matrix.os }}"
+          components: rustfmt, clippy
+
+      # Run 1: deterministic. Pins a regression surface that must not
+      # regress between builds. Seed is hard-coded in the Makefile; if
+      # you need to replay a different seed locally, prefix with
+      # `FFF_STRESS_SEED=<u64|0xHEX>`.
+      - name: Stress test (seeded / deterministic)
+        shell: bash
+        run: make test-stress-seeded
+        env:
+          FFF_STRESS_CASES: "3"
+          FFF_STRESS_MIN_OPS: "30"
+          FFF_STRESS_MAX_OPS: "50"
+
+      # Run 2: random. Every CI build explores a fresh slice of the op
+      # space. If this ever goes red, proptest persists the failing seed
+      # to `crates/fff-core/tests/fuzz_git_watcher_stress.proptest-regressions`;
+      # that file gets uploaded as an artifact so the failure is
+      # reproducible locally.
+      - name: Stress test (random / proptest)
+        shell: bash
+        run: make test-stress-random
+        env:
+          FFF_STRESS_CASES: "5"
+          FFF_STRESS_MIN_OPS: "30"
+          FFF_STRESS_MAX_OPS: "60"
+
+      - name: Upload proptest regressions on failure
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: proptest-regressions-${{ matrix.os }}
+          path: crates/fff-core/tests/fuzz_git_watcher_stress.proptest-regressions
+          if-no-files-found: ignore
+
   fmt:
     name: cargo fmt
     runs-on: ubuntu-latest

Cargo.lock

@@ -4,7 +4,17 @@ PREFIX ?= /usr/local
 LIBDIR ?= $(PREFIX)/lib
 INCLUDEDIR ?= $(PREFIX)/include
 
-.PHONY: build build-c-lib install uninstall test test-rust test-lua test-version test-bun test-node prepare-bun prepare-node set-npm-version header
+# Compile-time cfg that gates the watcher + git-status fuzz stress test.
+# Always set on `test-stress*` targets via RUSTFLAGS.
+STRESS_RUSTFLAGS := --cfg stress
+
+# Pinned seed for the deterministic stress run. Changing this value is
+# equivalent to picking a new regression surface, so keep it stable —
+# that's the whole point. Override ad-hoc on the command line via
+# `FFF_STRESS_SEED=<u64|0xhex> make test-stress-seeded`.
+FFF_STRESS_DEFAULT_SEED ?= 0xDEADBEEFCAFEBABE
+
+.PHONY: build build-c-lib install uninstall test test-rust test-lua test-version test-bun test-node prepare-bun prepare-node set-npm-version header test-stress test-stress-seeded test-stress-random
 
 all: format test lint
 
@@ -85,6 +95,26 @@ test-node: prepare-node
 
 test: test-rust test-lua test-version test-bun test-node
 
+
+test-stress-seeded:
+	FFF_STRESS_SEED="$${FFF_STRESS_SEED:-$(FFF_STRESS_DEFAULT_SEED)}" \
+	RUSTFLAGS="$(STRESS_RUSTFLAGS)" \
+	cargo test \
+		-p fff-search \
+		--test fuzz_git_watcher_stress \
+		--features zlob \
+		-- --nocapture stress_seeded
+
+test-stress-random:
+	RUSTFLAGS="$(STRESS_RUSTFLAGS)" \
+	cargo test \
+		-p fff-search \
+		--test fuzz_git_watcher_stress \
+		--features zlob \
+		-- --nocapture stress_random
+
+test-stress: test-stress-seeded test-stress-random
+
 # Update version in a package.json, including optionalDependencies.
 # Usage: make set-npm-version PKG=packages/fff-bun VERSION=1.0.0-nightly.abc1234
 set-npm-version:

Makefile

@@ -79,5 +79,6 @@ dunce = { workspace = true }
 
 [dev-dependencies]
 criterion = { version = "0.5", features = ["html_reports"] }
+proptest = { version = "1", default-features = false, features = ["std", "fork"] }
 rand = { version = "0.8", features = ["small_rng"] }
 tempfile = "3.8"

crates/fff-core/Cargo.toml

@@ -127,9 +127,7 @@ fn bench_bigram_build(c: &mut Criterion) {
                     ));
                     s.push_str("    let parsed = ctx.parse()?;\n");
                     s.push_str("    let validated = parsed.validate()?;\n");
-                    s.push_str(&format!(
-                        "    ctx.respond(validated, {}).await\n"
-                    , j));
+                    s.push_str(&format!("    ctx.respond(validated, {}).await\n", j));
                     s.push_str("}\n\n");
                 }
                 s

crates/fff-core/benches/bigram_bench.rs

@@ -1,4 +1,12 @@
 fn main() {
+    // Opt-in cfg for the long-running randomized stress tests
+    // used by tests/fuzz_git_watcher_stress.rs
+    println!("cargo::rustc-check-cfg=cfg(stress)");
+
+    // When the `zlob` feature is enabled (Zig-compiled C library):
+    // On Windows MSVC, explicitly link the C runtime libraries.
+    // Zig-compiled static libraries don't emit /DEFAULTLIB directives for the
+    // MSVC CRT, so symbols like strcmp, memcpy etc. would be unresolved.
     if std::env::var("CARGO_FEATURE_ZLOB").is_ok() {
         if !zig_available() {
             panic!(