ci(perf): isolate JMH runtime locks (#60)

TypeThe0ry · web-flow · commit 457086e6df50 · 2026-07-16T13:35:57.000+08:00
diff --git a/.github/workflows/performance-ab.yml b/.github/workflows/performance-ab.yml
@@ -130,6 +130,11 @@ jobs:
             --group "$(id -g)" \
             --mode 0711 \
             "$PERF_RESULTS_ROOT/evidence"
+          sudo install -d \
+            --owner "$(id -u)" \
+            --group "$(id -g)" \
+            --mode 0711 \
+            "$PERF_RESULTS_ROOT/runtime-tmp"
 
       - name: Make wrappers executable
         shell: bash
@@ -229,6 +234,7 @@ jobs:
           sudo -H -u mahjong-benchmark -- test ! -w "$PERF_RESULTS_ROOT/evidence/jars/base-jmh.jar"
           sudo -H -u mahjong-benchmark -- test ! -w "$PERF_RESULTS_ROOT/evidence/jars/candidate-jmh.jar"
           sudo -H -u mahjong-benchmark -- test ! -w "$PERF_RESULTS_ROOT/evidence"
+          sudo -H -u mahjong-benchmark -- test ! -w "$PERF_RESULTS_ROOT/runtime-tmp"
 
       - name: Run A/A drift control
         shell: bash
@@ -256,6 +262,7 @@ jobs:
             --candidate-sha "$CANDIDATE_SHA" \
             --config base/perf/ab/gate-config.json \
             --output-dir "$PERF_RESULTS_ROOT/evidence/aa" \
+            --runtime-temp-root "$PERF_RESULTS_ROOT/runtime-tmp" \
             --java "$JAVA_HOME/bin/java" \
             --candidate-command-prefix-json '["sudo","-H","-u","mahjong-benchmark","--"]' \
             --candidate-file-user mahjong-benchmark \
@@ -288,6 +295,7 @@ jobs:
             --candidate-sha "$CANDIDATE_SHA" \
             --config base/perf/ab/gate-config.json \
             --output-dir "$PERF_RESULTS_ROOT/evidence/ab" \
+            --runtime-temp-root "$PERF_RESULTS_ROOT/runtime-tmp" \
             --java "$JAVA_HOME/bin/java" \
             --candidate-command-prefix-json '["sudo","-H","-u","mahjong-benchmark","--"]' \
             --candidate-file-user mahjong-benchmark \
diff --git a/.github/workflows/performance-infra.yml b/.github/workflows/performance-infra.yml
@@ -79,6 +79,11 @@ jobs:
             --group "$(id -g)" \
             --mode 0711 \
             "$PROBE_ROOT/evidence"
+          sudo install -d \
+            --owner "$(id -u)" \
+            --group "$(id -g)" \
+            --mode 0711 \
+            "$PROBE_ROOT/runtime-tmp"
           python3 perf/ab/tests/isolation_probe.py \
             --user mahjong-probe \
             --output-dir "$PROBE_ROOT/evidence"
@@ -92,6 +97,49 @@ jobs:
           "-PmahjongPaperDevBundle=1.20.1-R0.1-SNAPSHOT"
           "-PmahjongJavaToolchain=25" "-PmahjongJavaTarget=17"
 
+      - name: Regress isolated JMH lock ownership
+        shell: bash
+        run: |
+          set -euo pipefail
+          PROBE_ROOT="/tmp/mahjong-performance-probe-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          SOURCE_JAR="$(find build/libs -maxdepth 1 -name '*-jmh.jar' -print -quit)"
+          test -n "$SOURCE_JAR"
+          mkdir -p "$PROBE_ROOT/evidence/jars"
+          cp "$SOURCE_JAR" "$PROBE_ROOT/evidence/jars/infra-jmh.jar"
+          chmod 0444 "$PROBE_ROOT/evidence/jars/infra-jmh.jar"
+          JAR="$PROBE_ROOT/evidence/jars/infra-jmh.jar"
+          sudo -H -u mahjong-probe -- test -r "$JAR"
+          sudo -H -u mahjong-probe -- test ! -w "$JAR"
+          python3 perf/ab/run_matrix.py \
+            --phase ab \
+            --profile infra \
+            --base-jar "$JAR" \
+            --candidate-jar "$JAR" \
+            --base-sha "$GITHUB_SHA" \
+            --candidate-sha "$GITHUB_SHA" \
+            --config perf/ab/gate-config.json \
+            --output-dir "$PROBE_ROOT/evidence/lock-regression" \
+            --runtime-temp-root "$PROBE_ROOT/runtime-tmp" \
+            --java "$JAVA_HOME/bin/java" \
+            --candidate-command-prefix-json '["sudo","-H","-u","mahjong-probe","--"]' \
+            --candidate-file-user mahjong-probe \
+            --forks 1 \
+            --warmup-iterations 1 \
+            --measurement-iterations 1 \
+            --warmup-time 10ms \
+            --measurement-time 10ms
+
+      - name: Stage isolated JMH lock evidence
+        if: always()
+        shell: bash
+        run: |
+          set -euo pipefail
+          PROBE_ROOT="/tmp/mahjong-performance-probe-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
+          if [[ -d "$PROBE_ROOT/evidence/lock-regression" ]]; then
+            mkdir -p build/reports/jmh/lock-regression
+            cp -a "$PROBE_ROOT/evidence/lock-regression/." build/reports/jmh/lock-regression/
+          fi
+
       - name: Run one smoke iteration for every protected benchmark
         shell: bash
         run: |
diff --git a/docs/performance-testing.md b/docs/performance-testing.md
@@ -62,8 +62,11 @@ dedicated no-login UID: only the current candidate JSON path receives candidate
 while stdout is confined to the current runner-opened log and the jars, parent directories,
 previous results and manifests remain runner-owned. After each run, residual processes under
 that UID are killed and audited, ownership is reclaimed, and the result/log are locked
-read-only before hashing. The final statistics are evaluated on a fresh runner from a fresh
-base checkout; candidate code cannot rewrite base/AA evidence or the decision gate.
+read-only before hashing. Every A/A and A/B execution also receives a distinct `java.io.tmpdir`
+under a runner-owned parent. The JMH launcher retains its lock without sharing `/tmp/jmh.lock`
+across the runner and isolated UIDs, and the same directory is passed to its measurement fork.
+The final statistics are evaluated on a fresh runner from a fresh base checkout; candidate code
+cannot rewrite base/AA evidence or the decision gate.
 The infrastructure workflow exercises this boundary on Linux with a probe that attempts to
 rewrite runner-owned evidence and leaves a delayed background writer; both must be contained.
 
diff --git a/perf/ab/gate.py b/perf/ab/gate.py
@@ -325,6 +325,7 @@ def load_pairs(
 
     by_pair: dict[int, dict[str, dict[str, dict[str, Any]]]] = defaultdict(dict)
     ordered_roles: dict[int, dict[int, str]] = defaultdict(dict)
+    runtime_temp_dirs: set[str] = set()
     for execution in manifest.get("executions", []):
         if execution.get("exit_code") != 0:
             raise ValueError(f"run contains failed execution: {manifest_path}")
@@ -342,8 +343,18 @@ def load_pairs(
             raise ValueError(f"execution jar digest mismatch in pair {pair_index}: {manifest_path}")
         if execution.get("fork_jvm_args_verified") is not True or execution.get("validation_error") is not None:
             raise ValueError(f"execution evidence was not validated in pair {pair_index}: {manifest_path}")
+        runtime_temp_dir = execution.get("runtime_temp_dir")
+        if not isinstance(runtime_temp_dir, str) or not runtime_temp_dir:
+            raise ValueError(f"execution omitted its JMH runtime temp directory in pair {pair_index}")
+        if runtime_temp_dir in runtime_temp_dirs:
+            raise ValueError(f"execution reused JMH runtime temp directory in pair {pair_index}")
+        runtime_temp_dirs.add(runtime_temp_dir)
+        runtime_temp_arg = f"-Djava.io.tmpdir={runtime_temp_dir}"
+        command = execution.get("command", [])
+        command_offset = len(isolation_prefix) if role == "candidate" else 0
+        if command[command_offset + 1 : command_offset + 2] != [runtime_temp_arg]:
+            raise ValueError(f"execution launcher omitted its JMH runtime temp directory in pair {pair_index}")
         if role == "candidate":
-            command = execution.get("command", [])
             if execution.get("isolated_candidate") is not True or command[: len(isolation_prefix)] != isolation_prefix:
                 raise ValueError(f"candidate execution escaped its command prefix in pair {pair_index}")
             if execution.get("result_mode_after_lock") != 0o444:
@@ -353,7 +364,11 @@ def load_pairs(
         if execution.get("log_mode_after_lock") != 0o444:
             raise ValueError(f"JMH log was not locked after pair {pair_index}")
         log_path = manifest_path.parent / execution["log_file"]
-        verify_fork_log(log_path, execution.get("log_sha256"), expected_jvm_args)
+        verify_fork_log(
+            log_path,
+            execution.get("log_sha256"),
+            [*expected_jvm_args, runtime_temp_arg],
+        )
         if role in by_pair[pair_index] or position in ordered_roles[pair_index]:
             raise ValueError(f"duplicate role or position in pair {pair_index}: {manifest_path}")
         by_pair[pair_index][role] = read_jmh_results(
diff --git a/perf/ab/run_matrix.py b/perf/ab/run_matrix.py
@@ -100,11 +100,20 @@ def command_for(
     jar: pathlib.Path,
     result_file: pathlib.Path,
     config: dict[str, Any],
+    runtime_temp_dir: pathlib.Path,
     command_prefix: list[str] | None = None,
 ) -> list[str]:
     jmh = config["jmh"]
-    fork_jvm_args = jmh.get("jvm_args", [])
-    command = [*(command_prefix or []), java, "-jar", str(jar), jmh["include"]]
+    runtime_temp_arg = f"-Djava.io.tmpdir={runtime_temp_dir}"
+    fork_jvm_args = [*jmh.get("jvm_args", []), runtime_temp_arg]
+    command = [
+        *(command_prefix or []),
+        java,
+        runtime_temp_arg,
+        "-jar",
+        str(jar),
+        jmh["include"],
+    ]
     command.extend(
         [
             "-bm",
@@ -194,6 +203,36 @@ def prepare_untrusted_result(path: pathlib.Path, file_user: str) -> None:
         )
 
 
+def prepare_runtime_temp(path: pathlib.Path, file_user: str | None) -> None:
+    if path.exists():
+        raise FileExistsError(f"JMH runtime temp directory already exists: {path}")
+    if file_user is None:
+        path.mkdir(mode=0o700)
+        return
+    subprocess.run(
+        [
+            "sudo",
+            "install",
+            "-d",
+            "--owner",
+            file_user,
+            "--group",
+            file_user,
+            "--mode",
+            "0700",
+            "--",
+            str(path),
+        ],
+        check=True,
+    )
+    writable = subprocess.run(
+        ["sudo", "-H", "-u", file_user, "--", "test", "-w", str(path)],
+        check=False,
+    )
+    if writable.returncode != 0:
+        raise RuntimeError(f"candidate user {file_user} cannot write JMH runtime temp {path}")
+
+
 def reclaim_untrusted_result(path: pathlib.Path, file_user: str) -> None:
     # A candidate can daemonize while retaining an open writable descriptor. Kill every
     # process under the dedicated UID before making the result immutable and hashing it.
@@ -298,6 +337,9 @@ def run(args: argparse.Namespace) -> int:
     }
 
     output_dir = args.output_dir.resolve()
+    runtime_temp_root = args.runtime_temp_root.resolve(strict=True)
+    if not runtime_temp_root.is_dir():
+        raise NotADirectoryError(runtime_temp_root)
     raw_dir = output_dir / "raw"
     log_dir = output_dir / "logs"
     raw_dir.mkdir(parents=True, exist_ok=True)
@@ -340,6 +382,7 @@ def run(args: argparse.Namespace) -> int:
             "github_runner_name": os.environ.get("RUNNER_NAME"),
             "github_run_id": os.environ.get("GITHUB_RUN_ID"),
             "github_run_attempt": os.environ.get("GITHUB_RUN_ATTEMPT"),
+            "runtime_temp_root": str(runtime_temp_root),
         },
         "jmh": effective_config["jmh"],
         "ci_overrides": {key: value for key, value in overrides.items() if value is not None},
@@ -371,11 +414,17 @@ def run(args: argparse.Namespace) -> int:
             log_file.chmod(0o600)
         if isolated_candidate:
             prepare_untrusted_result(result_file, candidate_file_user)
+        runtime_temp_dir = runtime_temp_root / stem
+        prepare_runtime_temp(
+            runtime_temp_dir,
+            candidate_file_user if isolated_candidate else None,
+        )
         command = command_for(
             args.java,
             jar,
             result_file,
             effective_config,
+            runtime_temp_dir,
             candidate_prefix if isolated_candidate else None,
         )
         started_at = utc_now()
@@ -402,7 +451,13 @@ def run(args: argparse.Namespace) -> int:
         validation_error = None
         try:
             if run_succeeded:
-                verify_fork_jvm_args(log_file, effective_config["jmh"].get("jvm_args", []))
+                verify_fork_jvm_args(
+                    log_file,
+                    [
+                        *effective_config["jmh"].get("jvm_args", []),
+                        f"-Djava.io.tmpdir={runtime_temp_dir}",
+                    ],
+                )
                 fork_jvm_args_verified = True
             verify_jars_unchanged(expected_jar_hashes)
         except ValueError as error:
@@ -427,6 +482,7 @@ def run(args: argparse.Namespace) -> int:
             "result_sha256": sha256_file(result_file) if result_file.is_file() else None,
             "log_file": log_file.relative_to(output_dir).as_posix(),
             "log_sha256": sha256_file(log_file),
+            "runtime_temp_dir": str(runtime_temp_dir),
             "command": command,
         }
         manifest["executions"].append(record)
@@ -451,6 +507,7 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument("--candidate-sha", required=True)
     parser.add_argument("--config", type=pathlib.Path, required=True)
     parser.add_argument("--output-dir", type=pathlib.Path, required=True)
+    parser.add_argument("--runtime-temp-root", type=pathlib.Path, required=True)
     parser.add_argument("--java", default="java")
     parser.add_argument("--forks", type=int)
     parser.add_argument("--warmup-iterations", type=int)
diff --git a/perf/ab/tests/test_gate.py b/perf/ab/tests/test_gate.py
