[v1.2.0-rc1 Feature] introduce safe warm workers

.github/actions/prepare-nix/action.yaml

@@ -15,6 +15,8 @@ runs:
         DeterminateSystems/nix-installer-action@ef8a148080ab6020fd15196c2084a2eea5ff2d25
       with:
         source-tag: v3.13.0
+        extra-conf: |
+          fallback = true
 
     - name: Setup attic cache
       uses: >- # v4

.github/styles/config/vocabularies/TraceMachina/accept.txt

@@ -9,6 +9,7 @@ Bazelisk
 Cloudflare
 Colab
 composable
+containerd
 CPUs
 [Dd]eduplication
 eviction_policy
@@ -122,6 +123,18 @@ Brex
 Citrix
 Menlo
 benchmarked
+[Rr]epos
+[Dd]ockerfile
+Dev
+max_workers
+min_warm_workers
+crictl
+runtimes
+enum
+crypto
+devs
+sudo
+fs
 Thanos
 Quickwit
 [Mm]iddleware

.github/workflows/native-cargo.yaml

@@ -47,6 +47,15 @@ jobs:
         run: rustup update && rustup default ${{ matrix.toolchain }}
         shell: bash
 
+      - name: Install protoc
+        run: |
+          if [ "$RUNNER_OS" == "Linux" ]; then
+            sudo apt-get update && sudo apt-get install -y protobuf-compiler
+          elif [ "$RUNNER_OS" == "Windows" ]; then
+            choco install protoc
+          fi
+        shell: bash
+
       - name: Rust cache
         # https://github.com/Swatinem/rust-cache/releases/tag/v2.9.1
         uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4

.github/workflows/sanitizers.yaml

@@ -37,6 +37,10 @@ jobs:
       - name: Free disk space
         uses: ./.github/actions/free-disk
 
+      - name: Install protoc
+        run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
+        shell: bash
+
       - name: Setup Bazel
         uses: >- # v0.19.0
           bazel-contrib/setup-bazel@c5acdfb288317d0b5c0bbd7a396a3dc868bb0f86

.hadolint.yaml

@@ -0,0 +1,5 @@
+---
+ignored:
+  - DL3008  # Pin versions in apt get install
+  - DL3016  # Pin versions in npm install
+  - DL3018  # Pin versions in apk add

Cargo.toml

@@ -4,6 +4,7 @@ exclude = [
   "nativelink-config/generate-stores-config",
   "tools/generate-bazel-rc",
 ]
+members = ["nativelink-crio-worker-pool"]
 resolver = "2"
 
 [package]
@@ -28,6 +29,7 @@ name = "nativelink"
 
 [features]
 nix = ["nativelink-worker/nix"]
+warm-worker-pools = ["nativelink-scheduler/warm-worker-pools"]
 
 [dependencies]
 nativelink-config = { path = "nativelink-config" }

MODULE.bazel

@@ -58,6 +58,7 @@ crate.from_cargo(
     cargo_lockfile = "//:Cargo.lock",
     manifests = [
         "//:Cargo.toml",
+        "//nativelink-crio-worker-pool:Cargo.toml",
         "//nativelink-config:Cargo.toml",
         "//nativelink-error:Cargo.toml",
         "//nativelink-macro:Cargo.toml",

deployment-examples/warm-worker-pools.json5

@@ -0,0 +1,190 @@
+// Example NativeLink configuration with warm worker pools
+// This demonstrates how to configure CRI-O based warm worker pools
+// for faster build times with Java and TypeScript projects.
+//
+// ISOLATION FEATURE:
+// Warm worker pools support Copy-on-Write (COW) isolation to prevent
+// state leakage between jobs. When enabled via isolation.strategy="overlayfs",
+// each job gets an isolated filesystem using OverlayFS:
+//   - Template (lower layer): Read-only base from warmed-up worker
+//   - Job workspace (upper layer): Ephemeral read-write layer per job
+//   - Automatic cleanup: Job workspace deleted after completion
+//
+// Benefits:
+//   - Security: No cross-tenant contamination
+//   - Performance: Fast cloning vs cold starts (30-45s warmup → <500ms clone)
+//   - Resource efficiency: One template serves many isolated jobs
+//
+// Recommended for production deployments with multi-tenant workloads.
+{
+  schedulers: {
+    // Main scheduler with warm worker pools enabled
+    main: {
+      simple: {
+        // Standard scheduler configuration
+        supported_platform_properties: {
+          cpu_count: "minimum",
+          cpu_arch: "exact",
+          lang: "exact",
+        },
+
+        // Warm worker pool configuration
+        // This requires the "warm-worker-pools" feature to be enabled
+        warm_worker_pools: {
+          pools: [
+            // Java/JVM worker pool
+            {
+              name: "java-pool",
+              language: "jvm",
+
+              // CRI-O socket path (Unix domain socket)
+              cri_socket: "unix:///var/run/crio/crio.sock",
+
+              // Container image with pre-installed JVM and build tools
+              container_image: "ghcr.io/tracemachina/nativelink-worker-java:latest",
+
+              // Pool sizing
+              min_warm_workers: 5, // Keep at least 5 workers warmed up
+              max_workers: 50, // Maximum 50 workers total
+
+              // Warmup configuration
+              warmup: {
+                // Commands to run when container starts (warms up JVM)
+                commands: [
+                  {
+                    argv: [
+                      "/opt/warmup/jvm-warmup.sh",
+                    ],
+                    timeout_s: 60,
+                  },
+                ],
+
+                // Commands to run after each job completes
+                post_job_cleanup: [
+                  {
+                    // Force garbage collection between jobs
+                    argv: [
+                      "jcmd",
+                      "1",
+                      "GC.run",
+                    ],
+                    timeout_s: 30,
+                  },
+                ],
+              },
+
+              // Worker lifecycle management
+              lifecycle: {
+                // Recycle workers after 1 hour
+                worker_ttl_seconds: 3600,
+
+                // Recycle workers after 200 jobs
+                max_jobs_per_worker: 200,
+
+                // Run GC every 20 jobs
+                gc_job_frequency: 20,
+              },
+
+              // Isolation configuration (RECOMMENDED for production)
+              // Provides Copy-on-Write filesystem isolation between jobs
+              isolation: {
+                // Strategy: "overlayfs" for COW isolation, "none" for shared state (backward compatible)
+                strategy: "overlayfs",
+
+                // Path where warm templates are stored (read-only base layer)
+                template_cache_path: "/var/lib/nativelink/warm-templates",
+
+                // Path where job-specific workspaces are created (ephemeral write layers)
+                job_workspace_path: "/var/lib/nativelink/warm-jobs",
+              },
+            },
+
+            // TypeScript/Node.js worker pool
+            {
+              name: "typescript-pool",
+              language: "node",
+              cri_socket: "unix:///var/run/crio/crio.sock",
+              container_image: "ghcr.io/tracemachina/nativelink-worker-node:latest",
+              min_warm_workers: 3,
+              max_workers: 30,
+              warmup: {
+                commands: [
+                  {
+                    argv: [
+                      "/opt/warmup/v8-warmup.sh",
+                    ],
+                    timeout_s: 45,
+                  },
+                ],
+                post_job_cleanup: [
+                  {
+                    // Clear V8 heap between jobs
+                    argv: [
+                      "node",
+                      "--expose-gc",
+                      "-e",
+                      "global.gc()",
+                    ],
+                    timeout_s: 20,
+                  },
+                ],
+              },
+              lifecycle: {
+                worker_ttl_seconds: 1800, // 30 minutes
+                max_jobs_per_worker: 100,
+                gc_job_frequency: 10,
+              },
+
+              // Isolation configuration
+              isolation: {
+                strategy: "overlayfs",
+                template_cache_path: "/var/lib/nativelink/warm-templates",
+                job_workspace_path: "/var/lib/nativelink/warm-jobs",
+              },
+            },
+          ],
+        },
+      },
+    },
+  },
+
+  // Store configuration (same as standard NativeLink)
+  stores: {
+    // ... your store configuration here
+  },
+
+  // Server configuration
+  servers: [
+    {
+      listener: {
+        http: {
+          socket_address: "0.0.0.0:50051",
+        },
+      },
+      services: {
+        ac: {
+          main: {
+            ac_store: "ac_store",
+          },
+        },
+        cas: {
+          main: {
+            cas_store: "cas_store",
+          },
+        },
+        execution: {
+          main: {
+            scheduler: "main",
+          },
+        },
+        capabilities: {
+          main: {
+            remote_execution: {
+              scheduler: "main",
+            },
+          },
+        },
+      },
+    },
+  ],
+}