perf(ci): promote fork-heavy plugins to own shards + weight-balanced rest

8-way run left only ubuntu rest-5 (redis+watcher) and rest-6 (security) over
60s — both bottlenecked by single fork-heavy plugin dirs riding in rest.

- Promote security (→2 slices), redis, multipart out of rest into dedicated
  heavy shards (with --maxWorkers cap), so no rest shard carries a fork-plugin
  giant.
- Replace rest index-interleave with deterministic greedy bin-packing by
  per-dir cpu weight, spreading the remaining heavy dirs (watcher, onerror,
  logrotator, tegg/plugin/tegg, core, logger) one-per-shard. rest now 6 bins of
  ~17.7 weight each.

22 test shard groups total. Windows failures remain absorbed by --retry.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: killagu

.github/workflows/ci.yml

@@ -92,14 +92,16 @@ jobs:
           - schedule-3
           - schedule-4
           - development
+          - security-1
+          - security-2
+          - redis
+          - multipart
           - rest-1
           - rest-2
           - rest-3
           - rest-4
           - rest-5
           - rest-6
-          - rest-7
-          - rest-8
 
     name: Test (${{ matrix.os }}, ${{ matrix.node }}, ${{ matrix.shard }})
     runs-on: ${{ matrix.os }}

scripts/run-shard.js

@@ -55,15 +55,40 @@ const HEAVY_SHARDS = {
   'schedule-4': { dir: 'plugins/schedule', shard: '4/4' },
   // development ~49s -> single shard is fine
   development: { dir: 'plugins/development' },
+  // Fork-based plugins that dominated rest-* and pushed it >60s on ubuntu.
+  // Pull them out of rest into their own (sliceable) shards.
+  'security-1': { dir: 'plugins/security', shard: '1/2' },
+  'security-2': { dir: 'plugins/security', shard: '2/2' },
+  redis: { dir: 'plugins/redis' },
+  multipart: { dir: 'plugins/multipart' },
 };
 
 const HEAVY_DIRS = new Set(Object.values(HEAVY_SHARDS).map((v) => v.dir));
 
-// `rest` is everything not in a heavy dir, interleaved into N light shards.
-// The rest tests are dominated by module transform/import (full egg + tegg
-// decorator stack per file), which barely shrinks with fewer files, so this
-// needs more slices than raw cpu-sum suggests to hit ≤60s on ubuntu.
-const REST_SHARD_COUNT = 8;
+// `rest` is every remaining project dir, balance-packed into N light shards by
+// approximate per-dir cpu weight (seconds; from a full local run). Greedy
+// largest-first bin-packing keeps the heaviest leftover dirs (onerror,
+// logrotator, watcher, tegg/plugin/tegg) on separate shards so no single shard
+// dominates. Unlisted dirs default to a small weight.
+const REST_SHARD_COUNT = 6;
+const REST_DIR_WEIGHTS = {
+  'plugins/watcher': 9.7,
+  'packages/core': 7.6,
+  'plugins/logrotator': 7.7,
+  'plugins/onerror': 7.3,
+  'packages/logger': 6.9,
+  'tegg/core/dynamic-inject': 6.8,
+  'plugins/session': 5.3,
+  'plugins/view-nunjucks': 3.2,
+  'packages/utils': 2.8,
+  'tools/create-egg': 2.7,
+  'tegg/standalone/standalone': 2.3,
+  'tegg/plugin/controller': 2.1,
+  'packages/koa': 2.0,
+  'plugins/view': 1.8,
+  'tegg/plugin/tegg': 11.2,
+};
+const REST_DEFAULT_WEIGHT = 0.5;
 
 // The vitest projects globs (see root vitest.config.ts):
 //   packages/*, plugins/*, tools/create-egg, tegg/core/*, tegg/plugin/*, tegg/standalone/*
@@ -123,6 +148,28 @@ const baseArgs = [
   '20000',
 ];
 
+// Greedy largest-first bin-packing of the rest dirs into REST_SHARD_COUNT bins
+// by approximate cpu weight, so the heaviest leftover dirs land on separate
+// shards. Deterministic: sort by (weight desc, name) then assign each dir to
+// the currently-lightest bin.
+function packRestDirs() {
+  const dirs = listProjectDirs().filter((d) => !HEAVY_DIRS.has(d));
+  dirs.sort((a, b) => {
+    const wa = REST_DIR_WEIGHTS[a] ?? REST_DEFAULT_WEIGHT;
+    const wb = REST_DIR_WEIGHTS[b] ?? REST_DEFAULT_WEIGHT;
+    return wb - wa || (a < b ? -1 : 1);
+  });
+  const bins = Array.from({ length: REST_SHARD_COUNT }, () => ({ total: 0, dirs: [] }));
+  for (const d of dirs) {
+    const w = REST_DIR_WEIGHTS[d] ?? REST_DEFAULT_WEIGHT;
+    let min = bins[0];
+    for (const b of bins) if (b.total < min.total) min = b;
+    min.total += w;
+    min.dirs.push(d);
+  }
+  return bins;
+}
+
 const restMatch = /^rest-(\d+)$/.exec(shardName);
 
 let targetArgs = [];
@@ -132,16 +179,14 @@ if (shardName === 'all') {
   // Single combined light shard (local fallback).
   targetArgs = listProjectDirs().filter((d) => !HEAVY_DIRS.has(d));
 } else if (restMatch) {
-  // Interleave the remaining dirs into REST_SHARD_COUNT shards so the heavier
-  // tegg dirs spread evenly across them. No --maxWorkers cap: fast in-process
-  // tests benefit from full parallelism.
+  // Weight-balanced bin for this rest shard. No --maxWorkers cap: fast
+  // in-process tests benefit from full parallelism.
   const idx = Number(restMatch[1]) - 1;
   if (idx < 0 || idx >= REST_SHARD_COUNT) {
     console.error(`Unknown rest shard: ${shardName} (valid: ${restShardNames().join(', ')})`);
     process.exit(2);
   }
-  const restDirs = listProjectDirs().filter((d) => !HEAVY_DIRS.has(d));
-  targetArgs = restDirs.filter((_, i) => i % REST_SHARD_COUNT === idx);
+  targetArgs = packRestDirs()[idx].dirs;
 } else if (HEAVY_SHARDS[shardName]) {
   const entry = HEAVY_SHARDS[shardName];
   targetArgs = [entry.dir, '--maxWorkers', HEAVY_MAX_WORKERS];