diff --git a/.changeset/ci-shard-test-core.md b/.changeset/ci-shard-test-core.md new file mode 100644 index 0000000000..d8288b03ce --- /dev/null +++ b/.changeset/ci-shard-test-core.md @@ -0,0 +1,8 @@ +--- +--- + +CI-only: shard the Test Core job by package (deterministic, test-file-count-balanced +halves via `scripts/partition-test-shards.mjs`), move the dogfood verify-CLI pass into +its own parallel job aggregated by the existing Dogfood Regression Gate, and repoint +the temporal-conformance Turbo cache fallback at the Build Core namespace. Releases +nothing. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 59843115de..c099036112 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -75,18 +75,37 @@ jobs: - '.github/workflows/ci.yml' test: - name: Test Core + # Sharded 2-way BY PACKAGE: a core-touching PR ran the affected suite + # ~11½ min on one 4-vCPU runner — the longest pole in the whole workflow. + # scripts/partition-test-shards.mjs splits the package list into two + # deterministic, test-file-count-balanced halves (573/572 at the time of + # writing) and each shard runs its half through turbo. NOT the dogfood + # job's vitest --shard passthrough, deliberately: that works for dogfood + # because it is ONE package with ~60 files, but applied workspace-wide, + # vitest 4 hard-fails every package with fewer test files than the shard + # count — and `--passWithNoTests` converts the failure into running NOTHING + # on either shard (three packages have exactly one test file today). See + # the script header for the verification. + # + # Branch protection requires the bare "Test Core" context, which a matrix + # can never publish again — the test-gate job below carries that name + # (the #3622 lesson; see dogfood-gate). + name: Test Core (${{ matrix.shard }}/2) needs: filter if: needs.filter.outputs.core == 'true' runs-on: ubuntu-latest # Backstop only — the stall guard on the test steps is the primary # detector for a #4250-style hang and fires well before this. 30 min is - # 2.5-3× a normal run (main ~9.5 min, PR ~12 min), with margin for a cold - # Turbo cache; the old 45 left a hung job "running" for half an hour past - # any plausible healthy finish. + # ~4× a normal sharded run (~6-7 min), with margin for a cold Turbo cache; + # the old 45 left a hung job "running" for half an hour past any plausible + # healthy finish. timeout-minutes: 30 permissions: contents: read + strategy: + fail-fast: false + matrix: + shard: [1, 2] steps: - name: Checkout repository @@ -123,14 +142,19 @@ jobs: # the repo's 10 GB Actions cache pool and evicted the main-branch seeds — # observed as sudden cold-cache spikes (Build Core 51s → 4m30s). PRs fall # back to main's entries via the prefix restore-keys; only main pushes - # save (the "Save Turbo cache" step at the end of the job). + # save (the "Save Turbo cache" step at the end of the job). Shard-scoped + # key: each shard builds/tests a different half of the workspace (same + # pattern as dogfood). Jobs that used to fall back to this job's + # namespace fall back to Build Core now — neither single shard builds a + # superset anymore. - name: Restore Turbo cache uses: actions/cache/restore@v6 with: path: .turbo/cache - key: ${{ runner.os }}-turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }} + key: ${{ runner.os }}-turbo-${{ github.job }}-${{ matrix.shard }}-${{ github.ref_name }}-${{ github.sha }} restore-keys: | - ${{ runner.os }}-turbo-${{ github.job }}-${{ github.ref_name }}- + ${{ runner.os }}-turbo-${{ github.job }}-${{ matrix.shard }}-${{ github.ref_name }}- + ${{ runner.os }}-turbo-${{ github.job }}-${{ matrix.shard }}- ${{ runner.os }}-turbo-${{ github.job }}- - name: Install dependencies @@ -139,16 +163,41 @@ jobs: # PRs: only test packages affected by the diff against the PR base. # spec sits at the root of the dependency graph, so spec-touching PRs # still run (close to) everything — but the many PRs that don't touch - # spec skip the bulk of the 75-package matrix. - # --concurrency=4: turbo's default (10) oversubscribes the 4-vCPU - # hosted runner; matching the core count bounds peak memory and the - # job is CPU-bound anyway. + # spec skip the bulk of the 75-package matrix. Push to main and + # merge-queue builds partition the FULL package list instead: the queue + # result IS the next main, so it gets main's validation, not the PR's + # affected-only subset. (Spec's suite runs here plain / uninstrumented; + # the coverage-instrumented pass lives in the nightly coverage-nightly + # workflow.) + # # !@objectstack/dogfood: the ~7½-minute dogfood suite is the dedicated # Dogfood job's whole purpose, and both jobs run under the same `core` # filter — without the exclusion every core PR executed the suite twice - # in parallel, and it dominated this job's critical path. The exclusion - # subtracts from the affected set (verified: turbo unions inclusive - # filters, then applies `!` negations to the result). + # in parallel, and it dominated this job's critical path. + # + # `turbo ls` is experimental; the partition script asserts its output + # shape loudly so an upgrade that changes it turns into a red step + # naming the cause, not a silently empty shard. An EMPTY shard file must + # short-circuit the test step below: `turbo run test` with zero --filter + # args runs the entire workspace. + - name: Compute this shard's package set + env: + TURBO_SCM_BASE: ${{ github.event.pull_request.base.sha }} + run: | + if [ "${{ github.event_name }}" = "pull_request" ]; then + pnpm exec turbo ls --affected --output=json > "$RUNNER_TEMP/turbo-ls.json" + else + pnpm exec turbo ls --output=json > "$RUNNER_TEMP/turbo-ls.json" + fi + node scripts/partition-test-shards.mjs "$RUNNER_TEMP/turbo-ls.json" \ + --shard ${{ matrix.shard }}/2 --exclude @objectstack/dogfood \ + > "$RUNNER_TEMP/shard-packages.txt" + echo "Packages on this shard:" + cat "$RUNNER_TEMP/shard-packages.txt" + + # --concurrency=4: turbo's default (10) oversubscribes the 4-vCPU + # hosted runner; matching the core count bounds peak memory and the + # job is CPU-bound anyway. # run-with-stall-guard replaces the old `… 2>&1 | tee $log` + # `set -o pipefail` idiom: the guard tees combined output to the log # itself and propagates the suite's real exit status, so there is no @@ -157,7 +206,7 @@ jobs: # job sits in_progress. Silence past --stall-minutes is declared a # stall — a labeled red naming the last output line — instead of a # 20-minute wait for a human (or the job timeout) to notice. 10 min is - # ~5× the longest healthy quiet gap and still under half a normal run. + # ~5× the longest healthy quiet gap. # # NODE_OPTIONS arms every node process (vitest workers included) to dump # a diagnostic report on SIGUSR2; on a stall the guard signals the frozen @@ -165,34 +214,19 @@ jobs: # a process whose event loop is alive, and a named "no report = blocked # loop" verdict for one that is sync-spinning. The next #4250 occurrence # identifies its own culprit instead of costing a diagnosis. - - name: Run affected tests (PR) - if: github.event_name == 'pull_request' - env: - TURBO_SCM_BASE: ${{ github.event.pull_request.base.sha }} - NODE_OPTIONS: --report-on-signal --report-signal=SIGUSR2 --report-directory=${{ runner.temp }}/stall-reports - run: | - mkdir -p "$RUNNER_TEMP/stall-reports" - node scripts/run-with-stall-guard.mjs --log "$RUNNER_TEMP/test-core.log" --stall-minutes 10 \ - --report-dir "$RUNNER_TEMP/stall-reports" -- \ - pnpm turbo run test --affected --filter=!@objectstack/dogfood --concurrency=4 - - # Push to main: full run. Spec's suite runs here plain (uninstrumented); - # the coverage-instrumented pass moved to the nightly Spec Coverage - # workflow (coverage-nightly.yml) — instrumentation added minutes to - # every main push for a trend artifact that is consulted occasionally at - # best. Dogfood is excluded for the same reason as the PR step: the - # Dogfood job runs it. Merge-queue builds take this full-run path too: - # the queue result IS the next main, so it gets main's validation, not - # the PR's affected-only subset. - - name: Run all tests (push) - if: github.event_name == 'push' || github.event_name == 'merge_group' + - name: Run this shard's tests env: NODE_OPTIONS: --report-on-signal --report-signal=SIGUSR2 --report-directory=${{ runner.temp }}/stall-reports run: | + if [ ! -s "$RUNNER_TEMP/shard-packages.txt" ]; then + echo "No packages on this shard — nothing to test." + exit 0 + fi + FILTERS=$(sed 's/^/--filter=/' "$RUNNER_TEMP/shard-packages.txt" | tr '\n' ' ') mkdir -p "$RUNNER_TEMP/stall-reports" node scripts/run-with-stall-guard.mjs --log "$RUNNER_TEMP/test-core.log" --stall-minutes 10 \ --report-dir "$RUNNER_TEMP/stall-reports" -- \ - pnpm turbo run test --filter=!@objectstack/dogfood --concurrency=4 + pnpm turbo run test $FILTERS --concurrency=4 # Runs even when the suite failed — that is when it earns its keep. A red # suite plus a GREEN completeness check means real test failures; a red @@ -210,25 +244,52 @@ jobs: # A stall's full diagnostic reports (JS stacks, libuv handles, heap # summary per process) outlive the in-log digest — keep them so a #4250 # occurrence can be dissected offline. Free when nothing stalled: the - # directory is empty and if-no-files-found skips the upload. + # directory is empty and if-no-files-found skips the upload. Shard-scoped + # name so the two matrix jobs don't collide. - name: Upload stall diagnostic reports if: failure() uses: actions/upload-artifact@v7 with: - name: stall-reports-test-core + name: stall-reports-test-core-${{ matrix.shard }} path: ${{ runner.temp }}/stall-reports/ if-no-files-found: ignore retention-days: 14 # Seed the shared Turbo cache from main only (see the restore step # above). always(): keep the seed fresh even when a test fails, matching - # the old actions/cache post-step behavior. + # the old actions/cache post-step behavior. Shard-scoped key so the two + # matrix jobs don't collide. - name: Save Turbo cache (main only) if: always() && github.event_name == 'push' uses: actions/cache/save@v6 with: path: .turbo/cache - key: ${{ runner.os }}-turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }} + key: ${{ runner.os }}-turbo-${{ github.job }}-${{ matrix.shard }}-${{ github.ref_name }}-${{ github.sha }} + + test-gate: + # Stable required-check name for the sharded Test Core matrix — the exact + # contract dogfood-gate documents below (#3622): branch protection requires + # the bare "Test Core" context, and once the job is a matrix that context + # can never appear again, deadlocking every PR. Keeping the contract HERE + # means a future shard-count change cannot deadlock the repo. See + # dogfood-gate for why `cancelled` passes and why this must not be + # `if: !cancelled()` on the job. + name: Test Core + needs: test + if: always() + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + steps: + - name: Verify test shard results + run: | + result="${{ needs.test.result }}" + echo "test matrix aggregate result: $result" + case "$result" in + success|skipped|cancelled) echo "Test Core gate satisfied ($result)." ;; + *) echo "::error::Test Core shards did not pass (aggregate result: $result)"; exit 1 ;; + esac # ── Temporal conformance against live, non-UTC servers (ADR-0053 D-A3) ───── @@ -332,9 +393,11 @@ jobs: restore-keys: | ${{ runner.os }}-pnpm-store-v3- - # Restore-only (same policy as every other job); falls back to the Test - # Core namespace because that job builds a superset of what this one - # needs and its cache is seeded from main. + # Restore-only (same policy as every other job); falls back to the Build + # Core namespace because that job builds every package (a superset of the + # build closure this one needs) and its cache is seeded from main. It + # used to fall back to Test Core, but that namespace is per-shard now and + # neither single shard builds a superset. - name: Restore Turbo cache uses: actions/cache/restore@v6 with: @@ -342,8 +405,8 @@ jobs: key: ${{ runner.os }}-turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }} restore-keys: | ${{ runner.os }}-turbo-${{ github.job }}- - ${{ runner.os }}-turbo-test-${{ github.ref_name }}- - ${{ runner.os }}-turbo-test- + ${{ runner.os }}-turbo-build-core-${{ github.ref_name }}- + ${{ runner.os }}-turbo-build-core- - name: Install dependencies run: pnpm install --frozen-lockfile @@ -439,8 +502,9 @@ jobs: if: needs.filter.outputs.core == 'true' runs-on: ubuntu-latest # Backstop only — the stall guard on the test step is the primary detector - # for a #4250-style hang (see Test Core). 30 min is ~2.5× the slower shard - # (shard 1 runs the verify-CLI step too, ~12 min all in). + # for a #4250-style hang (see Test Core). 30 min is ~4× a shard (~7 min; + # the verify-CLI pass that used to ride shard 1 is its own parallel job + # now — dogfood-verify below). timeout-minutes: 30 permissions: contents: read @@ -538,21 +602,6 @@ jobs: if-no-files-found: ignore retention-days: 14 - # Replaces the former auto-verify dogfood tests: runs the published - # `objectstack verify` engine over each example app through the CLI — - # auto-derived CRUD round-trip fidelity + the cross-owner RLS invariant. - # Exits non-zero on a real runtime failure, so it gates like the tests did. - # Not shard-dependent, so shard 1 alone runs it. - - name: Verify example apps via the `objectstack verify` CLI - if: matrix.shard == 1 - run: | - pnpm turbo run build --filter=@objectstack/cli - for app in examples/app-crm examples/app-showcase; do - echo "::group::objectstack verify $app --rls" - OS_LOG_LEVEL=error node packages/cli/bin/run.js verify --app "$app/objectstack.config.ts" --rls - echo "::endgroup::" - done - # Seed the shared Turbo cache from main only (see the restore step # above); shard-scoped key so the two matrix jobs don't collide. - name: Save Turbo cache (main only) @@ -562,6 +611,91 @@ jobs: path: .turbo/cache key: ${{ runner.os }}-turbo-${{ github.job }}-${{ matrix.shard }}-${{ github.ref_name }}-${{ github.sha }} + # Replaces the former auto-verify dogfood tests: runs the published + # `objectstack verify` engine over each example app through the CLI — + # auto-derived CRUD round-trip fidelity + the cross-owner RLS invariant. + # Exits non-zero on a real runtime failure, so it gates like the tests did. + # + # Its own job, not a rider on dogfood shard 1: the ~4½-minute pass ran + # SERIALLY after that shard's tests, making shard 1 (~12 min) nearly twice + # shard 2 (~6½ min) — the second-longest pole in the workflow for no + # parallelism reason. It reports through dogfood-gate (below) rather than + # its own required context, so the branch-protection contract is unchanged. + dogfood-verify: + name: Dogfood Verify CLI + needs: filter + if: needs.filter.outputs.core == 'true' + runs-on: ubuntu-latest + timeout-minutes: 20 + permissions: + contents: read + + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Setup Node.js + uses: actions/setup-node@v7 + with: + node-version: '22' + + - name: Enable Corepack + run: corepack enable + + - name: Verify pnpm version + run: pnpm --version + + - name: Get pnpm store directory + shell: bash + run: | + echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV + + - name: Setup pnpm cache + uses: actions/cache@v6 + with: + path: ${{ env.STORE_PATH }} + key: ${{ runner.os }}-pnpm-store-v3-${{ hashFiles('**/pnpm-lock.yaml') }} + restore-keys: | + ${{ runner.os }}-pnpm-store-v3- + + # Restore-only, with no save step at all (the console-pin pattern): the + # job's only build is the CLI closure, and the build-core fallbacks are + # the entries that actually hit — that job builds a superset and is + # seeded from main. + - name: Restore Turbo cache + uses: actions/cache/restore@v6 + with: + path: .turbo/cache + key: ${{ runner.os }}-turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-turbo-${{ github.job }}-${{ github.ref_name }}- + ${{ runner.os }}-turbo-${{ github.job }}- + ${{ runner.os }}-turbo-build-core-${{ github.ref_name }}- + ${{ runner.os }}-turbo-build-core- + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + # Build the CLI plus BOTH example apps' full dependency closures (the + # `pkg...` filter form). When this pass rode dogfood shard 1 the whole + # workspace was already built by the test step, so `--filter=@objectstack/cli` + # alone sufficed; standalone, the bundled objectstack.config imports + # runtime packages (e.g. @objectstack/connector-mcp for app-showcase) + # whose dist nothing here had built — ERR_MODULE_NOT_FOUND at verify + # time. The closure syntax keeps this self-maintaining as app deps move. + - name: Verify example apps via the `objectstack verify` CLI + run: | + pnpm turbo run build \ + --filter=@objectstack/cli... \ + --filter=@objectstack/example-crm... \ + --filter=@objectstack/example-showcase... \ + --concurrency=4 + for app in examples/app-crm examples/app-showcase; do + echo "::group::objectstack verify $app --rls" + OS_LOG_LEVEL=error node packages/cli/bin/run.js verify --app "$app/objectstack.config.ts" --rls + echo "::endgroup::" + done + dogfood-gate: # Stable required-check name for a SHARDED job (#3622 follow-up). # @@ -573,10 +707,14 @@ jobs: # branch protection; keeping the contract HERE instead means a future # shard-count change cannot deadlock the repo a second time. # + # Also aggregates dogfood-verify (the CLI pass that used to ride shard 1), + # so the one required context still covers everything it covered before + # the split. + # # `if: always()` + result inspection so a legitimately skipped matrix (the # `filter` job says no core paths changed) still satisfies the gate. name: Dogfood Regression Gate - needs: dogfood + needs: [dogfood, dogfood-verify] if: always() runs-on: ubuntu-latest timeout-minutes: 10 @@ -600,10 +738,16 @@ jobs: # Deliberately NOT `if: !cancelled()` on the job instead: a skipped # gate publishes no required-check context on the SHA, which is the # #3622 merge-deadlock all over again. - case "$result" in - success|skipped|cancelled) echo "Dogfood gate satisfied ($result)." ;; - *) echo "::error::Dogfood shards did not pass (aggregate result: $result)"; exit 1 ;; - esac + verify_result="${{ needs['dogfood-verify'].result }}" + echo "dogfood-verify result: $verify_result" + fail=0 + for r in "dogfood:$result" "dogfood-verify:$verify_result"; do + case "${r#*:}" in + success|skipped|cancelled) echo "Gate leg ${r%%:*} satisfied (${r#*:})." ;; + *) echo "::error::Gate leg ${r%%:*} did not pass (result: ${r#*:})"; fail=1 ;; + esac + done + exit "$fail" build-core: name: Build Core diff --git a/scripts/partition-test-shards.mjs b/scripts/partition-test-shards.mjs new file mode 100644 index 0000000000..7619ffcc37 --- /dev/null +++ b/scripts/partition-test-shards.mjs @@ -0,0 +1,158 @@ +#!/usr/bin/env node +// Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license. +// +// partition-test-shards -- deterministic, load-balanced split of a `turbo ls` +// package list across the Test Core shard matrix (ci.yml). +// +// Test Core shards BY PACKAGE, not by vitest --shard passthrough, on purpose. +// The dogfood job's file-level sharding works because dogfood is ONE package +// with ~60 test files; applied across the whole workspace it breaks on every +// package with fewer test files than the shard count. Verified on vitest +// 4.1.10 with a 1-file package: `--shard=1/2` AND `--shard=2/2` both fail with +// "--shard must be a smaller than count of test files" -- and adding +// `--passWithNoTests` converts that error into exit 0 with NO files run on +// EITHER shard. Three workspace packages have exactly one test file today, so +// the passthrough route is a silent-coverage-loss machine, not an option. +// +// Each package's weight is its test-file count. That is a deliberate proxy: +// suite wall-clock is dominated by fixed per-file cost (module-graph +// re-execution per file under isolation -- same measurement objectui's CI +// documents), so file count tracks duration far better than package count. +// Packages are placed heaviest-first into the lightest bin (LPT greedy), with +// all ties broken by name, so every shard computes the identical split from +// the same input without coordinating. +// +// Usage: +// node scripts/partition-test-shards.mjs --shard N/M \ +// [--exclude ]... +// node scripts/partition-test-shards.mjs --self-test +// +// is the output of `turbo ls [--affected] --output=json` +// (shape: {packages:{items:[{name,path}]}}; `turbo ls` is marked experimental, +// so the shape is asserted loudly below rather than defaulted around). +// Prints the selected shard's package names, one per line -- possibly zero +// lines, which the caller must treat as "nothing to run", NOT as "no filter": +// a `turbo run test` with no --filter args runs the entire workspace. + +import { readFileSync, readdirSync } from 'node:fs'; +import path from 'node:path'; +import process from 'node:process'; + +const TEST_FILE = /\.test\.[cm]?[jt]sx?$/; +const SKIP_DIRS = new Set(['node_modules', 'dist', 'coverage', '.turbo', '.next']); + +function countTestFiles(dir) { + let entries; + try { + entries = readdirSync(dir, { withFileTypes: true }); + } catch { + return 0; // package path missing locally -- weight 0, still assigned + } + let n = 0; + for (const e of entries) { + if (e.isDirectory()) { + if (!SKIP_DIRS.has(e.name)) n += countTestFiles(path.join(dir, e.name)); + } else if (TEST_FILE.test(e.name)) { + n++; + } + } + return n; +} + +// LPT greedy: heaviest package into the currently lightest bin. Deterministic: +// input order never matters because both the package sort and the bin choice +// break ties explicitly (by name / by lowest bin index). +export function partition(items, shardCount) { + const sorted = [...items].sort( + (a, b) => b.weight - a.weight || a.name.localeCompare(b.name, 'en') + ); + const bins = Array.from({ length: shardCount }, () => ({ total: 0, names: [] })); + for (const it of sorted) { + let best = 0; + for (let i = 1; i < bins.length; i++) { + if (bins[i].total < bins[best].total) best = i; + } + bins[best].names.push(it.name); + bins[best].total += it.weight; + } + return bins; +} + +function selfTest() { + const mk = (name, weight) => ({ name, weight }); + // Coverage + determinism: every package lands in exactly one bin, and two + // runs over differently-ordered input agree. + const items = [mk('e', 1), mk('a', 9), mk('c', 4), mk('b', 9), mk('d', 3)]; + const shuffled = [items[2], items[4], items[0], items[3], items[1]]; + const a = partition(items, 2); + const b = partition(shuffled, 2); + const flatA = a.flatMap((bin) => bin.names).sort(); + if (flatA.join() !== 'a,b,c,d,e') throw new Error(`coverage: got ${flatA.join()}`); + if (JSON.stringify(a) !== JSON.stringify(b)) throw new Error('determinism: input order changed the split'); + // LPT balance bound: bin spread never exceeds the heaviest single weight. + const totals = a.map((bin) => bin.total); + if (Math.max(...totals) - Math.min(...totals) > 9) throw new Error(`balance: totals ${totals}`); + // The two 9s must not share a bin. + const binOfA = a.findIndex((bin) => bin.names.includes('a')); + const binOfB = a.findIndex((bin) => bin.names.includes('b')); + if (binOfA === binOfB) throw new Error('balance: both heaviest packages in one bin'); + // Degenerate inputs: empty list, more shards than packages. + const empty = partition([], 2); + if (empty.some((bin) => bin.names.length > 0)) throw new Error('empty input produced packages'); + const sparse = partition([mk('only', 5)], 3); + if (sparse.flatMap((bin) => bin.names).join() !== 'only') throw new Error('sparse input lost the package'); + console.log('partition-test-shards: self-test OK'); +} + +function main() { + const argv = process.argv.slice(2); + if (argv.includes('--self-test')) { + selfTest(); + return; + } + + let listPath = null; + let shardSpec = null; + const excluded = new Set(); + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + if (arg === '--shard') shardSpec = argv[++i]; + else if (arg === '--exclude') excluded.add(argv[++i]); + else if (!arg.startsWith('--') && listPath === null) listPath = arg; + else throw new Error(`unrecognized argument: ${arg}`); + } + const shardMatch = /^([1-9]\d*)\/([1-9]\d*)$/.exec(shardSpec ?? ''); + if (!listPath || !shardMatch) { + console.error('usage: partition-test-shards.mjs --shard N/M [--exclude ]...'); + process.exit(1); + } + const shardIndex = Number(shardMatch[1]); + const shardCount = Number(shardMatch[2]); + if (shardIndex > shardCount) throw new Error(`--shard ${shardSpec}: index exceeds count`); + + const parsed = JSON.parse(readFileSync(listPath, 'utf8')); + const items = parsed?.packages?.items; + if (!Array.isArray(items)) { + throw new Error( + `${listPath}: expected \`turbo ls --output=json\` shape {packages:{items:[...]}} -- ` + + 'did an experimental-command upgrade change the output?' + ); + } + const weighted = []; + for (const it of items) { + if (typeof it?.name !== 'string' || typeof it?.path !== 'string') { + throw new Error(`${listPath}: package entry missing name/path: ${JSON.stringify(it)}`); + } + if (excluded.has(it.name)) continue; + weighted.push({ name: it.name, weight: countTestFiles(it.path) }); + } + const bins = partition(weighted, shardCount); + const mine = bins[shardIndex - 1]; + console.error( + `shard ${shardSpec}: ${mine.names.length}/${weighted.length} packages, ` + + `weight ${mine.total} (all bins: ${bins.map((b) => b.total).join('/')})` + ); + for (const name of mine.names) console.log(name); +} + +main();