test(contracts): prove the shard split drops no test (partition assert + CI count check)

Add two trust guards that the sharding/balancing took every test into account.
assignBalancedShards now asserts a disjoint, complete partition at runtime, backed by a unit
test (synthetic inputs via an injected weight, plus the real planShard partition over the
integration tree). CI gains a verify-test-count job that runs the integration suite once
(test:integration) and a merge-time reconciliation that the sum of the coverage shards'
passing counts equals it (2672), failing on mismatch. The oracle job runs in parallel with
the coverage matrix, so it adds no critical-path time.

Signed-off-by: Miguel_LZPF <miguel.carpena@io.builders>

Author: MiguelLZPF

.github/workflows/100-flow-ats-test.yaml

@@ -130,23 +130,92 @@ jobs:
         run: npm ci --ignore-scripts
 
       - name: Run contracts coverage shard
-        run: npm run test:coverage:shard --workspace=packages/ats/contracts
+        run: |
+          npm run test:coverage:shard --workspace=packages/ats/contracts 2>&1 | tee shard.log
+          # Record this shard's passing-test count (ANSI-stripped) for the merge-time reconciliation
+          # that proves the sharded runs executed every test — see the merge-coverage job.
+          sed -E 's/\x1b\[[0-9;]*m//g' shard.log | grep -oE '[0-9]+ passing' | head -1 | grep -oE '^[0-9]+' \
+            > packages/ats/contracts/coverage/shard-count.txt
+          echo "shard ${SHARD_INDEX} passing: $(cat packages/ats/contracts/coverage/shard-count.txt)"
 
       - name: Upload coverage shard artifact
         if: ${{ !cancelled() }}
         uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
         with:
           name: coverage-shard-${{ matrix.shard }}
-          path: packages/ats/contracts/coverage/lcov.info
+          path: |
+            packages/ats/contracts/coverage/lcov.info
+            packages/ats/contracts/coverage/shard-count.txt
+          if-no-files-found: error
+          retention-days: 1
+
+  # Sequential oracle for the trust check: run the integration suite in ONE pass and record its
+  # passing-test count. The merge job compares this against the sum of the coverage shards' counts to
+  # prove the sharding/balancing dropped no test. Integration-only (NOT the scripts suite, which the
+  # coverage shards don't run), and excludes the local-parallel `ats.shard.*` files. Runs in parallel
+  # with the coverage matrix, so it adds no critical-path time.
+  verify-test-count:
+    name: verify test count
+    runs-on: token-studio-linux-large
+    timeout-minutes: 45
+    env:
+      CONTRACT_SIZER_RUN_ON_COMPILE: "false"
+      REPORT_GAS: "false"
+
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@58077d3c7e43986b6b15fba718e8ea69e387dfcc # v2.15.1
+        with:
+          egress-policy: audit
+
+      - name: Checkout repository
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+
+      - name: Setup NodeJS Environment
+        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
+        with:
+          node-version-file: .nvmrc
+          cache: "npm"
+
+      - name: Restore dependencies from cache
+        id: deps-cache
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        with:
+          path: |
+            node_modules
+            */*/node_modules
+            */*/*/node_modules
+            ~/.npm
+          key: ${{ runner.os }}-deps-${{ hashFiles('package-lock.json') }}
+
+      - name: Install dependencies
+        if: steps.deps-cache.outputs.cache-hit != 'true'
+        run: npm ci --ignore-scripts
+
+      - name: Run the integration suite sequentially (oracle)
+        working-directory: packages/ats/contracts
+        run: |
+          npm run test:integration 2>&1 | tee oracle.log
+          sed -E 's/\x1b\[[0-9;]*m//g' oracle.log | grep -oE '[0-9]+ passing' | head -1 | grep -oE '^[0-9]+' \
+            > oracle-count.txt
+          echo "sequential integration passing: $(cat oracle-count.txt)"
+
+      - name: Upload oracle count
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
+        with:
+          name: oracle-count
+          path: packages/ats/contracts/oracle-count.txt
           if-no-files-found: error
           retention-days: 1
 
   # Merge the per-shard lcov reports into a single report and upload it once to Codecov — matching
   # the local `npm run test:coverage` single report, instead of uploading partial per-shard pieces.
-  # Runs only when every shard succeeded, so the merged report is complete.
+  # Also reconciles the sharded test count against the sequential oracle. Runs only when every shard
+  # and the oracle succeeded, so the merged report is complete.
   merge-coverage:
     name: merge coverage + upload
-    needs: coverage-ats
+    needs: [coverage-ats, verify-test-count]
     runs-on: token-studio-linux-large
     timeout-minutes: 15
     env:
@@ -188,6 +257,26 @@ jobs:
           pattern: coverage-shard-*
           path: coverage-shards
 
+      - name: Download sequential oracle count
+        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
+        with:
+          name: oracle-count
+          path: oracle
+
+      # Trust check: the sum of the coverage shards' passing-test counts must equal the sequential
+      # single-pass count, proving the sharding/balancing ran every test (none dropped or doubled).
+      - name: Reconcile sharded vs sequential test count
+        run: |
+          oracle=$(cat oracle/oracle-count.txt)
+          sharded=$(find coverage-shards -name shard-count.txt -exec cat {} + | awk '{s += $1} END {print s + 0}')
+          echo "sequential integration : ${oracle} passing"
+          echo "Σ coverage shards       : ${sharded} passing"
+          if [[ "${oracle}" != "${sharded}" ]]; then
+            echo "::error::Test-count mismatch — sequential ${oracle} vs sharded ${sharded}. A suite may be dropped or double-run by the sharding."
+            exit 1
+          fi
+          echo "✅ all tests accounted for (${oracle})"
+
       # Sum line/function/branch hits across the disjoint shards into one lcov, written to the same
       # path a local single-run coverage produces, so the Codecov upload is identical to local. Uses
       # the in-repo merger (preserves function coverage, which lcov-result-merger drops; no

packages/ats/contracts/package.json

@@ -130,6 +130,7 @@
     "test": "bash -c 'npx hardhat test $(find test/contracts/integration test/scripts -name \"*.test.ts\" -not -name \"ats.shard.*.test.ts\")'",
     "test:parallel": "bash -c 'ATS_TEST_MODE=true NODE_OPTIONS=\"--import tsx\" npx hardhat test --parallel $(find test/contracts/integration test/scripts -name \"*.test.ts\" -not -name \"ats.test.ts\")'",
     "test:parallel:ats": "bash -c 'ATS_TEST_MODE=true NODE_OPTIONS=\"--import tsx\" npx hardhat test --parallel $(find test/contracts/integration -name \"ats.shard.*.test.ts\")'",
+    "test:integration": "bash -c 'npx hardhat test $(find test/contracts/integration -name \"*.test.ts\" -not -name \"ats.shard.*.test.ts\")'",
     "test:contracts": "npx hardhat test",
     "test:contracts:parallel": "NODE_OPTIONS='--import tsx' npx hardhat test --parallel",
     "test:scripts": "bash -c 'npx hardhat test --no-compile $(find test/scripts -name \"*.test.ts\")'",

packages/ats/contracts/test/contracts/integration/suiteDiscovery.ts

@@ -52,18 +52,36 @@ export function suiteWeight(path: string): number {
  * round-robin's "whichever shard drew the heavy suites". Deterministic — files are sorted by weight
  * (desc) then path, and ties pick the lowest-index shard — so a given `shardCount` yields the same
  * partition wherever it is computed (the test entry and the coverage planner must agree).
+ *
+ * `weightOf` defaults to `suiteWeight` (reads the file); it is a parameter only so the partition
+ * invariant can be unit-tested with synthetic inputs that never touch the filesystem.
  */
-export function assignBalancedShards(files: string[], shardCount: number): string[][] {
+export function assignBalancedShards(
+  files: string[],
+  shardCount: number,
+  weightOf: (path: string) => number = suiteWeight,
+): string[][] {
   const bins: string[][] = Array.from({ length: shardCount }, () => []);
   const loads = new Array<number>(shardCount).fill(0);
   const ordered = files
-    .map((path) => ({ path, weight: suiteWeight(path) }))
+    .map((path) => ({ path, weight: weightOf(path) }))
     .sort((a, b) => b.weight - a.weight || (a.path < b.path ? -1 : 1));
   for (const { path, weight } of ordered) {
     let lightest = 0;
     for (let i = 1; i < shardCount; i++) if (loads[i] < loads[lightest]) lightest = i;
     bins[lightest].push(path);
     loads[lightest] += weight;
   }
+
+  // Trust invariant — the shards MUST be a disjoint, complete partition of the input: every suite
+  // runs in exactly one shard, none dropped or doubled. This is the guarantee the parallel/coverage
+  // split rests on, so assert it here rather than hope a future edit preserves it (cheap: O(files)).
+  const assigned = bins.flat();
+  if (assigned.length !== files.length || new Set(assigned).size !== new Set(files).size) {
+    throw new Error(
+      `assignBalancedShards: not a disjoint+complete partition — assigned ${assigned.length} ` +
+        `(${new Set(assigned).size} distinct) of ${files.length} input files`,
+    );
+  }
   return bins;
 }

packages/ats/contracts/test/scripts/unit/tools/shardPartition.test.ts

@@ -0,0 +1,102 @@
+// SPDX-License-Identifier: Apache-2.0
+
+/**
+ * Unit tests for the shard partitioner — the "no suite is silently dropped" guarantee.
+ *
+ * The whole parallel/coverage scheme trusts that splitting the suites into shards is a DISJOINT,
+ * COMPLETE partition: every suite runs in exactly one shard, none dropped or doubled. These tests
+ * assert that property of `assignBalancedShards` directly (with synthetic inputs + an injected
+ * weight, so no filesystem), and check the real coverage `planShard` partition end to end. They go
+ * red if a future edit to the balancing or planning logic ever drops, duplicates, or misroutes a
+ * suite — which a coverage % alone would not reveal.
+ *
+ * @module test/scripts/unit/tools/shardPartition.test
+ */
+
+import { expect } from "chai";
+import {
+  assignBalancedShards,
+  isMegaSuiteFile,
+  isShardEntryFile,
+  walkTestFiles,
+} from "@test/contracts/integration/suiteDiscovery";
+import { join } from "path";
+// planShard reads the integration tree and imports test/ helpers, so it can't live in the @scripts
+// barrel (that would pull test/ into the scripts build) — import it directly.
+import { planShard } from "../../../../scripts/tools/coverage-shard/planShard";
+
+/** Assert `shards` is a disjoint, complete partition of `input` (order-independent). */
+function expectPartition(shards: string[][], input: string[]): void {
+  const flat = shards.flat();
+  expect(flat.length, "no suite dropped or doubled").to.equal(input.length);
+  expect(new Set(flat).size, "no suite appears twice").to.equal(flat.length);
+  expect([...flat].sort()).to.deep.equal([...input].sort());
+}
+
+describe("assignBalancedShards — partition invariant", () => {
+  // A synthetic weight so the test never touches the filesystem (real suiteWeight reads each file).
+  const weight = (p: string) => p.length;
+  const files = ["aaaa", "bbb", "cc", "d", "eeeee", "ff", "ggggggg", "h", "iii", "jjjj"];
+
+  for (const n of [1, 2, 3, 4, 8]) {
+    it(`is disjoint + complete for ${n} shard(s)`, () => {
+      const shards = assignBalancedShards(files, n, weight);
+      expect(shards.length).to.equal(n);
+      expectPartition(shards, files);
+    });
+  }
+
+  it("handles more shards than files (some shards empty, still complete)", () => {
+    const few = ["x", "yy", "zzz"];
+    const shards = assignBalancedShards(few, 8, weight);
+    expect(shards.length).to.equal(8);
+    expect(shards.filter((s) => s.length === 0).length).to.equal(5);
+    expectPartition(shards, few);
+  });
+
+  it("is deterministic — same input yields the same partition", () => {
+    expect(assignBalancedShards(files, 4, weight)).to.deep.equal(assignBalancedShards(files, 4, weight));
+  });
+
+  it("balances weight: the heaviest shard exceeds the lightest by at most one suite's weight", () => {
+    const shards = assignBalancedShards(files, 4, weight);
+    const loads = shards.map((s) => s.reduce((sum, p) => sum + weight(p), 0));
+    const maxSingle = Math.max(...files.map(weight));
+    // Greedy LPT guarantees the spread is bounded by the largest single item.
+    expect(Math.max(...loads) - Math.min(...loads)).to.be.at.most(maxSingle);
+  });
+});
+
+describe("planShard — the real coverage shards cover every integration suite", () => {
+  const N = 4;
+  const shards = Array.from({ length: N }, (_, i) => planShard(i, N).map((p) => p.split("/integration/")[1] ?? p));
+  const integrationDir = join(process.cwd(), "test", "contracts", "integration");
+
+  it("includes the mega entry (ats.test.ts) in EVERY shard", () => {
+    for (let i = 0; i < N; i++) expect(shards[i], `shard ${i}`).to.include("ats.test.ts");
+  });
+
+  it("never lists a local parallel shard file or a discovered mega suite", () => {
+    for (const shard of shards) {
+      for (const f of shard) {
+        expect(f.startsWith("ats.shard."), `${f} is a local-parallel file`).to.be.false;
+      }
+    }
+  });
+
+  it("partitions the standalone suites disjointly and completely", () => {
+    const expectedStandalone = walkTestFiles(integrationDir)
+      .filter((p) => {
+        const base = p.split("/").pop() ?? "";
+        return !isShardEntryFile(base) && !isMegaSuiteFile(p);
+      })
+      .map((p) => p.split("/integration/")[1])
+      .sort();
+
+    const standaloneAcrossShards = shards.flat().filter((f) => f !== "ats.test.ts");
+    expect(new Set(standaloneAcrossShards).size, "no standalone suite duplicated").to.equal(
+      standaloneAcrossShards.length,
+    );
+    expect([...standaloneAcrossShards].sort()).to.deep.equal(expectedStandalone);
+  });
+});