check-type-check-coverage.mjs

#!/usr/bin/env node
/**
 * Validates that every workspace package's types are actually checked by CI.
 *
 * `pnpm type-check` runs `turbo run type-check`, and turbo silently skips any
 * package that has no `type-check` script — so a package without one is not
 * "passing", it is unchecked. That is how #2911 happened: `plugin-map` sat
 * broken on `main` for a day while build and tests were green.
 *
 * This guard makes that invisibility impossible. A package with no
 * `type-check` script must be declared below, with a reason, and the lists can
 * only shrink: once a package gains the script, its entry has to be deleted or
 * this guard fails.
 *
 * The same question applies one level down, to TESTS (objectstack#4118): a
 * package tsconfig excludes test files — correctly, it is the BUILD config and
 * they would emit into `dist` — and for most packages no other `tsc` invocation
 * read them either. Tests are where agents encode their understanding of a
 * contract, so unchecked tests let a wrong understanding accumulate silently and
 * then READ AS EVIDENCE: objectui#3009 found `spec-derived-unions.test.ts` had
 * built its whole contract on `satisfies` checks that never ran, under a header
 * calling them "the real enforcement". Reverting a derived alias produced zero
 * errors. So the second half of this guard: a package with test files either
 * type-checks them or carries a TEST_DEBT entry with a measured error count.
 *
 * Run:  node scripts/check-type-check-coverage.mjs
 * Exit: 0 = OK, 1 = coverage regressed or the lists are stale
 */

import { readFileSync, readdirSync, statSync } from "fs";
import { resolve, dirname, join } from "path";
import { fileURLToPath } from "url";

const root = resolve(dirname(fileURLToPath(import.meta.url)), "..");

// ── Known gaps ───────────────────────────────────────────────────────────────
// Packages whose types do NOT currently compile, so they cannot carry a
// `type-check` script yet. Every entry is real debt: fix the errors, add
// `"type-check": "tsc --noEmit"`, then delete the entry. Counts are from the
// #2911 sweep (bare `tsc --noEmit` with the `paths` override its type-checked
// peers already carry, so the TS6059 rootDir noise is excluded).
// Empty, and worth keeping that way: every workspace package now either carries
// a `type-check` script or is declared in one of the exemption lists below.
const DEBT = {};

// Packages that are not compiled at all: documentation snippets with no build
// script and no tsconfig, whose sources are read rather than run. Re-validated
// on every run — the moment one gains a build script or a tsconfig it is a real
// package, the exemption dies, and the guard fails.
const NOT_COMPILED = ["@object-ui/example-hello-world"];

// Packages whose own `build` type-checks them, so a separate `type-check` script
// would only run the compiler twice. Unlike the `vite build` packages — which
// transpile without checking, the hole that caused #2911 — `next build` runs a
// full type-check unless `typescript.ignoreBuildErrors` is set.
//
// That escape hatch is exactly how this exemption could rot, so it is verified
// on every run rather than trusted: setting `ignoreBuildErrors` fails the guard.
const CHECKED_BY_OWN_BUILD = {
  "@object-ui/site": {
    build: "next build",
    // Caveat worth knowing: the `docs` CI job runs this build only when
    // `apps/site/` or `content/` changed (plus every push to main). A PR that
    // only touches a workspace package in `transpilePackages` therefore does
    // not re-check the site until it lands. Closing that would mean paying a
    // Next build on many more PRs — a cost/coverage call, not a silent gap.
    verifyNoIgnoreBuildErrors: "apps/site/next.config.mjs",
  },
};

// ── Known gaps: tests that nothing type-checks ───────────────────────────────
// Packages whose tests do not compile yet, so they cannot chain a
// `tsconfig.test.json` from `type-check`. Every entry is real debt: fix the
// errors, add the config, then delete the entry.
//
// Counts are MEASURED, not estimated — a temp tsconfig per package lifting only
// the test exclusion, against `main` at the time of the sweep, with the same
// template the wired-up packages use (`noEmit`, `composite: false`, `paths: {}`,
// plus `lib`/`types` where the tests need them). Errors resolvable in that
// config are excluded, so these are code-tier only.
//
// The two dominant codes are not random, and each has a playbook:
//   - TS2741/TS2739 "missing required properties" — authoring fixtures typed as
//     PARSED OUTPUT, whose `.default()` fields are required. Type them as the
//     input surface (`z.input`), the fix objectstack#4074 applied in `types`.
//   - TS2339/TS2353 "property does not exist" — implementation wider than the
//     type, the dialect problem. Declare the dialect next to the vocabulary it
//     extends (see scripts/check-spec-symbol-derivation.mjs), don't widen the
//     type to silence it.
const TEST_DEBT = {
  "@object-ui/core": { errors: 72, issue: 4118, note: "TS2741x32, TS2322x17 — mostly the input-vs-output fixture confusion" },
  "@object-ui/app-shell": { errors: 53, issue: 4118, note: "TS2339x24 — implementation wider than the type" },
  "@object-ui/components": { errors: 31, issue: 4118, note: "TS7006x12, TS7031x12 — untyped test callback params" },
  "@object-ui/react": { errors: 27, issue: 4118, note: "TS2769x9 — overload mismatch on render helpers" },
  "@object-ui/i18n": { errors: 13, issue: 4118, note: "TS2769x9" },
  "@object-ui/plugin-form": { errors: 12, issue: 4118, note: "TS2322x10" },
  "@object-ui/plugin-dashboard": { errors: 6, issue: 4118 },
  "@object-ui/plugin-list": { errors: 6, issue: 4118, note: "TS2353x3 — dialect keys" },
  "@object-ui/permissions": { errors: 5, issue: 4118, note: "TS2741x5" },
  "@object-ui/plugin-detail": { errors: 5, issue: 4118, note: "TS2353x3 — dialect keys" },
  "@object-ui/plugin-gantt": { errors: 3, issue: 4118 },
  "@object-ui/auth": { errors: 2, issue: 4118 },
  "@object-ui/plugin-chatbot": { errors: 2, issue: 4118 },
  "@object-ui/plugin-grid": { errors: 2, issue: 4118 },
  "@object-ui/plugin-map": { errors: 1, issue: 4118 },
};

// ── Collect workspace packages ───────────────────────────────────────────────
const GROUPS = ["packages", "apps", "examples"];

function countTestFiles(dir) {
  let n = 0;
  let entries;
  try {
    entries = readdirSync(dir, { withFileTypes: true });
  } catch {
    return 0;
  }
  for (const entry of entries) {
    if (entry.isDirectory()) {
      if (entry.name === "node_modules" || entry.name === "dist") continue;
      n += countTestFiles(join(dir, entry.name));
    } else if (/\.test\.tsx?$/.test(entry.name)) {
      n++;
    }
  }
  return n;
}

function collect() {
  const out = [];
  for (const group of GROUPS) {
    let entries;
    try {
      entries = readdirSync(resolve(root, group));
    } catch {
      continue;
    }
    for (const entry of entries) {
      const dir = join(group, entry);
      const manifest = resolve(root, dir, "package.json");
      try {
        statSync(manifest);
      } catch {
        continue;
      }
      const pkg = JSON.parse(readFileSync(manifest, "utf8"));
      if (!pkg.name) continue;
      let tsconfig = null;
      try {
        tsconfig = readFileSync(resolve(root, dir, "tsconfig.json"), "utf8");
      } catch {
        /* no tsconfig */
      }
      let testConfig = null;
      try {
        testConfig = readFileSync(resolve(root, dir, "tsconfig.test.json"), "utf8");
      } catch {
        /* no test config */
      }
      const typeCheck = pkg.scripts?.["type-check"] ?? "";
      out.push({
        name: pkg.name,
        dir,
        hasScript: Boolean(pkg.scripts?.["type-check"]),
        typeCheck,
        build: pkg.scripts?.build,
        hasBuild: Boolean(pkg.scripts?.build),
        hasTsconfig: tsconfig !== null,
        testFiles: countTestFiles(resolve(root, dir, "src")),
        // A package tsconfig that never excludes a test glob compiles its tests
        // as part of the build program, so they are already covered.
        buildExcludesTests: tsconfig !== null && /\*\.test\./.test(tsconfig),
        hasTestConfig: testConfig !== null,
        testConfig,
        // The config existing is not the same as anything running it — that gap
        // IS objectui#3009. `type-check` has to chain it.
        chainsTestConfig: /-p\s+tsconfig\.test\.json/.test(typeCheck),
      });
    }
  }
  return out;
}

const packages = collect();
const byName = new Map(packages.map((p) => [p.name, p]));

const errors = [];

// 1. Undeclared gap — a package born without a type-check script.
for (const pkg of packages) {
  if (pkg.hasScript) continue;
  if (DEBT[pkg.name] || NOT_COMPILED.includes(pkg.name) || CHECKED_BY_OWN_BUILD[pkg.name]) continue;
  errors.push(
    `${pkg.name} (${pkg.dir}) has no "type-check" script, so \`pnpm type-check\` skips it entirely.\n` +
      `      Add  "type-check": "tsc --noEmit"  to its package.json. If its types do not compile\n` +
      `      yet, add it to DEBT in scripts/check-type-check-coverage.mjs with an error count.`
  );
}

// 2. Ratchet — a declared gap that has been closed must leave the list.
for (const name of Object.keys(DEBT)) {
  const pkg = byName.get(name);
  if (!pkg) {
    errors.push(`${name} is listed in DEBT but is not a workspace package any more — delete the entry.`);
  } else if (pkg.hasScript) {
    errors.push(
      `${name} now has a "type-check" script — delete its DEBT entry so the gap cannot reopen` +
        `${DEBT[name].issue ? ` (and close #${DEBT[name].issue} if it is done)` : ""}.`
    );
  }
}

// 3. Ratchet — an exemption only holds while the package really is not compiled.
for (const name of NOT_COMPILED) {
  const pkg = byName.get(name);
  if (!pkg) {
    errors.push(`${name} is listed in NOT_COMPILED but is not a workspace package any more — delete the entry.`);
    continue;
  }
  if (pkg.hasScript) {
    errors.push(`${name} now has a "type-check" script — delete its NOT_COMPILED entry.`);
    continue;
  }
  const acquired = [pkg.hasBuild && "a build script", pkg.hasTsconfig && "a tsconfig.json"].filter(Boolean);
  if (acquired.length > 0) {
    errors.push(
      `${name} is listed in NOT_COMPILED but has gained ${acquired.join(" and ")} — it is a real package now.\n` +
        `      Add  "type-check": "tsc --noEmit"  and remove the exemption.`
    );
  }
}

// 4. Ratchet — "its own build checks it" only holds while that stays true.
for (const [name, spec] of Object.entries(CHECKED_BY_OWN_BUILD)) {
  const pkg = byName.get(name);
  if (!pkg) {
    errors.push(`${name} is listed in CHECKED_BY_OWN_BUILD but is not a workspace package any more — delete the entry.`);
    continue;
  }
  if (pkg.hasScript) {
    errors.push(`${name} now has a "type-check" script — delete its CHECKED_BY_OWN_BUILD entry.`);
    continue;
  }
  if (pkg.build !== spec.build) {
    errors.push(
      `${name} is exempt because its build is \`${spec.build}\`, which type-checks — but the build\n` +
        `      script is now \`${pkg.build}\`. Re-confirm it still type-checks, then update or drop the entry.`
    );
    continue;
  }
  // `next build` type-checks by default; `ignoreBuildErrors` silently disables
  // it, which would turn this exemption into exactly the hole #2911 was about.
  if (spec.verifyNoIgnoreBuildErrors) {
    const configPath = resolve(root, spec.verifyNoIgnoreBuildErrors);
    let config;
    try {
      config = readFileSync(configPath, "utf8");
    } catch {
      errors.push(
        `${name}: cannot read ${spec.verifyNoIgnoreBuildErrors}, so the exemption cannot be verified.\n` +
          `      Point verifyNoIgnoreBuildErrors at the real config, or drop the exemption.`
      );
      continue;
    }
    if (/ignoreBuildErrors\s*:\s*true/.test(config)) {
      errors.push(
        `${name} sets \`ignoreBuildErrors: true\` in ${spec.verifyNoIgnoreBuildErrors}, so \`${spec.build}\`\n` +
          `      no longer type-checks it and nothing else does either. Remove that flag, or add a\n` +
          `      "type-check" script and delete this exemption.`
      );
    }
  }
}

// ── 5. Tests are code too (objectstack#4118) ─────────────────────────────────
// Only asked of packages that are type-checked at all: one whose whole package
// is unchecked is already covered by DEBT above, and stacking a second entry on
// it would just make the same gap look like two.
const testsCovered = (pkg) => !pkg.buildExcludesTests || (pkg.hasTestConfig && pkg.chainsTestConfig);

for (const pkg of packages) {
  if (!pkg.hasScript || pkg.testFiles === 0) continue;

  // 5a. A `tsconfig.test.json` nothing runs is the objectui#3009 shape exactly:
  //     a file that looks like enforcement while no `tsc` invocation reads it.
  if (pkg.hasTestConfig && !pkg.chainsTestConfig) {
    errors.push(
      `${pkg.name} (${pkg.dir}) has a tsconfig.test.json that its "type-check" script never runs,\n` +
        `      so the file looks like enforcement while nothing compiles it — the exact shape of\n` +
        `      objectui#3009. Chain it:  "type-check": "tsc --noEmit && tsc -p tsconfig.test.json"`
    );
    continue;
  }

  // 5b. A config that emits, or that does not actually include the tests, would
  //     pass 5a while checking nothing.
  if (pkg.hasTestConfig && pkg.chainsTestConfig) {
    if (!/"noEmit"\s*:\s*true/.test(pkg.testConfig)) {
      errors.push(
        `${pkg.name} (${pkg.dir}): tsconfig.test.json must set "noEmit": true — it is a checking\n` +
          `      project, and emitting would put test output in the published dist.`
      );
    }
    if (!/\*\.test\./.test(pkg.testConfig)) {
      errors.push(
        `${pkg.name} (${pkg.dir}): tsconfig.test.json does not "include" any test glob, so it\n` +
          `      compiles nothing. Add  "include": ["src/**/*.test.ts", "src/**/*.test.tsx"]`
      );
    }
    continue;
  }

  // 5c. Undeclared gap — tests exist and nothing reads them.
  if (!testsCovered(pkg) && !TEST_DEBT[pkg.name]) {
    errors.push(
      `${pkg.name} (${pkg.dir}) has ${pkg.testFiles} test file${pkg.testFiles === 1 ? "" : "s"} that no \`tsc\`\n` +
        `      invocation reads: its tsconfig.json excludes them (correctly — it is the build config)\n` +
        `      and no tsconfig.test.json chains off "type-check". An unchecked test can assert a\n` +
        `      contract the compiler never checked, and then read as evidence that the contract holds.\n` +
        `      Add a tsconfig.test.json (see packages/types/tsconfig.test.json) and chain it, or add a\n` +
        `      TEST_DEBT entry with the measured error count.`
    );
  }
}

// 6. Ratchet — a declared test gap that has been closed must leave the list.
for (const [name, spec] of Object.entries(TEST_DEBT)) {
  const pkg = byName.get(name);
  if (!pkg) {
    errors.push(`${name} is listed in TEST_DEBT but is not a workspace package any more — delete the entry.`);
    continue;
  }
  if (pkg.testFiles === 0) {
    errors.push(`${name} is listed in TEST_DEBT but has no test files any more — delete the entry.`);
    continue;
  }
  // A package that is not type-checked AT ALL is already declared in DEBT. A
  // second entry here would make one gap read as two, and would survive the day
  // DEBT is paid off, so the test gap has to be re-measured then anyway.
  if (!pkg.hasScript) {
    errors.push(
      `${name} is listed in TEST_DEBT but has no "type-check" script, so its whole package is\n` +
        `      unchecked and DEBT already declares that. Delete the TEST_DEBT entry; add it back with\n` +
        `      a fresh measurement once the package type-checks at all.`
    );
    continue;
  }
  if (testsCovered(pkg)) {
    errors.push(
      `${name} type-checks its tests now — delete its TEST_DEBT entry so the gap cannot reopen` +
        `${spec.issue ? ` (and close #${spec.issue} if the list is empty)` : ""}.`
    );
  }
}

// ── Report ───────────────────────────────────────────────────────────────────
const checked = packages.filter((p) => p.hasScript).length;
const debtCount = Object.keys(DEBT).length;
const debtErrors = Object.values(DEBT).reduce((sum, d) => sum + d.errors, 0);
const byBuild = Object.keys(CHECKED_BY_OWN_BUILD).length;

const withTests = packages.filter((p) => p.hasScript && p.testFiles > 0);
const testsChecked = withTests.filter(testsCovered).length;
const testDebtErrors = Object.values(TEST_DEBT).reduce((sum, d) => sum + d.errors, 0);

if (errors.length === 0) {
  console.log(
    `✅  type-check coverage: ${checked}/${packages.length} via \`type-check\`, ` +
      `${byBuild} via their own build, ` +
      `${debtCount} known-broken (${debtErrors} errors outstanding), ` +
      `${NOT_COMPILED.length} not compiled.`
  );
  console.log(
    `✅  test type-check coverage: ${testsChecked}/${withTests.length} packages compile their tests, ` +
      `${Object.keys(TEST_DEBT).length} declared debt (${testDebtErrors} errors outstanding).`
  );
  process.exit(0);
}

console.error("❌  type-check coverage regressed:\n");
for (const message of errors) {
  console.error(`    • ${message}`);
}
console.error(
  "\nA package with no `type-check` script is not passing — turbo skips it and CI sees nothing.\n" +
    "An excluded TEST file is not passing either — nothing compiles it, so what it asserts about\n" +
    "a contract was never checked (objectstack#4118).\n" +
    "See https://github.com/objectstack-ai/objectui/issues/2911 for why this guard exists."
);
process.exit(1);