feat: add opt-in test coverage analysis tool (SchemaStore#5383)

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>

Author: miteshashar

.github/workflows/validate.yml

@@ -18,3 +18,4 @@ jobs:
       - run: 'npm run typecheck'
       - run: 'npm run eslint'
       - run: 'node ./cli.js check'
+      - run: 'node ./cli.js coverage'

CONTRIBUTING.md

@@ -40,6 +40,7 @@
   - [How to add a `$ref` to a JSON Schema that's hosted in this repository](#how-to-add-a-ref-to-a-json-schema-thats-hosted-in-this-repository)
   - [How to add a `$ref` to a JSON Schema that's self-hosted](#how-to-add-a-ref-to-a-json-schema-thats-self-hosted)
   - [How to validate a JSON Schema](#how-to-validate-a-json-schema)
+  - [How to check test coverage for a JSON Schema](#how-to-check-test-coverage-for-a-json-schema)
   - [How to ignore validation errors in a JSON Schema](#how-to-ignore-validation-errors-in-a-json-schema)
   - [How to name schemas that are subschemas (`partial-`)](#how-to-name-schemas-that-are-subschemas-partial-)
 - [Older Links](#older-links)
@@ -670,6 +671,39 @@ For example, to validate the [`ava.json`](https://github.com/SchemaStore/schemas
 
 Note that `<schemaName.json>` refers to the _filename_ that the schema has under `src/schemas/json`.
 
+### How to check test coverage for a JSON Schema
+
+The coverage tool analyzes how thoroughly your schema's test files exercise its constraints. It runs 8 checks:
+
+1. **Unused `$defs`** — flags `$defs`/`definitions` entries not referenced by any `$ref`
+2. **Description coverage** — flags properties missing a `description`
+3. **Test completeness** — checks that every top-level schema property appears in at least one positive test
+4. **Enum coverage** — checks that each enum value has positive test coverage and at least one invalid value in negative tests
+5. **Pattern coverage** — checks that each `pattern` constraint has a matching and a violating test value
+6. **Required field coverage** — checks that negative tests omit required fields
+7. **Default value coverage** — checks that positive tests include non-default values
+8. **Negative test isolation** — flags negative test files that test multiple unrelated violation types
+
+**Opting in:** Add your schema to the `coverage` array in `src/schema-validation.jsonc`:
+
+```jsonc
+"coverage": [
+  { "schema": "my-schema.json" },
+  { "schema": "my-strict-schema.json", "strict": true }
+]
+```
+
+- `strict` (default: `false`) — when `true`, coverage failures cause a non-zero exit code, enforced in CI.
+- Without `strict: true`, the tool reports findings but does not fail CI.
+
+**Running locally:**
+
+```console
+node ./cli.js coverage --schema-name=my-schema.json
+```
+
+Coverage is opt-in and runs in CI. Schemas with `strict: true` will block PRs on coverage failures. Schemas without `strict` get an advisory report only.
+
 ### How to ignore validation errors in a JSON Schema
 
 > **Note**

cli.js

@@ -19,6 +19,17 @@ import jsonlint from '@prantlf/jsonlint'
 import * as jsoncParser from 'jsonc-parser'
 import ora from 'ora'
 import chalk from 'chalk'
+import {
+  checkUnusedDefs,
+  checkDescriptionCoverage,
+  checkTestCompleteness,
+  checkEnumCoverage,
+  checkPatternCoverage,
+  checkRequiredCoverage,
+  checkDefaultCoverage,
+  checkNegativeIsolation,
+  printCoverageReport,
+} from './src/helpers/coverage.js'
 import minimist from 'minimist'
 import fetch, { FetchError } from 'node-fetch'
 import { execFile } from 'node:child_process'
@@ -144,6 +155,7 @@ if (argv.SchemaName) {
  * @property {string[]} highSchemaVersion
  * @property {string[]} missingCatalogUrl
  * @property {string[]} skiptest
+ * @property {{schema: string, strict?: boolean}[]} coverage
  * @property {string[]} catalogEntryNoLintNameOrDescription
  * @property {Record<string, SchemaValidationJsonOption>} options
  */
@@ -1481,6 +1493,10 @@ async function assertSchemaValidationJsonReferencesNoNonexistentFiles() {
   schemaNamesMustExist(SchemaValidation.skiptest, 'skiptest')
   schemaNamesMustExist(SchemaValidation.missingCatalogUrl, 'missingCatalogUrl')
   schemaNamesMustExist(SchemaValidation.highSchemaVersion, 'highSchemaVersion')
+  schemaNamesMustExist(
+    (SchemaValidation.coverage ?? []).map((c) => c.schema),
+    'coverage',
+  )
   for (const schemaName in SchemaValidation.options) {
     if (!SchemasToBeTested.includes(schemaName)) {
       printErrorAndExit(new Error(), [
@@ -2060,6 +2076,7 @@ TASKS:
   check-remote: Run all build checks for remote schemas
   maintenance: Run maintenance checks
   build-xregistry: Build the xRegistry from the catalog.json
+  coverage: Run test coverage analysis on opted-in schemas
 
 EXAMPLES:
   node ./cli.js check
@@ -2132,6 +2149,113 @@ EXAMPLES:
     }
   }
 
+  // ---------------------------------------------------------------------------
+  // Coverage task
+  // ---------------------------------------------------------------------------
+
+  async function taskCoverage() {
+    const coverageSchemas = SchemaValidation.coverage ?? []
+    if (coverageSchemas.length === 0) {
+      console.info(
+        'No schemas opted into coverage. Add schemas to "coverage" in schema-validation.jsonc',
+      )
+      return
+    }
+
+    const spinner = ora()
+    spinner.start()
+    let hasFailure = false
+    let hasMatch = false
+
+    for (const entry of coverageSchemas) {
+      const schemaName = entry.schema
+      const strict = entry.strict ?? false
+      if (argv['schema-name'] && argv['schema-name'] !== schemaName) {
+        continue
+      }
+      hasMatch = true
+
+      const schemaId = schemaName.replace('.json', '')
+      spinner.text = `Running coverage checks on "${schemaName}"${strict ? ' (strict)' : ''}`
+
+      // Load schema
+      const schemaFile = await toFile(path.join(SchemaDir, schemaName))
+      const schema = /** @type {Record<string, unknown>} */ (schemaFile.json)
+
+      // Load positive test files
+      const positiveTests = new Map()
+      const posDir = path.join(TestPositiveDir, schemaId)
+      for (const testfile of await fs.readdir(posDir).catch(() => [])) {
+        if (isIgnoredFile(testfile)) continue
+        const file = await toFile(path.join(posDir, testfile))
+        positiveTests.set(testfile, file.json)
+      }
+
+      // Load negative test files
+      const negativeTests = new Map()
+      const negDir = path.join(TestNegativeDir, schemaId)
+      for (const testfile of await fs.readdir(negDir).catch(() => [])) {
+        if (isIgnoredFile(testfile)) continue
+        const file = await toFile(path.join(negDir, testfile))
+        negativeTests.set(testfile, file.json)
+      }
+
+      // Run all 8 checks
+      const results = [
+        { name: '1. Unused $defs', result: checkUnusedDefs(schema) },
+        {
+          name: '2. Description Coverage',
+          result: checkDescriptionCoverage(schema),
+        },
+        {
+          name: '3. Test Completeness',
+          result: checkTestCompleteness(schema, positiveTests),
+        },
+        {
+          name: '4. Enum Coverage',
+          result: checkEnumCoverage(schema, positiveTests, negativeTests),
+        },
+        {
+          name: '5. Pattern Coverage',
+          result: checkPatternCoverage(schema, positiveTests, negativeTests),
+        },
+        {
+          name: '6. Required Field Coverage',
+          result: checkRequiredCoverage(schema, negativeTests),
+        },
+        {
+          name: '7. Default Value Coverage',
+          result: checkDefaultCoverage(schema, positiveTests),
+        },
+        {
+          name: '8. Negative Test Isolation',
+          result: checkNegativeIsolation(schema, negativeTests),
+        },
+      ]
+
+      spinner.stop()
+      printCoverageReport(schemaName, results)
+      if (strict && results.some((r) => r.result.status === 'fail'))
+        hasFailure = true
+
+      // Restart spinner for next schema
+      if (coverageSchemas.indexOf(entry) < coverageSchemas.length - 1) {
+        spinner.start()
+      }
+    }
+
+    if (!hasMatch) {
+      spinner.stop()
+      printErrorAndExit(null, [
+        `Schema "${argv['schema-name']}" is not in the coverage list in "${SchemaValidationFile}"`,
+      ])
+    }
+
+    if (hasFailure) {
+      process.exit(1)
+    }
+  }
+
   /** @type {Record<string, () => Promise<unknown>>} */
   const taskMap = {
     'new-schema': taskNewSchema,
@@ -2143,6 +2267,7 @@ EXAMPLES:
     maintenance: taskMaintenance,
     'build-website': taskBuildWebsite,
     'build-xregistry': taskBuildXRegistry,
+    coverage: taskCoverage,
     build: taskCheck, // Undocumented alias.
   }
   const taskOrFn = argv._[0]