feat(validator): add optional constraints parameter to analyzeCoverage

libraz · libraz · commit 1565c46d6aa0 · 2026-04-07T23:42:58.000+09:00
Previously, analyzing a constrained test suite reported
constraint-impossible tuples as uncovered, making it impossible to
distinguish real coverage gaps from impossible combinations. Analyzing
the output of a constrained generate() call could never yield
coverageRatio === 1.0 even when coverage was complete.

When constraints are supplied to analyzeCoverage, tuples that violate
any constraint are excluded from the coverage universe entirely — they
do not appear in totalTuples, coveredTuples, or uncovered. This matches
ExcludeInvalidTuples semantics used in the generator, so a generated
test suite analyzed against the same model + constraints always reports
full coverage.

Changes:
- src/validator/coverage_validator.{h,cpp}: add optional constraints
  vector to ValidateCoverage; exclude invalid tuples before scoring
- src/wasm/bindings.cpp: expose constraints arg in WASM binding
- js/types.ts: add constraints field to AnalyzeCoverageOptions
- js/index.ts, js/pure/index.ts: forward constraints through JS APIs
- src/ts/validator/coverage-validator.ts: pure TS validator updated
- docs/en/js-api.md, docs/ja/js-api.md, README.npm.md: document
  the new parameter

Tests: 240 C++ (ctest) + 536 JS (vitest) all pass
Version: 1.0.2 → 1.1.0 (backwards-compatible addition)
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1,5 +1,5 @@
 cmake_minimum_required(VERSION 3.16)
-project(coverwise VERSION 1.0.2 LANGUAGES CXX)
+project(coverwise VERSION 1.1.0 LANGUAGES CXX)
 
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
diff --git a/README.npm.md b/README.npm.md
@@ -89,7 +89,7 @@ const result = cw.generate({ parameters: [/* ... */] });
 | Method | Description |
 |--------|-------------|
 | `Coverwise.create()` | Create instance (loads WASM once) |
-| `cw.analyzeCoverage(params, tests, strength?)` | Measure t-wise coverage, list uncovered combinations |
+| `cw.analyzeCoverage(params, tests, strength?, constraints?)` | Measure t-wise coverage, list uncovered combinations (constraint-excluded tuples are removed from the universe) |
 | `cw.extendTests(existing, input)` | Add only the tests needed to close coverage gaps |
 | `cw.generate(input)` | Generate minimal covering array from scratch |
 | `cw.estimateModel(input)` | Preview model statistics |
diff --git a/docs/en/js-api.md b/docs/en/js-api.md
@@ -185,7 +185,7 @@ interface ClassCoverage {
 }
 ```
 
-## `analyzeCoverage(parameters, tests, strength?)`
+## `analyzeCoverage(parameters, tests, strength?, constraints?)`
 
 Analyze the t-wise coverage of an existing test suite. Independent of the generator — validates any set of tests.
 
@@ -194,9 +194,12 @@ function analyzeCoverage(
   parameters: Parameter[],
   tests: TestCase[],
   strength?: number,        // Default: 2
+  constraints?: string[],   // Optional constraint DSL strings
 ): CoverageReport
 ```
 
+When `constraints` is supplied, tuples that violate any constraint are **removed from the coverage universe entirely** — they do not count toward `totalTuples`, `coveredTuples`, or `uncovered`. This matches the generator's semantics, so analyzing a generated suite always yields `coverageRatio === 1.0`.
+
 ### CoverageReport
 
 ```typescript
diff --git a/docs/ja/js-api.md b/docs/ja/js-api.md
@@ -185,7 +185,7 @@ interface ClassCoverage {
 }
 ```
 
-## `analyzeCoverage(parameters, tests, strength?)`
+## `analyzeCoverage(parameters, tests, strength?, constraints?)`
 
 既存テストスイートの t-wise カバレッジを分析します。ジェネレータとは独立しており、任意のテストセットを検証できます。
 
@@ -194,9 +194,12 @@ function analyzeCoverage(
   parameters: Parameter[],
   tests: TestCase[],
   strength?: number,        // デフォルト: 2
+  constraints?: string[],   // 制約 DSL 文字列(省略可)
 ): CoverageReport
 ```
 
+`constraints` を渡すと、制約に違反する tuple は **カバレッジ universe から完全に除外**されます(`totalTuples` / `coveredTuples` / `uncovered` のいずれにも計上されません)。これはジェネレータと同じセマンティクスで、生成済みテスト集合を解析すると常に `coverageRatio === 1.0` になります。
+
 ### CoverageReport
 
 ```typescript
diff --git a/js/index.test.ts b/js/index.test.ts
@@ -197,6 +197,23 @@ describe('analyzeCoverage()', () => {
     // 2-wise for 2 params = 4 tuples
     expect(report.totalTuples).toBe(4);
   });
+
+  it('excludes constraint-invalid tuples from the coverage universe', () => {
+    const params: Parameter[] = [
+      { name: 'os', values: ['win', 'mac'] },
+      { name: 'browser', values: ['chrome', 'ie'] },
+    ];
+    // IF os=mac THEN browser!=ie -> removes (mac, ie). 3 valid tuples remain.
+    const tests: TestCase[] = [
+      { os: 'win', browser: 'chrome' },
+      { os: 'mac', browser: 'chrome' },
+    ];
+    const report = analyzeCoverage(params, tests, 2, ['IF os=mac THEN browser!=ie']);
+    expect(report.totalTuples).toBe(3);
+    expect(report.coveredTuples).toBe(2);
+    expect(report.uncovered).toHaveLength(1);
+    expect(report.uncovered[0].tuple).toEqual(expect.arrayContaining(['os=win', 'browser=ie']));
+  });
 });
 
 describe('extendTests()', () => {
diff --git a/js/index.ts b/js/index.ts
@@ -50,6 +50,7 @@ interface WasmModule {
     params: Parameter[],
     tests: TestCase[],
     strength: number,
+    constraints?: string[],
   ): CoverageReport | { error: true; code?: string; message?: string };
   extendTests(
     existing: TestCase[],
@@ -151,9 +152,12 @@ export function analyzeCoverage(
   parameters: Parameter[],
   tests: TestCase[],
   strength?: number,
+  constraints?: string[],
 ): CoverageReport {
   const mod = getModule();
-  const result = checkResult<CoverageReport>(mod.analyzeCoverage(parameters, tests, strength ?? 2));
+  const result = checkResult<CoverageReport>(
+    mod.analyzeCoverage(parameters, tests, strength ?? 2, constraints ?? []),
+  );
   // When there are no tuples (e.g. fewer parameters than strength), coverage is vacuously 1.0.
   if (result.totalTuples === 0) {
     result.coverageRatio = 1.0;
@@ -225,9 +229,14 @@ export class Coverwise {
   /**
    * Analyze t-wise coverage of an existing test suite.
    */
-  analyzeCoverage(parameters: Parameter[], tests: TestCase[], strength?: number): CoverageReport {
+  analyzeCoverage(
+    parameters: Parameter[],
+    tests: TestCase[],
+    strength?: number,
+    constraints?: string[],
+  ): CoverageReport {
     const result = checkResult<CoverageReport>(
-      this.module.analyzeCoverage(parameters, tests, strength ?? 2),
+      this.module.analyzeCoverage(parameters, tests, strength ?? 2, constraints ?? []),
     );
     if (result.totalTuples === 0) {
       result.coverageRatio = 1.0;
diff --git a/js/pure/index.ts b/js/pure/index.ts
@@ -33,6 +33,8 @@ import {
   extend as internalExtend,
   generate as internalGenerate,
 } from '../../src/ts/core/generator.js';
+import type { ConstraintNode } from '../../src/ts/model/constraint-ast.js';
+import { parseConstraint } from '../../src/ts/model/constraint-parser.js';
 import {
   annotateClassCoverage,
   validateCoverage as internalValidateCoverage,
@@ -144,12 +146,30 @@ export function analyzeCoverage(
   parameters: Parameter[],
   tests: TestCase[],
   strength?: number,
+  constraints?: string[],
 ): CoverageReport {
   validateParameters(parameters);
   const s = validateStrength(strength);
   const params = toInternalParams(parameters);
   const internalTests = tests.map((tc) => toInternalTestCase(tc, params));
-  const report = internalValidateCoverage(params, internalTests, s);
+
+  // Parse optional constraint expressions.
+  const parsedConstraints: ConstraintNode[] = [];
+  if (constraints && constraints.length > 0) {
+    for (const expr of constraints) {
+      const parseResult = parseConstraint(expr, params);
+      if (parseResult.error.code !== 0 || !parseResult.constraint) {
+        throw new Error(
+          `Invalid constraint "${expr}": ${parseResult.error.message}${
+            parseResult.error.detail ? ` — ${parseResult.error.detail}` : ''
+          }`,
+        );
+      }
+      parsedConstraints.push(parseResult.constraint);
+    }
+  }
+
+  const report = internalValidateCoverage(params, internalTests, s, parsedConstraints);
   const result = toPublicCoverageReport(report);
   // When there are no tuples, coverage is vacuously 1.0.
   if (result.totalTuples === 0) {
@@ -219,8 +239,13 @@ export class Coverwise {
   /**
    * Analyze t-wise coverage of an existing test suite.
    */
-  analyzeCoverage(parameters: Parameter[], tests: TestCase[], strength?: number): CoverageReport {
-    return analyzeCoverage(parameters, tests, strength);
+  analyzeCoverage(
+    parameters: Parameter[],
+    tests: TestCase[],
+    strength?: number,
+    constraints?: string[],
+  ): CoverageReport {
+    return analyzeCoverage(parameters, tests, strength, constraints);
   }
 
   /**
diff --git a/js/types.ts b/js/types.ts
@@ -49,7 +49,14 @@ export interface UncoveredTuple {
   tuple: string[];
   /** Parameter names involved. */
   params: string[];
-  /** Why this tuple is uncovered. */
+  /**
+   * Why this tuple is uncovered.
+   *
+   * Currently always `"never covered"` — tuples that are impossible due to
+   * constraints are removed from the coverage universe entirely (they do not
+   * appear here and are not counted in `totalTuples`), matching the
+   * generator's `excludeInvalidTuples` semantics.
+   */
   reason: string;
   /** Human-readable display string. */
   display: string;
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@libraz/coverwise",
-  "version": "1.0.2",
+  "version": "1.1.0",
   "type": "module",
   "packageManager": "yarn@4.12.0",
   "description": "Combinatorial test coverage engine — analyze, generate, and extend t-wise test suites via WASM",
diff --git a/src/core/generator.cpp b/src/core/generator.cpp
@@ -320,12 +320,10 @@ model::GenerateResult Generate(const GenerateOptions& options) {
   if (!AllComplete(coverage, sub_engines)) {
     if (opts.max_tests > 0 && result.tests.size() >= static_cast<size_t>(opts.max_tests)) {
       result.warnings.push_back("Generation stopped at max_tests (" +
-                                std::to_string(opts.max_tests) +
-                                ") before reaching 100% coverage");
+                                std::to_string(opts.max_tests) + ") before reaching 100% coverage");
     } else {
-      result.warnings.push_back(
-          "Generation stopped before reaching 100% coverage after " +
-          std::to_string(kMaxRetries) + " consecutive zero-score candidates");
+      result.warnings.push_back("Generation stopped before reaching 100% coverage after " +
+                                std::to_string(kMaxRetries) + " consecutive zero-score candidates");
     }
   }
 
diff --git a/src/model/constraint_parser.cpp b/src/model/constraint_parser.cpp
@@ -235,8 +235,7 @@ TokenizeResult Tokenize(const std::string& expr) {
       if (j >= len) {
         return {{},
                 {Error::Code::kConstraintError,
-                 "Unterminated string literal starting at position " + std::to_string(start),
-                 ""}};
+                 "Unterminated string literal starting at position " + std::to_string(start), ""}};
       }
       std::string content = expr.substr(i + 1, j - i - 1);
       tokens.push_back({TokenType::kIdentifier, content, start});
diff --git a/src/ts/validator/coverage-validator.test.ts b/src/ts/validator/coverage-validator.test.ts
@@ -1,4 +1,5 @@
 import { describe, expect, it } from 'vitest';
+import { parseConstraint } from '../model/constraint-parser.js';
 import { Parameter } from '../model/parameter.js';
 import type { GenerateResult, TestCase } from '../model/test-case.js';
 import { createGenerateResult } from '../model/test-case.js';
@@ -139,6 +140,35 @@ describe('validateCoverage', () => {
       expect(u.reason).toBe('never covered');
     }
   });
+
+  it('excludes constraint-invalid tuples from the universe', () => {
+    // os = {win, mac}, browser = {chrome, ie}
+    // Constraint: IF os=mac THEN browser!=ie
+    //   removes (os=mac, browser=ie) from the universe -> 3 valid tuples.
+    const params = [
+      new Parameter('os', ['win', 'mac']),
+      new Parameter('browser', ['chrome', 'ie']),
+    ];
+    const parse = parseConstraint('IF os=mac THEN browser!=ie', params);
+    expect(parse.error.code).toBe(0);
+    expect(parse.constraint).toBeDefined();
+
+    // Cover 2 of the 3 valid tuples; (win, ie) is missing.
+    const tests: TestCase[] = [
+      { values: [0, 0] }, // win, chrome
+      { values: [1, 0] }, // mac, chrome
+    ];
+
+    const report = validateCoverage(params, tests, 2, [parse.constraint!]);
+    expect(report.totalTuples).toBe(3);
+    expect(report.coveredTuples).toBe(2);
+    expect(report.uncovered).toHaveLength(1);
+    expect(report.uncovered[0].tuple).toEqual(expect.arrayContaining(['os=win', 'browser=ie']));
+    // The excluded tuple must NOT appear as uncovered.
+    for (const u of report.uncovered) {
+      expect(u.tuple).not.toEqual(expect.arrayContaining(['os=mac', 'browser=ie']));
+    }
+  });
 });
 
 describe('validateCoverage with invalid value indices', () => {
diff --git a/src/ts/validator/coverage-validator.ts b/src/ts/validator/coverage-validator.ts
@@ -1,7 +1,13 @@
 /// Independent coverage validation (does NOT depend on generator/core).
 
+import { type ConstraintNode, ConstraintResult } from '../model/constraint-ast.js';
 import type { Parameter } from '../model/parameter.js';
-import type { GenerateResult, TestCase, UncoveredTuple } from '../model/test-case.js';
+import {
+  type GenerateResult,
+  type TestCase,
+  UNASSIGNED,
+  type UncoveredTuple,
+} from '../model/test-case.js';
 
 /** Coverage validation report with human-readable uncovered tuples. */
 export interface CoverageReport {
@@ -68,6 +74,7 @@ export function validateCoverage(
   params: Parameter[],
   tests: TestCase[],
   strength: number,
+  constraints: ConstraintNode[] = [],
 ): CoverageReport {
   const report: CoverageReport = {
     totalTuples: 0,
@@ -88,6 +95,9 @@ export function validateCoverage(
   // Step 1: Generate all C(n, strength) combinations of parameter indices.
   const combinations = generateCombinations(n, strength);
 
+  // Reusable assignment buffer for constraint evaluation.
+  const assignment = new Array<number>(n).fill(UNASSIGNED);
+
   for (const combo of combinations) {
     // Step 2: Enumerate all value tuples (cartesian product) for this combination.
     let numTuples = 1;
@@ -106,6 +116,27 @@ export function validateCoverage(
         remainder = Math.trunc(remainder / radix);
       }
 
+      // Step 2b: Exclude constraint-invalid tuples from the universe entirely
+      // (matches CoverageEngine.excludeInvalidTuples).
+      if (constraints.length > 0) {
+        for (let i = 0; i < strength; ++i) {
+          assignment[combo[i]] = valueIndices[i];
+        }
+        let excluded = false;
+        for (const c of constraints) {
+          if (c.evaluate(assignment) === ConstraintResult.False) {
+            excluded = true;
+            break;
+          }
+        }
+        for (let i = 0; i < strength; ++i) {
+          assignment[combo[i]] = UNASSIGNED;
+        }
+        if (excluded) {
+          continue;
+        }
+      }
+
       ++report.totalTuples;
 
       // Step 3: Check if any test case covers this value tuple.
diff --git a/src/validator/coverage_validator.cpp b/src/validator/coverage_validator.cpp
@@ -35,7 +35,8 @@ bool TestCovers(const model::TestCase& test, const std::vector<uint32_t>& param_
 }  // namespace
 
 CoverageReport ValidateCoverage(const std::vector<model::Parameter>& params,
-                                const std::vector<model::TestCase>& tests, uint32_t strength) {
+                                const std::vector<model::TestCase>& tests, uint32_t strength,
+                                const std::vector<model::Constraint>& constraints) {
   CoverageReport report;
   uint32_t n = static_cast<uint32_t>(params.size());
 
@@ -49,6 +50,9 @@ CoverageReport ValidateCoverage(const std::vector<model::Parameter>& params,
   // Step 1: Generate all C(n, strength) combinations of parameter indices.
   auto combinations = util::GenerateCombinations(n, strength);
 
+  // Reusable assignment buffer for constraint evaluation.
+  std::vector<uint32_t> assignment(n, model::kUnassigned);
+
   for (const auto& combo : combinations) {
     // Step 2: Enumerate all value tuples (cartesian product) for this combination.
     // Compute the number of tuples for this combination.
@@ -68,6 +72,29 @@ CoverageReport ValidateCoverage(const std::vector<model::Parameter>& params,
         remainder /= radix;
       }
 
+      // Step 2b: If any constraint marks this partial assignment as kFalse,
+      // exclude this tuple from the coverage universe entirely (matches the
+      // generator's CoverageEngine::ExcludeInvalidTuples semantics).
+      if (!constraints.empty()) {
+        for (uint32_t j = 0; j < strength; ++j) {
+          assignment[combo[j]] = value_indices[j];
+        }
+        bool excluded = false;
+        for (const auto& c : constraints) {
+          if (c->Evaluate(assignment) == model::ConstraintResult::kFalse) {
+            excluded = true;
+            break;
+          }
+        }
+        // Reset assignment for reuse.
+        for (uint32_t j = 0; j < strength; ++j) {
+          assignment[combo[j]] = model::kUnassigned;
+        }
+        if (excluded) {
+          continue;
+        }
+      }
+
       ++report.total_tuples;
 
       // Step 3: Check if any test case covers this value tuple.
diff --git a/src/validator/coverage_validator.h b/src/validator/coverage_validator.h
diff --git a/src/wasm/bindings.cpp b/src/wasm/bindings.cpp
diff --git a/tests/core/generator_test.cpp b/tests/core/generator_test.cpp
diff --git a/tests/integration/benchmark_test.cpp b/tests/integration/benchmark_test.cpp
diff --git a/tests/model/constraint_parser_test.cpp b/tests/model/constraint_parser_test.cpp
diff --git a/tests/validator/coverage_validator_test.cpp b/tests/validator/coverage_validator_test.cpp

Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"name": "@libraz/coverwise",`
`3`		`- "version": "1.0.2",`
	`3`	`+ "version": "1.1.0",`
`4`	`4`	`"type": "module",`
`5`	`5`	`"packageManager": "yarn@4.12.0",`
`6`	`6`	`"description": "Combinatorial test coverage engine — analyze, generate, and extend t-wise test suites via WASM",`